diff --git a/agents/addon/extension/chat_transcriber/extension.go b/agents/addon/extension/chat_transcriber/extension.go deleted file mode 100644 index 33fba48a..00000000 --- a/agents/addon/extension/chat_transcriber/extension.go +++ /dev/null @@ -1,147 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by Wei Hu in 2022-10. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -package extension - -import ( - "chat_transcriber/pb" - "fmt" - "log/slog" - "time" - - "agora.io/rte/rtego" - "google.golang.org/protobuf/proto" -) - -const ( - textDataTextField = "text" - textDataFinalField = "is_final" - textDataStreamIdField = "stream_id" - textDataEndOfSegmentField = "end_of_segment" -) - -var ( - logTag = slog.String("extension", "CHAT_TRANSCRIBER_EXTENSION") -) - -type chatTranscriberExtension struct { - rtego.DefaultExtension - - cachedTextMap map[uint32]string // record the cached text data for each stream id -} - -func newExtension(name string) rtego.Extension { - return &chatTranscriberExtension{ - cachedTextMap: make(map[uint32]string), - } -} - -// OnData receives data from rte graph. -// current supported data: -// - name: text_data -// example: -// {"name": "text_data", "properties": {"text": "hello", "is_final": true, "stream_id": 123, "end_of_segment": true}} -func (p *chatTranscriberExtension) OnData( - rte rtego.Rte, - data rtego.Data, -) { - // Get the text data from data. - text, err := data.GetPropertyString(textDataTextField) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s error: %v", textDataTextField, err), logTag) - return - } - - // Get the 'is_final' flag from data which indicates whether the text is final, - // otherwise it could be overwritten by the next text. - final, err := data.GetPropertyBool(textDataFinalField) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s error: %v", textDataFinalField, err), logTag) - return - } - - // Get the stream id from data. - streamId, err := data.GetPropertyUint32(textDataStreamIdField) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s error: %v", textDataStreamIdField, err), logTag) - return - } - - // Get the 'end_of_segment' flag from data which indicates whether a line break is needed. - endOfSegment, err := data.GetPropertyBool(textDataEndOfSegmentField) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s error: %v", textDataEndOfSegmentField, err), logTag) - return - } - - slog.Debug(fmt.Sprintf( - "OnData %s: %s %s: %t %s: %d %s: %t", - textDataTextField, - text, - textDataFinalField, - final, - textDataStreamIdField, - streamId, - textDataEndOfSegmentField, - endOfSegment), logTag) - - // We cache all final text data and append the non-final text data to the cached data - // until the end of the segment. - if endOfSegment { - if cachedText, ok := p.cachedTextMap[streamId]; ok { - text = cachedText + text - delete(p.cachedTextMap, streamId) - } - } else { - if final { - if cachedText, ok := p.cachedTextMap[streamId]; ok { - text = cachedText + text - p.cachedTextMap[streamId] = text - } else { - p.cachedTextMap[streamId] = text - } - } - } - - pb := pb.Text{ - Uid: int32(streamId), - DataType: "transcribe", - Texttime: time.Now().UnixMilli(), - Words: []*pb.Word{ - { - Text: text, - IsFinal: endOfSegment, - }, - }, - } - - pbData, err := proto.Marshal(&pb) - if err != nil { - slog.Warn(fmt.Sprintf("OnData Marshal error: %v", err), logTag) - return - } - - // convert the origin text data to the protobuf data and send it to the graph. - rteData, err := rtego.NewData("data") - rteData.SetPropertyBytes("data", pbData) - if err != nil { - slog.Warn(fmt.Sprintf("OnData NewData error: %v", err), logTag) - return - } - - rte.SendData(rteData) -} - -func init() { - slog.Info("chat_transcriber extension init", logTag) - - // Register addon - rtego.RegisterAddonAsExtension( - "chat_transcriber", - rtego.NewDefaultExtensionAddon(newExtension), - ) -} diff --git a/agents/addon/extension/chat_transcriber/go.mod b/agents/addon/extension/chat_transcriber/go.mod deleted file mode 100644 index 311ff8aa..00000000 --- a/agents/addon/extension/chat_transcriber/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module chat_transcriber - -go 1.18 - -replace agora.io/rte => ../../../interface - -require ( - agora.io/rte v0.0.0-00010101000000-000000000000 - google.golang.org/protobuf v1.34.2 -) diff --git a/agents/addon/extension/chat_transcriber/go.sum b/agents/addon/extension/chat_transcriber/go.sum deleted file mode 100644 index 73d32b16..00000000 --- a/agents/addon/extension/chat_transcriber/go.sum +++ /dev/null @@ -1,4 +0,0 @@ -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/agents/addon/extension/chat_transcriber/manifest.json b/agents/addon/extension/chat_transcriber/manifest.json deleted file mode 100644 index 269a5a53..00000000 --- a/agents/addon/extension/chat_transcriber/manifest.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "type": "extension", - "name": "chat_transcriber", - "version": "0.1.0", - "language": "go", - "dependencies": [ - { - "type": "system", - "name": "rte_runtime", - "version": "0.1.0" - }, - { - "type": "system", - "name": "rte_runtime_go", - "version": "0.1.0" - } - ], - "api": { - "property": {}, - "data_in": [ - { - "name": "text_data", - "property": { - "text": { - "type": "string" - }, - "is_final": { - "type": "bool" - }, - "stream_id": { - "type": "uint32" - }, - "end_of_segment": { - "type": "bool" - } - } - } - ], - "data_out": [ - { - "name": "data" - } - ] - } -} \ No newline at end of file diff --git a/agents/addon/extension/chat_transcriber/pb/chat_text.pb.go b/agents/addon/extension/chat_transcriber/pb/chat_text.pb.go deleted file mode 100644 index 034473fd..00000000 --- a/agents/addon/extension/chat_transcriber/pb/chat_text.pb.go +++ /dev/null @@ -1,475 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc (unknown) -// source: chat_text.proto - -package pb - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Text struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Vendor int32 `protobuf:"varint,1,opt,name=vendor,proto3" json:"vendor,omitempty"` - Version int32 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` - Seqnum int32 `protobuf:"varint,3,opt,name=seqnum,proto3" json:"seqnum,omitempty"` - Uid int32 `protobuf:"varint,4,opt,name=uid,proto3" json:"uid,omitempty"` - Flag int32 `protobuf:"varint,5,opt,name=flag,proto3" json:"flag,omitempty"` - Time int64 `protobuf:"varint,6,opt,name=time,proto3" json:"time,omitempty"` // final time =first nofinal time - Lang int32 `protobuf:"varint,7,opt,name=lang,proto3" json:"lang,omitempty"` - Starttime int32 `protobuf:"varint,8,opt,name=starttime,proto3" json:"starttime,omitempty"` - Offtime int32 `protobuf:"varint,9,opt,name=offtime,proto3" json:"offtime,omitempty"` - Words []*Word `protobuf:"bytes,10,rep,name=words,proto3" json:"words,omitempty"` - EndOfSegment bool `protobuf:"varint,11,opt,name=end_of_segment,json=endOfSegment,proto3" json:"end_of_segment,omitempty"` - DurationMs int32 `protobuf:"varint,12,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` - DataType string `protobuf:"bytes,13,opt,name=data_type,json=dataType,proto3" json:"data_type,omitempty"` // transcribe ,translate - Trans []*Translation `protobuf:"bytes,14,rep,name=trans,proto3" json:"trans,omitempty"` - Culture string `protobuf:"bytes,15,opt,name=culture,proto3" json:"culture,omitempty"` - Texttime int64 `protobuf:"varint,16,opt,name=texttime,proto3" json:"texttime,omitempty"` // pkg timestamp -} - -func (x *Text) Reset() { - *x = Text{} - if protoimpl.UnsafeEnabled { - mi := &file_chat_text_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Text) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Text) ProtoMessage() {} - -func (x *Text) ProtoReflect() protoreflect.Message { - mi := &file_chat_text_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Text.ProtoReflect.Descriptor instead. -func (*Text) Descriptor() ([]byte, []int) { - return file_chat_text_proto_rawDescGZIP(), []int{0} -} - -func (x *Text) GetVendor() int32 { - if x != nil { - return x.Vendor - } - return 0 -} - -func (x *Text) GetVersion() int32 { - if x != nil { - return x.Version - } - return 0 -} - -func (x *Text) GetSeqnum() int32 { - if x != nil { - return x.Seqnum - } - return 0 -} - -func (x *Text) GetUid() int32 { - if x != nil { - return x.Uid - } - return 0 -} - -func (x *Text) GetFlag() int32 { - if x != nil { - return x.Flag - } - return 0 -} - -func (x *Text) GetTime() int64 { - if x != nil { - return x.Time - } - return 0 -} - -func (x *Text) GetLang() int32 { - if x != nil { - return x.Lang - } - return 0 -} - -func (x *Text) GetStarttime() int32 { - if x != nil { - return x.Starttime - } - return 0 -} - -func (x *Text) GetOfftime() int32 { - if x != nil { - return x.Offtime - } - return 0 -} - -func (x *Text) GetWords() []*Word { - if x != nil { - return x.Words - } - return nil -} - -func (x *Text) GetEndOfSegment() bool { - if x != nil { - return x.EndOfSegment - } - return false -} - -func (x *Text) GetDurationMs() int32 { - if x != nil { - return x.DurationMs - } - return 0 -} - -func (x *Text) GetDataType() string { - if x != nil { - return x.DataType - } - return "" -} - -func (x *Text) GetTrans() []*Translation { - if x != nil { - return x.Trans - } - return nil -} - -func (x *Text) GetCulture() string { - if x != nil { - return x.Culture - } - return "" -} - -func (x *Text) GetTexttime() int64 { - if x != nil { - return x.Texttime - } - return 0 -} - -type Word struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Text string `protobuf:"bytes,1,opt,name=text,proto3" json:"text,omitempty"` - StartMs int32 `protobuf:"varint,2,opt,name=start_ms,json=startMs,proto3" json:"start_ms,omitempty"` - DurationMs int32 `protobuf:"varint,3,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` - IsFinal bool `protobuf:"varint,4,opt,name=is_final,json=isFinal,proto3" json:"is_final,omitempty"` - Confidence float64 `protobuf:"fixed64,5,opt,name=confidence,proto3" json:"confidence,omitempty"` -} - -func (x *Word) Reset() { - *x = Word{} - if protoimpl.UnsafeEnabled { - mi := &file_chat_text_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Word) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Word) ProtoMessage() {} - -func (x *Word) ProtoReflect() protoreflect.Message { - mi := &file_chat_text_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Word.ProtoReflect.Descriptor instead. -func (*Word) Descriptor() ([]byte, []int) { - return file_chat_text_proto_rawDescGZIP(), []int{1} -} - -func (x *Word) GetText() string { - if x != nil { - return x.Text - } - return "" -} - -func (x *Word) GetStartMs() int32 { - if x != nil { - return x.StartMs - } - return 0 -} - -func (x *Word) GetDurationMs() int32 { - if x != nil { - return x.DurationMs - } - return 0 -} - -func (x *Word) GetIsFinal() bool { - if x != nil { - return x.IsFinal - } - return false -} - -func (x *Word) GetConfidence() float64 { - if x != nil { - return x.Confidence - } - return 0 -} - -type Translation struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - IsFinal bool `protobuf:"varint,1,opt,name=is_final,json=isFinal,proto3" json:"is_final,omitempty"` - Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` - Texts []string `protobuf:"bytes,3,rep,name=texts,proto3" json:"texts,omitempty"` -} - -func (x *Translation) Reset() { - *x = Translation{} - if protoimpl.UnsafeEnabled { - mi := &file_chat_text_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Translation) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Translation) ProtoMessage() {} - -func (x *Translation) ProtoReflect() protoreflect.Message { - mi := &file_chat_text_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Translation.ProtoReflect.Descriptor instead. -func (*Translation) Descriptor() ([]byte, []int) { - return file_chat_text_proto_rawDescGZIP(), []int{2} -} - -func (x *Translation) GetIsFinal() bool { - if x != nil { - return x.IsFinal - } - return false -} - -func (x *Translation) GetLang() string { - if x != nil { - return x.Lang - } - return "" -} - -func (x *Translation) GetTexts() []string { - if x != nil { - return x.Texts - } - return nil -} - -var File_chat_text_proto protoreflect.FileDescriptor - -var file_chat_text_proto_rawDesc = []byte{ - 0x0a, 0x0f, 0x63, 0x68, 0x61, 0x74, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x16, 0x61, 0x67, 0x6f, 0x72, 0x61, 0x2e, 0x63, 0x68, 0x61, 0x74, 0x5f, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x22, 0xdf, 0x03, 0x0a, 0x04, 0x54, 0x65, - 0x78, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x65, 0x71, 0x6e, 0x75, 0x6d, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x65, 0x71, 0x6e, 0x75, 0x6d, 0x12, 0x10, 0x0a, 0x03, - 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x66, 0x6c, 0x61, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x66, 0x6c, - 0x61, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x66, 0x66, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6f, 0x66, 0x66, 0x74, 0x69, - 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x61, 0x67, 0x6f, 0x72, 0x61, 0x2e, 0x63, 0x68, 0x61, 0x74, 0x5f, 0x74, - 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x2e, 0x57, 0x6f, 0x72, 0x64, 0x52, - 0x05, 0x77, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x65, 0x6e, 0x64, 0x5f, 0x6f, 0x66, - 0x5f, 0x73, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x65, 0x6e, 0x64, 0x4f, 0x66, 0x53, 0x65, 0x67, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, - 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x0a, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x73, 0x12, 0x1b, 0x0a, - 0x09, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x05, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x61, 0x67, 0x6f, 0x72, - 0x61, 0x2e, 0x63, 0x68, 0x61, 0x74, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x63, 0x72, 0x69, 0x62, - 0x65, 0x72, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x6c, 0x74, 0x75, 0x72, 0x65, - 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x75, 0x6c, 0x74, 0x75, 0x72, 0x65, 0x12, - 0x1a, 0x0a, 0x08, 0x74, 0x65, 0x78, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x10, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x08, 0x74, 0x65, 0x78, 0x74, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x91, 0x01, 0x0a, 0x04, - 0x57, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x4d, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x61, 0x6c, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x12, - 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x22, - 0x52, 0x0a, 0x0b, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x19, - 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x69, 0x73, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, - 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x14, 0x0a, - 0x05, 0x74, 0x65, 0x78, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x74, 0x65, - 0x78, 0x74, 0x73, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x3b, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, -} - -var ( - file_chat_text_proto_rawDescOnce sync.Once - file_chat_text_proto_rawDescData = file_chat_text_proto_rawDesc -) - -func file_chat_text_proto_rawDescGZIP() []byte { - file_chat_text_proto_rawDescOnce.Do(func() { - file_chat_text_proto_rawDescData = protoimpl.X.CompressGZIP(file_chat_text_proto_rawDescData) - }) - return file_chat_text_proto_rawDescData -} - -var file_chat_text_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_chat_text_proto_goTypes = []interface{}{ - (*Text)(nil), // 0: agora.chat_transcriber.Text - (*Word)(nil), // 1: agora.chat_transcriber.Word - (*Translation)(nil), // 2: agora.chat_transcriber.Translation -} -var file_chat_text_proto_depIdxs = []int32{ - 1, // 0: agora.chat_transcriber.Text.words:type_name -> agora.chat_transcriber.Word - 2, // 1: agora.chat_transcriber.Text.trans:type_name -> agora.chat_transcriber.Translation - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name -} - -func init() { file_chat_text_proto_init() } -func file_chat_text_proto_init() { - if File_chat_text_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_chat_text_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Text); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_chat_text_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Word); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_chat_text_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Translation); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_chat_text_proto_rawDesc, - NumEnums: 0, - NumMessages: 3, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_chat_text_proto_goTypes, - DependencyIndexes: file_chat_text_proto_depIdxs, - MessageInfos: file_chat_text_proto_msgTypes, - }.Build() - File_chat_text_proto = out.File - file_chat_text_proto_rawDesc = nil - file_chat_text_proto_goTypes = nil - file_chat_text_proto_depIdxs = nil -} diff --git a/agents/addon/extension/chat_transcriber/pb/chat_text.proto b/agents/addon/extension/chat_transcriber/pb/chat_text.proto deleted file mode 100644 index 9ee4e504..00000000 --- a/agents/addon/extension/chat_transcriber/pb/chat_text.proto +++ /dev/null @@ -1,37 +0,0 @@ -syntax = "proto3"; - -package agora.chat_transcriber; -option go_package = ".;pb"; - -message Text { - int32 vendor = 1; - int32 version = 2; - int32 seqnum = 3; - int32 uid = 4; - int32 flag = 5; - int64 time = 6; // final time =first nofinal time - int32 lang = 7; - int32 starttime = 8; - int32 offtime = 9; - repeated Word words = 10; - bool end_of_segment = 11; - int32 duration_ms = 12; - string data_type = 13; // transcribe ,translate - repeated Translation trans = 14; - string culture = 15; - int64 texttime = 16; // pkg timestamp -} - -message Word { - string text = 1; - int32 start_ms = 2; - int32 duration_ms = 3; - bool is_final = 4; - double confidence = 5; -} - -message Translation { - bool is_final = 1; - string lang = 2; - repeated string texts = 3; -} \ No newline at end of file diff --git a/agents/addon/extension/chat_transcriber/property.json b/agents/addon/extension/chat_transcriber/property.json deleted file mode 100644 index 9e26dfee..00000000 --- a/agents/addon/extension/chat_transcriber/property.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/agents/addon/extension/elevenlabs_tts/elevenlabs_tts.go b/agents/addon/extension/elevenlabs_tts/elevenlabs_tts.go deleted file mode 100644 index 4d712e4f..00000000 --- a/agents/addon/extension/elevenlabs_tts/elevenlabs_tts.go +++ /dev/null @@ -1,82 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by XinHui Li in 2024-07. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package extension - -import ( - "context" - "fmt" - "io" - "time" - - elevenlabs "github.com/haguro/elevenlabs-go" -) - -type elevenlabsTTS struct { - client *elevenlabs.Client - config elevenlabsTTSConfig -} - -type elevenlabsTTSConfig struct { - ApiKey string - ModelId string - OptimizeStreamingLatency int - RequestTimeoutSeconds int - SimilarityBoost float32 - SpeakerBoost bool - Stability float32 - Style float32 - VoiceId string -} - -func defaultElevenlabsTTSConfig() elevenlabsTTSConfig { - return elevenlabsTTSConfig{ - ApiKey: "", - ModelId: "eleven_multilingual_v2", - OptimizeStreamingLatency: 0, - RequestTimeoutSeconds: 30, - SimilarityBoost: 0.75, - SpeakerBoost: false, - Stability: 0.5, - Style: 0.0, - VoiceId: "pNInz6obpgDQGcFmaJgB", - } -} - -func newElevenlabsTTS(config elevenlabsTTSConfig) (*elevenlabsTTS, error) { - return &elevenlabsTTS{ - config: config, - client: elevenlabs.NewClient(context.Background(), config.ApiKey, time.Duration(config.RequestTimeoutSeconds)*time.Second), - }, nil -} - -func (e *elevenlabsTTS) textToSpeechStream(streamWriter io.Writer, text string) (err error) { - req := elevenlabs.TextToSpeechRequest{ - Text: text, - ModelID: e.config.ModelId, - VoiceSettings: &elevenlabs.VoiceSettings{ - SimilarityBoost: e.config.SimilarityBoost, - SpeakerBoost: e.config.SpeakerBoost, - Stability: e.config.Stability, - Style: e.config.Style, - }, - } - queries := []elevenlabs.QueryFunc{ - elevenlabs.LatencyOptimizations(e.config.OptimizeStreamingLatency), - elevenlabs.OutputFormat("pcm_16000"), - } - - err = e.client.TextToSpeechStream(streamWriter, e.config.VoiceId, req, queries...) - if err != nil { - return fmt.Errorf("TextToSpeechStream failed, err: %v", err) - } - - return nil -} diff --git a/agents/addon/extension/elevenlabs_tts/elevenlabs_tts_extension.go b/agents/addon/extension/elevenlabs_tts/elevenlabs_tts_extension.go deleted file mode 100644 index 3b6ef4fd..00000000 --- a/agents/addon/extension/elevenlabs_tts/elevenlabs_tts_extension.go +++ /dev/null @@ -1,340 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by XinHui Li in 2024-07. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package extension - -import ( - "fmt" - "io" - "log/slog" - "sync" - "sync/atomic" - "time" - - "agora.io/rte/rtego" -) - -const ( - cmdInFlush = "flush" - cmdOutFlush = "flush" - dataInTextDataPropertyText = "text" - - propertyApiKey = "api_key" // Required - propertyModelId = "model_id" // Optional - propertyOptimizeStreamingLatency = "optimize_streaming_latency" // Optional - propertyRequestTimeoutSeconds = "request_timeout_seconds" // Optional - propertySimilarityBoost = "similarity_boost" // Optional - propertySpeakerBoost = "speaker_boost" // Optional - propertyStability = "stability" // Optional - propertyStyle = "style" // Optional - propertyVoiceId = "voice_id" // Optional -) - -const ( - textChanMax = 1024 -) - -var ( - logTag = slog.String("extension", "ELEVENLABS_TTS_EXTENSION") - - outdateTs atomic.Int64 - textChan chan *message - wg sync.WaitGroup -) - -type elevenlabsTTSExtension struct { - rtego.DefaultExtension - elevenlabsTTS *elevenlabsTTS -} - -type message struct { - text string - receivedTs int64 -} - -func newElevenlabsTTSExtension(name string) rtego.Extension { - return &elevenlabsTTSExtension{} -} - -// OnStart will be called when the extension is starting, -// properies can be read here to initialize and start the extension. -// current supported properties: -// - api_key (required) -// - model_id -// - optimize_streaming_latency -// - request_timeout_seconds -// - similarity_boost -// - speaker_boost -// - stability -// - style -// - voice_id -func (e *elevenlabsTTSExtension) OnStart(rte rtego.Rte) { - slog.Info("OnStart", logTag) - - // prepare configuration - elevenlabsTTSConfig := defaultElevenlabsTTSConfig() - - if apiKey, err := rte.GetPropertyString(propertyApiKey); err != nil { - slog.Error(fmt.Sprintf("GetProperty required %s failed, err: %v", propertyApiKey, err), logTag) - return - } else { - elevenlabsTTSConfig.ApiKey = apiKey - } - - if modelId, err := rte.GetPropertyString(propertyModelId); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyModelId, err), logTag) - } else { - if len(modelId) > 0 { - elevenlabsTTSConfig.ModelId = modelId - } - } - - if optimizeStreamingLatency, err := rte.GetPropertyInt64(propertyOptimizeStreamingLatency); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyOptimizeStreamingLatency, err), logTag) - } else { - if optimizeStreamingLatency > 0 { - elevenlabsTTSConfig.OptimizeStreamingLatency = int(optimizeStreamingLatency) - } - } - - if requestTimeoutSeconds, err := rte.GetPropertyInt64(propertyRequestTimeoutSeconds); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyRequestTimeoutSeconds, err), logTag) - } else { - if requestTimeoutSeconds > 0 { - elevenlabsTTSConfig.RequestTimeoutSeconds = int(requestTimeoutSeconds) - } - } - - if similarityBoost, err := rte.GetPropertyFloat64(propertySimilarityBoost); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertySimilarityBoost, err), logTag) - } else { - elevenlabsTTSConfig.SimilarityBoost = float32(similarityBoost) - } - - if speakerBoost, err := rte.GetPropertyBool(propertySpeakerBoost); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertySpeakerBoost, err), logTag) - } else { - elevenlabsTTSConfig.SpeakerBoost = speakerBoost - } - - if stability, err := rte.GetPropertyFloat64(propertyStability); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyStability, err), logTag) - } else { - elevenlabsTTSConfig.Stability = float32(stability) - } - - if style, err := rte.GetPropertyFloat64(propertyStyle); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyStyle, err), logTag) - } else { - elevenlabsTTSConfig.Style = float32(style) - } - - if voiceId, err := rte.GetPropertyString(propertyVoiceId); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyVoiceId, err), logTag) - } else { - if len(voiceId) > 0 { - elevenlabsTTSConfig.VoiceId = voiceId - } - } - - // create elevenlabsTTS instance - elevenlabsTTS, err := newElevenlabsTTS(elevenlabsTTSConfig) - if err != nil { - slog.Error(fmt.Sprintf("newElevenlabsTTS failed, err: %v", err), logTag) - return - } - - slog.Info(fmt.Sprintf("newElevenlabsTTS succeed with ModelId: %s, VoiceId: %s", - elevenlabsTTSConfig.ModelId, elevenlabsTTSConfig.VoiceId), logTag) - - // set elevenlabsTTS instance - e.elevenlabsTTS = elevenlabsTTS - - // create pcm instance - pcm := newPcm(defaultPcmConfig()) - pcmFrameSize := pcm.getPcmFrameSize() - - // init chan - textChan = make(chan *message, textChanMax) - - go func() { - slog.Info("process textChan", logTag) - - for msg := range textChan { - if msg.receivedTs < outdateTs.Load() { // Check whether to interrupt - slog.Info(fmt.Sprintf("textChan interrupt and flushing for input text: [%s], receivedTs: %d, outdateTs: %d", - msg.text, msg.receivedTs, outdateTs.Load()), logTag) - continue - } - - wg.Add(1) - slog.Info(fmt.Sprintf("textChan text: [%s]", msg.text), logTag) - - r, w := io.Pipe() - startTime := time.Now() - - go func() { - defer wg.Done() - defer w.Close() - - slog.Info(fmt.Sprintf("textToSpeechStream text: [%s]", msg.text), logTag) - - err = e.elevenlabsTTS.textToSpeechStream(w, msg.text) - if err != nil { - slog.Error(fmt.Sprintf("textToSpeechStream failed, err: %v", err), logTag) - return - } - }() - - slog.Info(fmt.Sprintf("read pcm stream, text:[%s], pcmFrameSize:%d", msg.text, pcmFrameSize), logTag) - - var ( - firstFrameLatency int64 - n int - pcmFrameRead int - readBytes int - sentFrames int - ) - buf := pcm.newBuf() - - // read pcm stream - for { - if msg.receivedTs < outdateTs.Load() { // Check whether to interrupt - slog.Info(fmt.Sprintf("read pcm stream interrupt and flushing for input text: [%s], receivedTs: %d, outdateTs: %d", - msg.text, msg.receivedTs, outdateTs.Load()), logTag) - break - } - - n, err = r.Read(buf[pcmFrameRead:]) - readBytes += n - pcmFrameRead += n - - if err != nil { - if err == io.EOF { - slog.Info("read pcm stream EOF", logTag) - break - } - - slog.Error(fmt.Sprintf("read pcm stream failed, err: %v", err), logTag) - break - } - - if pcmFrameRead != pcmFrameSize { - slog.Debug(fmt.Sprintf("the number of bytes read is [%d] inconsistent with pcm frame size", pcmFrameRead), logTag) - continue - } - - pcm.send(rte, buf) - // clear buf - buf = pcm.newBuf() - pcmFrameRead = 0 - sentFrames++ - - if firstFrameLatency == 0 { - firstFrameLatency = time.Since(startTime).Milliseconds() - slog.Info(fmt.Sprintf("first frame available for text: [%s], receivedTs: %d, firstFrameLatency: %dms", msg.text, msg.receivedTs, firstFrameLatency), logTag) - } - - slog.Debug(fmt.Sprintf("sending pcm data, text: [%s]", msg.text), logTag) - } - - if pcmFrameRead > 0 { - pcm.send(rte, buf) - sentFrames++ - slog.Info(fmt.Sprintf("sending pcm remain data, text: [%s], pcmFrameRead: %d", msg.text, pcmFrameRead), logTag) - } - - r.Close() - slog.Info(fmt.Sprintf("send pcm data finished, text: [%s], receivedTs: %d, readBytes: %d, sentFrames: %d, firstFrameLatency: %dms, finishLatency: %dms", - msg.text, msg.receivedTs, readBytes, sentFrames, firstFrameLatency, time.Since(startTime).Milliseconds()), logTag) - } - }() - - rte.OnStartDone() -} - -// OnCmd receives cmd from rte graph. -// current supported cmd: -// - name: flush -// example: -// {"name": "flush"} -func (e *elevenlabsTTSExtension) OnCmd( - rte rtego.Rte, - cmd rtego.Cmd, -) { - cmdName, err := cmd.CmdName() - if err != nil { - slog.Error(fmt.Sprintf("OnCmd get name failed, err: %v", err), logTag) - rte.ReturnString(rtego.Error, "error", cmd) - return - } - - slog.Info(fmt.Sprintf("OnCmd %s", cmdInFlush), logTag) - - switch cmdName { - case cmdInFlush: - outdateTs.Store(time.Now().UnixMicro()) - - // send out - outCmd, err := rtego.NewCmd(cmdOutFlush) - if err != nil { - slog.Error(fmt.Sprintf("new cmd %s failed, err: %v", cmdOutFlush, err), logTag) - rte.ReturnString(rtego.Error, "error", cmd) - return - } - - if err := rte.SendCmd(outCmd, nil); err != nil { - slog.Error(fmt.Sprintf("send cmd %s failed, err: %v", cmdOutFlush, err), logTag) - rte.ReturnString(rtego.Error, "error", cmd) - return - } else { - slog.Info(fmt.Sprintf("cmd %s sent", cmdOutFlush), logTag) - } - } - - rte.ReturnString(rtego.Ok, "ok", cmd) -} - -// OnData receives data from rte graph. -// current supported data: -// - name: text_data -// example: -// {name: text_data, properties: {text: "hello"} -func (e *elevenlabsTTSExtension) OnData( - rte rtego.Rte, - data rtego.Data, -) { - text, err := data.GetPropertyString(dataInTextDataPropertyText) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s failed, err: %v", dataInTextDataPropertyText, err), logTag) - return - } - - if len(text) == 0 { - slog.Debug("OnData text is empty, ignored", logTag) - return - } - - slog.Info(fmt.Sprintf("OnData input text: [%s]", text), logTag) - - go func() { - textChan <- &message{text: text, receivedTs: time.Now().UnixMicro()} - }() -} - -func init() { - slog.Info("elevenlabs_tts extension init", logTag) - - // Register addon - rtego.RegisterAddonAsExtension( - "elevenlabs_tts", - rtego.NewDefaultExtensionAddon(newElevenlabsTTSExtension), - ) -} diff --git a/agents/addon/extension/elevenlabs_tts/go.mod b/agents/addon/extension/elevenlabs_tts/go.mod deleted file mode 100644 index bb90f1c6..00000000 --- a/agents/addon/extension/elevenlabs_tts/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module elevenlabs_tts - -go 1.21 - -replace agora.io/rte => ../../../interface - -require ( - agora.io/rte v0.0.0-00010101000000-000000000000 - github.com/haguro/elevenlabs-go v0.2.4 -) diff --git a/agents/addon/extension/elevenlabs_tts/go.sum b/agents/addon/extension/elevenlabs_tts/go.sum deleted file mode 100644 index 6c1feddc..00000000 --- a/agents/addon/extension/elevenlabs_tts/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/haguro/elevenlabs-go v0.2.4 h1:Z1a/I+b5fAtGSfrhEj97dYG1EbV9uRzSfvz5n5+ud34= -github.com/haguro/elevenlabs-go v0.2.4/go.mod h1:j15h9w2BpgxlIGWXmCKWPPDaTo2QAO83zFy5J+pFCt8= diff --git a/agents/addon/extension/elevenlabs_tts/manifest.json b/agents/addon/extension/elevenlabs_tts/manifest.json deleted file mode 100644 index 620fb224..00000000 --- a/agents/addon/extension/elevenlabs_tts/manifest.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "type": "extension", - "name": "elevenlabs_tts", - "version": "0.1.0", - "language": "go", - "dependencies": [ - { - "type": "system", - "name": "rte_runtime", - "version": "0.1.0" - }, - { - "type": "system", - "name": "rte_runtime_go", - "version": "0.1.0" - } - ], - "api": { - "property": { - "api_key": { - "type": "string" - }, - "model_id": { - "type": "string" - }, - "request_timeout_seconds": { - "type": "int64" - }, - "similarity_boost": { - "type": "float64" - }, - "speaker_boost": { - "type": "bool" - }, - "stability": { - "type": "float64" - }, - "style": { - "type": "float64" - }, - "optimize_streaming_latency": { - "type": "int64" - }, - "voice_id": { - "type": "string" - } - }, - "data_in": [ - { - "name": "text_data", - "property": { - "text": { - "type": "string" - } - } - } - ], - "cmd_in": [ - { - "name": "flush" - } - ], - "cmd_out": [ - { - "name": "flush" - } - ], - "pcm_frame_out": [ - { - "name": "pcm_frame" - } - ] - } -} diff --git a/agents/addon/extension/elevenlabs_tts/pcm.go b/agents/addon/extension/elevenlabs_tts/pcm.go deleted file mode 100644 index c3454b10..00000000 --- a/agents/addon/extension/elevenlabs_tts/pcm.go +++ /dev/null @@ -1,104 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by XinHui Li in 2024-07. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package extension - -import ( - "fmt" - "log/slog" - - "agora.io/rte/rtego" -) - -type pcm struct { - config *pcmConfig -} - -type pcmConfig struct { - BytesPerSample int32 - Channel int32 - ChannelLayout uint64 - Name string - SampleRate int32 - SamplesPerChannel int32 - Timestamp int64 -} - -func defaultPcmConfig() *pcmConfig { - return &pcmConfig{ - BytesPerSample: 2, - Channel: 1, - ChannelLayout: 1, - Name: "pcm_frame", - SampleRate: 16000, - SamplesPerChannel: 16000 / 100, - Timestamp: 0, - } -} - -func newPcm(config *pcmConfig) *pcm { - return &pcm{ - config: config, - } -} - -func (p *pcm) getPcmFrame(buf []byte) (pcmFrame rtego.PcmFrame, err error) { - pcmFrame, err = rtego.NewPcmFrame(p.config.Name) - if err != nil { - slog.Error(fmt.Sprintf("NewPcmFrame failed, err: %v", err), logTag) - return - } - - // set pcm frame - pcmFrame.SetBytesPerSample(p.config.BytesPerSample) - pcmFrame.SetSampleRate(p.config.SampleRate) - pcmFrame.SetChannelLayout(p.config.ChannelLayout) - pcmFrame.SetNumberOfChannels(p.config.Channel) - pcmFrame.SetTimestamp(p.config.Timestamp) - pcmFrame.SetDataFmt(rtego.PcmFrameDataFmtInterleave) - pcmFrame.SetSamplesPerChannel(p.config.SamplesPerChannel) - pcmFrame.AllocBuf(p.getPcmFrameSize()) - - borrowedBuf, err := pcmFrame.BorrowBuf() - if err != nil { - slog.Error(fmt.Sprintf("BorrowBuf failed, err: %v", err), logTag) - return - } - - // copy data - copy(borrowedBuf, buf) - - pcmFrame.GiveBackBuf(&borrowedBuf) - return -} - -func (p *pcm) getPcmFrameSize() int { - return int(p.config.SamplesPerChannel * p.config.Channel * p.config.BytesPerSample) -} - -func (p *pcm) newBuf() []byte { - return make([]byte, p.getPcmFrameSize()) -} - -func (p *pcm) send(rte rtego.Rte, buf []byte) (err error) { - pcmFrame, err := p.getPcmFrame(buf) - if err != nil { - slog.Error(fmt.Sprintf("getPcmFrame failed, err: %v", err), logTag) - return - } - - // send pcm - if err = rte.SendPcmFrame(pcmFrame); err != nil { - slog.Error(fmt.Sprintf("SendPcmFrame failed, err: %v", err), logTag) - return - } - - return -} diff --git a/agents/addon/extension/elevenlabs_tts/property.json b/agents/addon/extension/elevenlabs_tts/property.json deleted file mode 100644 index 9e26dfee..00000000 --- a/agents/addon/extension/elevenlabs_tts/property.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/agents/addon/extension/interrupt_detector/extension.go b/agents/addon/extension/interrupt_detector/extension.go deleted file mode 100644 index 962940e0..00000000 --- a/agents/addon/extension/interrupt_detector/extension.go +++ /dev/null @@ -1,78 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by Wei Hu in 2022-10. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package extension - -import ( - "fmt" - "log/slog" - - "agora.io/rte/rtego" -) - -const ( - textDataTextField = "text" - textDataFinalField = "is_final" - - cmdNameFlush = "flush" -) - -var ( - logTag = slog.String("extension", "INTERRUPT_DETECTOR_EXTENSION") -) - -type interruptDetectorExtension struct { - rtego.DefaultExtension -} - -func newExtension(name string) rtego.Extension { - return &interruptDetectorExtension{} -} - -// OnData receives data from rte graph. -// current supported data: -// - name: text_data -// example: -// {name: text_data, properties: {text: "hello", is_final: false} -func (p *interruptDetectorExtension) OnData( - rte rtego.Rte, - data rtego.Data, -) { - text, err := data.GetPropertyString(textDataTextField) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s error: %v", textDataTextField, err), logTag) - return - } - - final, err := data.GetPropertyBool(textDataFinalField) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s error: %v", textDataFinalField, err), logTag) - return - } - - slog.Debug(fmt.Sprintf("OnData %s: %s %s: %t", textDataTextField, text, textDataFinalField, final), logTag) - - if final || len(text) >= 2 { - flushCmd, _ := rtego.NewCmd(cmdNameFlush) - rte.SendCmd(flushCmd, nil) - - slog.Info(fmt.Sprintf("sent cmd: %s", cmdNameFlush), logTag) - } -} - -func init() { - slog.Info("interrupt_detector extension init", logTag) - - // Register addon - rtego.RegisterAddonAsExtension( - "interrupt_detector", - rtego.NewDefaultExtensionAddon(newExtension), - ) -} diff --git a/agents/addon/extension/interrupt_detector/go.mod b/agents/addon/extension/interrupt_detector/go.mod deleted file mode 100644 index bced26e0..00000000 --- a/agents/addon/extension/interrupt_detector/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module extension - -go 1.18 - -replace agora.io/rte => ../../../interface - -require agora.io/rte v0.0.0-00010101000000-000000000000 diff --git a/agents/addon/extension/interrupt_detector/manifest.json b/agents/addon/extension/interrupt_detector/manifest.json deleted file mode 100644 index 05781cf5..00000000 --- a/agents/addon/extension/interrupt_detector/manifest.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "type": "extension", - "name": "interrupt_detector", - "version": "0.1.0", - "language": "go", - "dependencies": [ - { - "type": "system", - "name": "rte_runtime", - "version": "0.1.0" - }, - { - "type": "system", - "name": "rte_runtime_go", - "version": "0.1.0" - } - ], - "api": { - "data_in": [ - { - "name": "text_data", - "property": { - "text": { - "type": "string" - }, - "is_final": { - "type": "bool" - } - } - } - ], - "cmd_out": [ - { - "name": "flush" - } - ] - } -} \ No newline at end of file diff --git a/agents/addon/extension/interrupt_detector/property.json b/agents/addon/extension/interrupt_detector/property.json deleted file mode 100644 index 9e26dfee..00000000 --- a/agents/addon/extension/interrupt_detector/property.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/agents/addon/extension/openai_chatgpt/README.md b/agents/addon/extension/openai_chatgpt/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/agents/addon/extension/openai_chatgpt/go.mod b/agents/addon/extension/openai_chatgpt/go.mod deleted file mode 100644 index 5bb6b52b..00000000 --- a/agents/addon/extension/openai_chatgpt/go.mod +++ /dev/null @@ -1,17 +0,0 @@ -module openai_chatgpt - -go 1.21 - -replace agora.io/rte => ../../../interface - -require ( - agora.io/rte v0.0.0-00010101000000-000000000000 - github.com/sashabaranov/go-openai v1.24.1 - github.com/stretchr/testify v1.9.0 -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/agents/addon/extension/openai_chatgpt/go.sum b/agents/addon/extension/openai_chatgpt/go.sum deleted file mode 100644 index 64a09f35..00000000 --- a/agents/addon/extension/openai_chatgpt/go.sum +++ /dev/null @@ -1,12 +0,0 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sashabaranov/go-openai v1.24.1 h1:DWK95XViNb+agQtuzsn+FyHhn3HQJ7Va8z04DQDJ1MI= -github.com/sashabaranov/go-openai v1.24.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/agents/addon/extension/openai_chatgpt/manifest.json b/agents/addon/extension/openai_chatgpt/manifest.json deleted file mode 100644 index aa4f6d50..00000000 --- a/agents/addon/extension/openai_chatgpt/manifest.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "type": "extension", - "name": "openai_chatgpt", - "version": "0.1.0", - "language": "go", - "support": [], - "dependencies": [ - { - "type": "system", - "name": "rte_runtime", - "version": "0.1.0" - }, - { - "type": "system", - "name": "rte_runtime_go", - "version": "0.1.0" - } - ], - "api": { - "property": { - "api_key": { - "type": "string" - }, - "frequency_penalty": { - "type": "float64" - }, - "presence_penalty": { - "type": "float64" - }, - "model": { - "type": "string" - }, - "max_tokens": { - "type": "int64" - }, - "prompt": { - "type": "string" - }, - "greeting": { - "type": "string" - }, - "max_memory_length": { - "type": "int64" - } - }, - "data_in": [ - { - "name": "text_data", - "property": { - "text": { - "type": "string" - }, - "is_final": { - "type": "bool" - } - } - } - ], - "data_out": [ - { - "name": "text_data", - "property": { - "text": { - "type": "string" - }, - "end_of_segment": { - "type": "bool" - } - } - } - ], - "cmd_in": [ - { - "name": "flush" - } - ], - "cmd_out": [ - { - "name": "flush" - } - ] - } -} \ No newline at end of file diff --git a/agents/addon/extension/openai_chatgpt/openai_chatgpt.go b/agents/addon/extension/openai_chatgpt/openai_chatgpt.go deleted file mode 100644 index 1a09ed14..00000000 --- a/agents/addon/extension/openai_chatgpt/openai_chatgpt.go +++ /dev/null @@ -1,111 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by lixinhui in 2024. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package extension - -import ( - "context" - "fmt" - "math/rand" - "net/http" - "net/url" - - openai "github.com/sashabaranov/go-openai" -) - -type openaiChatGPT struct { - client *openai.Client - config openaiChatGPTConfig -} - -type openaiChatGPTConfig struct { - BaseUrl string - ApiKey string - - Model string - Prompt string - - FrequencyPenalty float32 - PresencePenalty float32 - TopP float32 - Temperature float32 - MaxTokens int - Seed int - - ProxyUrl string -} - -func defaultOpenaiChatGPTConfig() openaiChatGPTConfig { - return openaiChatGPTConfig{ - BaseUrl: "https://api.openai.com/v1", - ApiKey: "", - - Model: openai.GPT4o, - Prompt: "You are a voice assistant who talks in a conversational way and can chat with me like my friends. i will speak to you in english or chinese, and you will answer in the corrected and improved version of my text with the language i use. Don't talk like a robot, instead i would like you to talk like real human with emotions. i will use your answer for text-to-speech, so don't return me any meaningless characters. I want you to be helpful, when i'm asking you for advices, give me precise, practical and useful advices instead of being vague. When giving me list of options, express the options in a narrative way instead of bullet points.", - - FrequencyPenalty: 0.9, - PresencePenalty: 0.9, - TopP: 1.0, - Temperature: 0.1, - MaxTokens: 512, - Seed: rand.Int(), - - ProxyUrl: "", - } -} - -func newOpenaiChatGPT(config openaiChatGPTConfig) (*openaiChatGPT, error) { - conf := openai.DefaultConfig(config.ApiKey) - - if config.BaseUrl != "" { - conf.BaseURL = config.BaseUrl - } - - if config.ProxyUrl != "" { - proxyUrl, err := url.Parse(config.ProxyUrl) - if err != nil { - return nil, fmt.Errorf("newOpenaiChatGPT failed on parsing proxy url, err: %v", err) - } - conf.HTTPClient = &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} - } - - return &openaiChatGPT{ - config: config, - client: openai.NewClientWithConfig(conf), - }, nil -} - -func (c *openaiChatGPT) getChatCompletionsStream(messages []openai.ChatCompletionMessage) (*openai.ChatCompletionStream, error) { - req := openai.ChatCompletionRequest{ - Temperature: c.config.Temperature, - TopP: c.config.TopP, - PresencePenalty: c.config.PresencePenalty, - FrequencyPenalty: c.config.FrequencyPenalty, - MaxTokens: c.config.MaxTokens, - Seed: &c.config.Seed, - Messages: append( - []openai.ChatCompletionMessage{ - { - Role: openai.ChatMessageRoleSystem, - Content: c.config.Prompt, - }, - }, - messages..., - ), - Model: c.config.Model, - Stream: true, - } - - resp, err := c.client.CreateChatCompletionStream(context.Background(), req) - if err != nil { - return nil, fmt.Errorf("CreateChatCompletionStream failed,err: %v", err) - } - return resp, nil -} diff --git a/agents/addon/extension/openai_chatgpt/openai_chatgpt_extension.go b/agents/addon/extension/openai_chatgpt/openai_chatgpt_extension.go deleted file mode 100644 index 6abdaa51..00000000 --- a/agents/addon/extension/openai_chatgpt/openai_chatgpt_extension.go +++ /dev/null @@ -1,391 +0,0 @@ -/** - * - * Agora Real Time Engagement - * Created by lixinhui in 2024. - * Copyright (c) 2024 Agora IO. All rights reserved. - * - */ -// Note that this is just an example extension written in the GO programming -// language, so the package name does not equal to the containing directory -// name. However, it is not common in Go. -package extension - -import ( - "errors" - "fmt" - "io" - "log/slog" - "sync" - "sync/atomic" - "time" - - "agora.io/rte/rtego" - openai "github.com/sashabaranov/go-openai" -) - -var ( - logTag = slog.String("extension", "OPENAI_CHATGPT_EXTENSION") -) - -type openaiChatGPTExtension struct { - rtego.DefaultExtension - openaiChatGPT *openaiChatGPT -} - -const ( - cmdInFlush = "flush" - cmdOutFlush = "flush" - dataInTextDataPropertyText = "text" - dataInTextDataPropertyIsFinal = "is_final" - dataOutTextDataPropertyText = "text" - dataOutTextDataPropertyTextEndOfSegment = "end_of_segment" - - propertyBaseUrl = "base_url" // Optional - propertyApiKey = "api_key" // Required - propertyModel = "model" // Optional - propertyPrompt = "prompt" // Optional - propertyFrequencyPenalty = "frequency_penalty" // Optional - propertyPresencePenalty = "presence_penalty" // Optional - propertyTemperature = "temperature" // Optional - propertyTopP = "top_p" // Optional - propertyMaxTokens = "max_tokens" // Optional - propertyGreeting = "greeting" // Optional - propertyProxyUrl = "proxy_url" // Optional - propertyMaxMemoryLength = "max_memory_length" // Optional -) - -var ( - memory []openai.ChatCompletionMessage - memoryChan chan openai.ChatCompletionMessage - maxMemoryLength = 10 - - outdateTs atomic.Int64 - wg sync.WaitGroup -) - -func newChatGPTExtension(name string) rtego.Extension { - return &openaiChatGPTExtension{} -} - -// OnStart will be called when the extension is starting, -// properies can be read here to initialize and start the extension. -// current supported properties: -// - api_key (required) -// - model -// - prompt -// - frequency_penalty -// - presence_penalty -// - temperature -// - top_p -// - max_tokens -// - greeting -// - proxy_url -func (p *openaiChatGPTExtension) OnStart(rte rtego.Rte) { - slog.Info("OnStart", logTag) - - // prepare configuration - openaiChatGPTConfig := defaultOpenaiChatGPTConfig() - - if baseUrl, err := rte.GetPropertyString(propertyBaseUrl); err != nil { - slog.Error(fmt.Sprintf("GetProperty required %s failed, err: %v", propertyBaseUrl, err), logTag) - } else { - if len(baseUrl) > 0 { - openaiChatGPTConfig.BaseUrl = baseUrl - } - } - - if apiKey, err := rte.GetPropertyString(propertyApiKey); err != nil { - slog.Error(fmt.Sprintf("GetProperty required %s failed, err: %v", propertyApiKey, err), logTag) - return - } else { - openaiChatGPTConfig.ApiKey = apiKey - } - - if model, err := rte.GetPropertyString(propertyModel); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s error:%v", propertyModel, err), logTag) - } else { - if len(model) > 0 { - openaiChatGPTConfig.Model = model - } - } - - if prompt, err := rte.GetPropertyString(propertyPrompt); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s error:%v", propertyPrompt, err), logTag) - } else { - if len(prompt) > 0 { - openaiChatGPTConfig.Prompt = prompt - } - } - - if frequencyPenalty, err := rte.GetPropertyFloat64(propertyFrequencyPenalty); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyFrequencyPenalty, err), logTag) - } else { - openaiChatGPTConfig.FrequencyPenalty = float32(frequencyPenalty) - } - - if presencePenalty, err := rte.GetPropertyFloat64(propertyPresencePenalty); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyPresencePenalty, err), logTag) - } else { - openaiChatGPTConfig.PresencePenalty = float32(presencePenalty) - } - - if temperature, err := rte.GetPropertyFloat64(propertyTemperature); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyTemperature, err), logTag) - } else { - openaiChatGPTConfig.Temperature = float32(temperature) - } - - if topP, err := rte.GetPropertyFloat64(propertyTopP); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyTopP, err), logTag) - } else { - openaiChatGPTConfig.TopP = float32(topP) - } - - if maxTokens, err := rte.GetPropertyInt64(propertyMaxTokens); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyMaxTokens, err), logTag) - } else { - if maxTokens > 0 { - openaiChatGPTConfig.MaxTokens = int(maxTokens) - } - } - - if proxyUrl, err := rte.GetPropertyString(propertyProxyUrl); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyProxyUrl, err), logTag) - } else { - openaiChatGPTConfig.ProxyUrl = proxyUrl - } - - greeting, err := rte.GetPropertyString(propertyGreeting) - if err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyGreeting, err), logTag) - } - - if propMaxMemoryLength, err := rte.GetPropertyInt64(propertyMaxMemoryLength); err != nil { - slog.Warn(fmt.Sprintf("GetProperty optional %s failed, err: %v", propertyMaxMemoryLength, err), logTag) - } else { - if propMaxMemoryLength > 0 { - maxMemoryLength = int(propMaxMemoryLength) - } - } - - // create openaiChatGPT instance - openaiChatgpt, err := newOpenaiChatGPT(openaiChatGPTConfig) - if err != nil { - slog.Error(fmt.Sprintf("newOpenaiChatGPT failed, err: %v", err), logTag) - return - } - slog.Info(fmt.Sprintf("newOpenaiChatGPT succeed with max_tokens: %d, model: %s", - openaiChatGPTConfig.MaxTokens, openaiChatGPTConfig.Model), logTag) - - p.openaiChatGPT = openaiChatgpt - - memoryChan = make(chan openai.ChatCompletionMessage, maxMemoryLength*2) - - // send greeting if available - if len(greeting) > 0 { - outputData, _ := rtego.NewData("text_data") - outputData.SetProperty(dataOutTextDataPropertyText, greeting) - outputData.SetProperty(dataOutTextDataPropertyTextEndOfSegment, true) - if err := rte.SendData(outputData); err != nil { - slog.Error(fmt.Sprintf("greeting [%s] send failed, err: %v", greeting, err), logTag) - } else { - slog.Info(fmt.Sprintf("greeting [%s] sent", greeting), logTag) - } - } - - rte.OnStartDone() -} - -// OnCmd receives cmd from rte graph. -// current supported cmd: -// - name: flush -// example: -// {"name": "flush"} -func (p *openaiChatGPTExtension) OnCmd( - rte rtego.Rte, - cmd rtego.Cmd, -) { - cmdName, err := cmd.CmdName() - if err != nil { - slog.Error(fmt.Sprintf("OnCmd get name failed, err: %v", err), logTag) - rte.ReturnString(rtego.Error, "error", cmd) - return - } - slog.Info(fmt.Sprintf("OnCmd %s", cmdInFlush), logTag) - - switch cmdName { - case cmdInFlush: - outdateTs.Store(time.Now().UnixMicro()) - - wg.Wait() // wait for chat completion stream to finish - - // send out - outCmd, err := rtego.NewCmd(cmdOutFlush) - if err != nil { - slog.Error(fmt.Sprintf("new cmd %s failed, err: %v", cmdOutFlush, err), logTag) - rte.ReturnString(rtego.Error, "error", cmd) - return - } - if err := rte.SendCmd(outCmd, nil); err != nil { - slog.Error(fmt.Sprintf("send cmd %s failed, err: %v", cmdOutFlush, err), logTag) - rte.ReturnString(rtego.Error, "error", cmd) - return - } else { - slog.Info(fmt.Sprintf("cmd %s sent", cmdOutFlush), logTag) - } - } - rte.ReturnString(rtego.Ok, "ok", cmd) -} - -// OnData receives data from rte graph. -// current supported data: -// - name: text_data -// example: -// {"name": "text_data", "properties": {"text": "hello", "is_final": true} -func (p *openaiChatGPTExtension) OnData( - rte rtego.Rte, - data rtego.Data, -) { - // Get isFinal - isFinal, err := data.GetPropertyBool(dataInTextDataPropertyIsFinal) - if err != nil { - slog.Warn(fmt.Sprintf("OnData GetProperty %s failed, err: %v", dataInTextDataPropertyIsFinal, err), logTag) - return - } - if !isFinal { // ignore non-final - slog.Debug("ignore non-final input", logTag) - return - } - - // Get input text - inputText, err := data.GetPropertyString(dataInTextDataPropertyText) - if err != nil { - slog.Error(fmt.Sprintf("OnData GetProperty %s failed, err: %v", dataInTextDataPropertyText, err), logTag) - return - } - if len(inputText) == 0 { - slog.Debug("ignore empty text", logTag) - return - } - slog.Info(fmt.Sprintf("OnData input text: [%s]", inputText), logTag) - - // prepare memory - for len(memoryChan) > 0 { - m, ok := <-memoryChan - if !ok { - break - } - memory = append(memory, m) - if len(memory) > maxMemoryLength { - memory = memory[1:] - } - } - memory = append(memory, openai.ChatCompletionMessage{ - Role: openai.ChatMessageRoleUser, - Content: inputText, - }) - if len(memory) > maxMemoryLength { - memory = memory[1:] - } - - // start goroutine to request and read responses from openai - wg.Add(1) - go func(startTime time.Time, inputText string, memory []openai.ChatCompletionMessage) { - defer wg.Done() - slog.Info(fmt.Sprintf("GetChatCompletionsStream for input text: [%s] memory: %v", inputText, memory), logTag) - - // Get result from ai - resp, err := p.openaiChatGPT.getChatCompletionsStream(memory) - if err != nil { - slog.Error(fmt.Sprintf("GetChatCompletionsStream for input text: [%s] failed, err: %v", inputText, err), logTag) - return - } - defer func() { - if resp != nil { // Close stream object - resp.Close() - } - }() - slog.Debug(fmt.Sprintf("GetChatCompletionsStream start to recv for input text: [%s]", inputText), logTag) - - var sentence, fullContent string - var firstSentenceSent bool - for { - if startTime.UnixMicro() < outdateTs.Load() { // Check whether to interrupt - slog.Info(fmt.Sprintf("GetChatCompletionsStream recv interrupt and flushing for input text: [%s], startTs: %d, outdateTs: %d", - inputText, startTime.UnixMicro(), outdateTs.Load()), logTag) - break - } - - chatCompletions, err := resp.Recv() - if errors.Is(err, io.EOF) { - slog.Debug(fmt.Sprintf("GetChatCompletionsStream recv for input text: [%s], io.EOF break", inputText), logTag) - break - } - - var content string - if len(chatCompletions.Choices) > 0 && chatCompletions.Choices[0].Delta.Content != "" { - content = chatCompletions.Choices[0].Delta.Content - } - fullContent += content - - for { - // feed content and check whether sentence is available - var sentenceIsFinal bool - sentence, content, sentenceIsFinal = parseSentence(sentence, content) - if len(sentence) == 0 || !sentenceIsFinal { - slog.Debug(fmt.Sprintf("sentence %s is empty or not final", sentence), logTag) - break - } - slog.Debug(fmt.Sprintf("GetChatCompletionsStream recv for input text: [%s] got sentence: [%s]", inputText, sentence), logTag) - - // send sentence - outputData, err := rtego.NewData("text_data") - if err != nil { - slog.Error(fmt.Sprintf("NewData failed, err: %v", err), logTag) - break - } - outputData.SetProperty(dataOutTextDataPropertyText, sentence) - outputData.SetProperty(dataOutTextDataPropertyTextEndOfSegment, false) - if err := rte.SendData(outputData); err != nil { - slog.Error(fmt.Sprintf("GetChatCompletionsStream recv for input text: [%s] send sentence [%s] failed, err: %v", inputText, sentence, err), logTag) - break - } else { - slog.Info(fmt.Sprintf("GetChatCompletionsStream recv for input text: [%s] sent sentence [%s]", inputText, sentence), logTag) - } - sentence = "" - - if !firstSentenceSent { - firstSentenceSent = true - slog.Info(fmt.Sprintf("GetChatCompletionsStream recv for input text: [%s] first sentence sent, first_sentency_latency %dms", - inputText, time.Since(startTime).Milliseconds()), logTag) - } - } - } - - // remember response as assistant content in memory - memoryChan <- openai.ChatCompletionMessage{ - Role: openai.ChatMessageRoleAssistant, - Content: fullContent, - } - - // send end of segment - outputData, _ := rtego.NewData("text_data") - outputData.SetProperty(dataOutTextDataPropertyText, sentence) - outputData.SetProperty(dataOutTextDataPropertyTextEndOfSegment, true) - if err := rte.SendData(outputData); err != nil { - slog.Error(fmt.Sprintf("GetChatCompletionsStream for input text: [%s] end of segment with sentence [%s] send failed, err: %v", inputText, sentence, err), logTag) - } else { - slog.Info(fmt.Sprintf("GetChatCompletionsStream for input text: [%s] end of segment with sentence [%s] sent", inputText, sentence), logTag) - } - }(time.Now(), inputText, append([]openai.ChatCompletionMessage{}, memory...)) -} - -func init() { - slog.Info("init") - - // Register addon - rtego.RegisterAddonAsExtension( - "openai_chatgpt", - rtego.NewDefaultExtensionAddon(newChatGPTExtension), - ) -} diff --git a/agents/addon/extension/openai_chatgpt/property.json b/agents/addon/extension/openai_chatgpt/property.json deleted file mode 100644 index 9e26dfee..00000000 --- a/agents/addon/extension/openai_chatgpt/property.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/agents/addon/extension/openai_chatgpt/sentence.go b/agents/addon/extension/openai_chatgpt/sentence.go deleted file mode 100644 index e9b9d310..00000000 --- a/agents/addon/extension/openai_chatgpt/sentence.go +++ /dev/null @@ -1,30 +0,0 @@ -package extension - -func isPunctuation(r rune) bool { - if r == ',' || r == ',' || - r == '.' || r == '。' || - r == '?' || r == '?' || - r == '!' || r == '!' { - return true - } - return false -} - -func parseSentence(sentence, content string) (string, string, bool) { - var remain string - var foundPunc bool - - for _, r := range content { - if !foundPunc { - sentence += string(r) - } else { - remain += string(r) - } - - if !foundPunc && isPunctuation(r) { - foundPunc = true - } - } - - return sentence, remain, foundPunc -} diff --git a/agents/addon/extension/openai_chatgpt/sentence_test.go b/agents/addon/extension/openai_chatgpt/sentence_test.go deleted file mode 100644 index b09fe307..00000000 --- a/agents/addon/extension/openai_chatgpt/sentence_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package extension - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestIsPunctuation(t *testing.T) { - cases := []struct { - r rune - expect bool - }{ - {',', true}, - {',', true}, - {'.', true}, - {'。', true}, - {'?', true}, - {'?', true}, - {'!', true}, - {'!', true}, - - {'a', false}, - {'0', false}, - } - - for i, c := range cases { - require.Equal(t, c.expect, isPunctuation(c.r), "case %d", i) - } -} - -func TestSplitByPunctuation(t *testing.T) { - cases := []struct { - s string - expect []string - }{ - {"Hello world!", []string{"Hello world"}}, - {"Hey, there!", []string{"Hey", " there"}}, - } - - for i, c := range cases { - out := strings.FieldsFunc(c.s, isPunctuation) - require.Equal(t, c.expect, out, "case %d", i) - } -} - -func TestParseSentence_Should_NoFinalSentence(t *testing.T) { - cases := []struct { - sentence string - content string - - expectSentence string - expectContent string - }{ - { - sentence: "", - content: "", - expectSentence: "", - expectContent: "", - }, - { - sentence: "a", - content: "", - expectSentence: "a", - expectContent: "", - }, - { - sentence: "", - content: "a", - expectSentence: "a", - expectContent: "", - }, - { - sentence: "abc", - content: "ddd", - expectSentence: "abcddd", - expectContent: "", - }, - } - - for i, c := range cases { - sentence, content, final := parseSentence(c.sentence, c.content) - require.False(t, final, "case %d", i) - - require.Equal(t, c.expectSentence, sentence, "case %d", i) - require.Equal(t, c.expectContent, content, "case %d", i) - } -} - -func TestParseSentence_Should_FinalSentence(t *testing.T) { - cases := []struct { - sentence string - content string - - expectSentence string - expectContent string - }{ - { - sentence: "", - content: ",", - expectSentence: ",", - expectContent: "", - }, - { - sentence: "", - content: ",ddd", - expectSentence: ",", - expectContent: "ddd", - }, - { - sentence: "abc", - content: ",ddd", - expectSentence: "abc,", - expectContent: "ddd", - }, - { - sentence: "abc", - content: "dd,d", - expectSentence: "abcdd,", - expectContent: "d", - }, - { - sentence: "abc", - content: "ddd,", - expectSentence: "abcddd,", - expectContent: "", - }, - { - sentence: "abc", - content: "ddd,eee,fff,", - expectSentence: "abcddd,", - expectContent: "eee,fff,", - }, - { - sentence: "我的", - content: "你好,啊!", - expectSentence: "我的你好,", - expectContent: "啊!", - }, - } - - for i, c := range cases { - sentence, content, final := parseSentence(c.sentence, c.content) - require.True(t, final, "case %d", i) - - require.Equal(t, c.expectSentence, sentence, "case %d", i) - require.Equal(t, c.expectContent, content, "case %d", i) - } -}