diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/Conversation.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/Conversation.kt index ced836639..7a55f9da7 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/Conversation.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/Conversation.kt @@ -4,6 +4,7 @@ import com.xebia.functional.xef.AIError import com.xebia.functional.xef.llm.* import com.xebia.functional.xef.llm.models.functions.CFunction import com.xebia.functional.xef.llm.models.images.ImagesGenerationResponse +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore @@ -17,6 +18,8 @@ interface Conversation : AutoClose, AutoCloseable { val store: VectorStore + val metric: Metric + val conversationId: ConversationId? val conversation: Conversation @@ -80,14 +83,16 @@ interface Conversation : AutoClose, AutoCloseable { operator fun invoke( store: VectorStore, + metric: Metric, conversationId: ConversationId? = ConversationId(UUID.generateUUID().toString()) - ): PlatformConversation = PlatformConversation.create(store, conversationId) + ): PlatformConversation = PlatformConversation.create(store, metric, conversationId) @JvmSynthetic suspend operator fun invoke( store: VectorStore, + metric: Metric, conversationId: ConversationId? = ConversationId(UUID.generateUUID().toString()), block: suspend PlatformConversation.() -> A - ): A = block(invoke(store, conversationId)) + ): A = block(invoke(store, metric, conversationId)) } } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.kt index 9b122aee8..24944e1d9 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.kt @@ -1,5 +1,6 @@ package com.xebia.functional.xef.conversation +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore @@ -11,6 +12,7 @@ expect abstract class PlatformConversation( companion object { fun create( store: VectorStore, + metric: Metric, conversationId: ConversationId?, ): PlatformConversation } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 13d59c28d..e776f160e 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -45,22 +45,25 @@ interface Chat : LLM { promptMessages(prompt, scope).firstOrNull() ?: throw AIError.NoResponse() @AiDsl - suspend fun promptMessages(prompt: Prompt, scope: Conversation): List { - val promptMemories = prompt.messages.toMemory(scope) - val adaptedPrompt = PromptCalculator.adaptPromptToConversationAndModel(prompt, scope, this@Chat) + suspend fun promptMessages(prompt: Prompt, scope: Conversation): List = + scope.metric.promptSpan(scope, prompt) { + val promptMemories = prompt.messages.toMemory(scope) + val adaptedPrompt = + PromptCalculator.adaptPromptToConversationAndModel(prompt, scope, this@Chat) - val request = - ChatCompletionRequest( - user = adaptedPrompt.configuration.user, - messages = adaptedPrompt.messages, - n = adaptedPrompt.configuration.numberOfPredictions, - temperature = adaptedPrompt.configuration.temperature, - maxTokens = adaptedPrompt.configuration.minResponseTokens, - ) + val request = + ChatCompletionRequest( + user = adaptedPrompt.configuration.user, + messages = adaptedPrompt.messages, + n = adaptedPrompt.configuration.numberOfPredictions, + temperature = adaptedPrompt.configuration.temperature, + maxTokens = adaptedPrompt.configuration.minResponseTokens, + ) - return createChatCompletion(request) - .choices - .addChoiceToMemory(scope, promptMemories) - .mapNotNull { it.message?.content } - } + createChatCompletion(request) + .addMetrics(scope) + .choices + .addChoiceToMemory(scope, promptMemories) + .mapNotNull { it.message?.content } + } } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index f6f98e9b2..5fc814a88 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -64,37 +64,36 @@ interface ChatWithFunctions : LLM { scope: Conversation, function: CFunction, serializer: (json: String) -> A, - ): A { - val promptWithFunctions = prompt.copy(function = function) - val adaptedPrompt = - PromptCalculator.adaptPromptToConversationAndModel( - promptWithFunctions, - scope, - this@ChatWithFunctions - ) + ): A = + scope.metric.promptSpan(scope, prompt) { + val promptWithFunctions = prompt.copy(function = function) + val adaptedPrompt = + PromptCalculator.adaptPromptToConversationAndModel( + promptWithFunctions, + scope, + this@ChatWithFunctions + ) - val request = - FunChatCompletionRequest( - user = adaptedPrompt.configuration.user, - messages = adaptedPrompt.messages, - n = adaptedPrompt.configuration.numberOfPredictions, - temperature = adaptedPrompt.configuration.temperature, - maxTokens = adaptedPrompt.configuration.minResponseTokens, - functions = adaptedPrompt.function!!.nel(), - functionCall = mapOf("name" to (adaptedPrompt.function.name)), - ) + val request = + FunChatCompletionRequest( + user = adaptedPrompt.configuration.user, + messages = adaptedPrompt.messages, + n = adaptedPrompt.configuration.numberOfPredictions, + temperature = adaptedPrompt.configuration.temperature, + maxTokens = adaptedPrompt.configuration.minResponseTokens, + functions = adaptedPrompt.function!!.nel(), + functionCall = mapOf("name" to (adaptedPrompt.function.name)), + ) - return tryDeserialize( - serializer, - promptWithFunctions.configuration.maxDeserializationAttempts - ) { - val requestedMemories = prompt.messages.toMemory(scope) - createChatCompletionWithFunctions(request) - .choices - .addChoiceWithFunctionsToMemory(scope, requestedMemories) - .mapNotNull { it.message?.functionCall?.arguments } + tryDeserialize(serializer, promptWithFunctions.configuration.maxDeserializationAttempts) { + val requestedMemories = prompt.messages.toMemory(scope) + createChatCompletionWithFunctions(request) + .addMetrics(scope) + .choices + .addChoiceWithFunctionsToMemory(scope, requestedMemories) + .mapNotNull { it.message?.functionCall?.arguments } + } } - } @AiDsl fun promptStreaming( diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt index f41e4bfe1..e11e28bba 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt @@ -18,7 +18,9 @@ interface Embeddings : LLM { else texts .chunked(chunkSize ?: 400) - .parMap { createEmbeddings(EmbeddingRequest(name, it, requestConfig.user.id)).data } + .parMap { + createEmbeddings(EmbeddingRequest(modelType.name, it, requestConfig.user.id)).data + } .flatten() suspend fun embedQuery(text: String, requestConfig: RequestConfig): List = diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/MetricManagement.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/MetricManagement.kt new file mode 100644 index 000000000..c0a1be18d --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/MetricManagement.kt @@ -0,0 +1,25 @@ +package com.xebia.functional.xef.llm + +import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.llm.models.chat.ChatCompletionResponse +import com.xebia.functional.xef.llm.models.chat.ChatCompletionResponseWithFunctions + +fun ChatCompletionResponseWithFunctions.addMetrics( + conversation: Conversation +): ChatCompletionResponseWithFunctions { + conversation.metric.log(conversation, "Model: ${`object`}") + conversation.metric.log( + conversation, + "Tokens: ${usage.promptTokens} (prompt) + ${usage.completionTokens} (completion) = ${usage.totalTokens}" + ) + return this +} + +fun ChatCompletionResponse.addMetrics(conversation: Conversation): ChatCompletionResponse { + conversation.metric.log(conversation, "Model: ${`object`}") + conversation.metric.log( + conversation, + "Tokens: ${usage.promptTokens} (prompt) + ${usage.completionTokens} (completion) = ${usage.totalTokens}" + ) + return this +} diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/metrics/LogsMetric.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/metrics/LogsMetric.kt new file mode 100644 index 000000000..e7279e2cb --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/metrics/LogsMetric.kt @@ -0,0 +1,29 @@ +package com.xebia.functional.xef.metrics + +import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.prompt.Prompt +import io.ktor.util.date.* + +class LogsMetric : Metric { + + private val identSize = 4 + + override suspend fun promptSpan( + conversation: Conversation, + prompt: Prompt, + block: suspend Metric.() -> A + ): A { + val milis = getTimeMillis() + val name = prompt.messages.lastOrNull()?.content ?: "empty" + println("Prompt-Span: $name") + val output = block() + println("${writeIdent()}|-- Finished in ${getTimeMillis()-milis} ms") + return output + } + + override fun log(conversation: Conversation, message: String) { + println("${writeIdent()}|-- $message".padStart(identSize, ' ')) + } + + private fun writeIdent(times: Int = 1) = (1..identSize * times).fold("") { a, b -> "$a " } +} diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/metrics/Metric.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/metrics/Metric.kt new file mode 100644 index 000000000..f8b9f4e62 --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/metrics/Metric.kt @@ -0,0 +1,14 @@ +package com.xebia.functional.xef.metrics + +import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.prompt.Prompt + +interface Metric { + suspend fun promptSpan( + conversation: Conversation, + prompt: Prompt, + block: suspend Metric.() -> A + ): A + + fun log(conversation: Conversation, message: String) +} diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index ffd818ecb..49128af5c 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -4,6 +4,7 @@ import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.data.* import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.chat.Role +import com.xebia.functional.xef.metrics.LogsMetric import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistant import com.xebia.functional.xef.prompt.templates.system @@ -26,7 +27,12 @@ class ConversationSpec : val model = TestModel(modelType = ModelType.ADA) - val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) + val scope = + Conversation( + LocalVectorStore(TestEmbeddings()), + LogsMetric(), + conversationId = conversationId + ) val vectorStore = scope.store @@ -48,7 +54,12 @@ class ConversationSpec : |""" { val messages = generateRandomMessages(50, 40, 60) val conversationId = ConversationId(UUID.generateUUID().toString()) - val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) + val scope = + Conversation( + LocalVectorStore(TestEmbeddings()), + LogsMetric(), + conversationId = conversationId + ) val vectorStore = scope.store val modelAda = TestModel(modelType = ModelType.ADA, responses = messages) @@ -85,7 +96,12 @@ class ConversationSpec : |""" { val messages = generateRandomMessages(50, 40, 60) val conversationId = ConversationId(UUID.generateUUID().toString()) - val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) + val scope = + Conversation( + LocalVectorStore(TestEmbeddings()), + LogsMetric(), + conversationId = conversationId + ) val vectorStore = scope.store val modelGPTTurbo16K = @@ -122,7 +138,12 @@ class ConversationSpec : val message = mapOf(question to Json.encodeToString(answer)) val conversationId = ConversationId(UUID.generateUUID().toString()) - val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) + val scope = + Conversation( + LocalVectorStore(TestEmbeddings()), + LogsMetric(), + conversationId = conversationId + ) val model = TestFunctionsModel(modelType = ModelType.GPT_3_5_TURBO_FUNCTIONS, responses = message) @@ -146,7 +167,12 @@ class ConversationSpec : val message = mapOf(questionJsonString to answerJsonString) val conversationId = ConversationId(UUID.generateUUID().toString()) - val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) + val scope = + Conversation( + LocalVectorStore(TestEmbeddings()), + LogsMetric(), + conversationId = conversationId + ) val model = TestFunctionsModel(modelType = ModelType.GPT_3_5_TURBO_FUNCTIONS, responses = message) @@ -170,7 +196,12 @@ class ConversationSpec : val model = TestModel(modelType = ModelType.ADA) - val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) + val scope = + Conversation( + LocalVectorStore(TestEmbeddings()), + LogsMetric(), + conversationId = conversationId + ) val vectorStore = scope.store @@ -218,7 +249,7 @@ class ConversationSpec : val vectorStore = LocalVectorStore(TestEmbeddings()) - val scope1 = Conversation(vectorStore, conversationId = conversationId) + val scope1 = Conversation(vectorStore, LogsMetric(), conversationId = conversationId) val firstPrompt = Prompt { +user("question in scope 1") @@ -227,7 +258,7 @@ class ConversationSpec : model.promptMessages(prompt = firstPrompt, scope = scope1) - val scope2 = Conversation(vectorStore, conversationId = conversationId) + val scope2 = Conversation(vectorStore, LogsMetric(), conversationId = conversationId) val secondPrompt = Prompt { +user("question in scope 2") diff --git a/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/JSConversation.kt b/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/JSConversation.kt index 2b9373f78..4f54be1f2 100644 --- a/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/JSConversation.kt +++ b/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/JSConversation.kt @@ -1,10 +1,12 @@ package com.xebia.functional.xef.conversation +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore class JSConversation( override val store: VectorStore, + override val metric: Metric, override val conversationId: ConversationId? ) : PlatformConversation(store, conversationId) { diff --git a/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.js.kt b/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.js.kt index c16804cb7..d4307a70a 100644 --- a/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.js.kt +++ b/core/src/jsMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.js.kt @@ -1,14 +1,19 @@ package com.xebia.functional.xef.conversation +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore actual abstract class PlatformConversation actual constructor(store: VectorStore, conversationId: ConversationId?) : Conversation, AutoClose { actual companion object { - actual fun create(store: VectorStore, conversationId: ConversationId?): PlatformConversation { + actual fun create( + store: VectorStore, + metric: Metric, + conversationId: ConversationId? + ): PlatformConversation { conversationId?.let { store.updateIndexByConversationId(conversationId) } - return JSConversation(store, conversationId) + return JSConversation(store, metric, conversationId) } } } diff --git a/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/JVMConversation.kt b/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/JVMConversation.kt index c25a76333..ce45a38c3 100644 --- a/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/JVMConversation.kt +++ b/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/JVMConversation.kt @@ -1,11 +1,13 @@ package com.xebia.functional.xef.conversation +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore import java.io.Closeable open class JVMConversation( override val store: VectorStore, + override val metric: Metric, override val conversationId: ConversationId?, ) : PlatformConversation(store, conversationId), AutoClose by autoClose(), AutoCloseable, Closeable { diff --git a/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.jvm.kt b/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.jvm.kt index 67884e51b..2f34b8de2 100644 --- a/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.jvm.kt +++ b/core/src/jvmMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.jvm.kt @@ -6,6 +6,7 @@ import com.xebia.functional.xef.llm.ChatWithFunctions import com.xebia.functional.xef.llm.Images import com.xebia.functional.xef.llm.models.functions.CFunction import com.xebia.functional.xef.llm.models.images.ImagesGenerationResponse +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore @@ -97,9 +98,13 @@ actual constructor( .asCompletableFuture() actual companion object { - actual fun create(store: VectorStore, conversationId: ConversationId?): PlatformConversation { + actual fun create( + store: VectorStore, + metric: Metric, + conversationId: ConversationId? + ): PlatformConversation { conversationId?.let { store.updateIndexByConversationId(conversationId) } - return JVMConversation(store, conversationId) + return JVMConversation(store, metric, conversationId) } } diff --git a/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/NativeConversation.kt b/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/NativeConversation.kt index 48954b611..f940fc4bd 100644 --- a/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/NativeConversation.kt +++ b/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/NativeConversation.kt @@ -1,10 +1,12 @@ package com.xebia.functional.xef.conversation +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore class NativeConversation( override val store: VectorStore, + override val metric: Metric, override val conversationId: ConversationId?, ) : PlatformConversation(store, conversationId) { diff --git a/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.native.kt b/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.native.kt index 9a1b832f4..58a158e30 100644 --- a/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.native.kt +++ b/core/src/nativeMain/kotlin/com/xebia/functional/xef/conversation/PlatformConversation.native.kt @@ -1,5 +1,6 @@ package com.xebia.functional.xef.conversation +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.ConversationId import com.xebia.functional.xef.store.VectorStore @@ -7,9 +8,13 @@ actual abstract class PlatformConversation actual constructor(store: VectorStore, conversationId: ConversationId?) : Conversation, AutoClose, AutoCloseable { actual companion object { - actual fun create(store: VectorStore, conversationId: ConversationId?): PlatformConversation { + actual fun create( + store: VectorStore, + metric: Metric, + conversationId: ConversationId? + ): PlatformConversation { conversationId?.let { store.updateIndexByConversationId(conversationId) } - return NativeConversation(store, conversationId) + return NativeConversation(store, metric, conversationId) } } } diff --git a/examples/kotlin/build.gradle.kts b/examples/kotlin/build.gradle.kts index 2b9329b1b..4b76004af 100644 --- a/examples/kotlin/build.gradle.kts +++ b/examples/kotlin/build.gradle.kts @@ -26,6 +26,7 @@ dependencies { implementation(projects.xefGcp) implementation(projects.xefOpenai) implementation(projects.xefReasoning) + implementation(projects.xefOpentelemetry) implementation(libs.kotlinx.serialization.json) implementation(libs.logback) implementation(libs.klogging) diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt index 251fce55a..685d63873 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/conversations/Animal.kt @@ -14,6 +14,12 @@ import kotlinx.serialization.Serializable data class Invention(val name: String, val inventor: String, val year: Int, val purpose: String) suspend fun main() { + // This example contemplate the case of using OpenTelemetry for metrics + // To run the example with OpenTelemetry, you can execute the following commands: + // - # docker compose-up server/docker/opentelemetry + + // OpenAI.conversation(LocalVectorStore(OpenAI().DEFAULT_EMBEDDING), OpenTelemetryMetric()) + OpenAI.conversation { val animal: Animal = prompt("A unique animal species.") val invention: Invention = prompt("A groundbreaking invention from the 20th century.") diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/manual/NoAI.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/manual/NoAI.kt index de72345ae..5d2403806 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/manual/NoAI.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/manual/NoAI.kt @@ -4,6 +4,7 @@ import com.xebia.functional.gpt4all.GPT4All import com.xebia.functional.gpt4all.HuggingFaceLocalEmbeddings import com.xebia.functional.gpt4all.huggingFaceUrl import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.metrics.LogsMetric import com.xebia.functional.xef.pdf.pdf import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.store.LocalVectorStore @@ -24,7 +25,7 @@ suspend fun main() { // Create an instance of the embeddings val embeddings = HuggingFaceLocalEmbeddings.DEFAULT - val scope = Conversation(LocalVectorStore(embeddings)) + val scope = Conversation(LocalVectorStore(embeddings), LogsMetric()) // Fetch and add texts from a PDF document to the vector store val results = pdf("https://arxiv.org/pdf/2305.10601.pdf") diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/serialization/Movie.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/serialization/Movie.kt index 3032ae927..7b35df57d 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/serialization/Movie.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/serialization/Movie.kt @@ -2,6 +2,7 @@ package com.xebia.functional.xef.conversation.serialization import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.conversation.llm.openai.OpenAI +import com.xebia.functional.xef.metrics.LogsMetric import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.store.LocalVectorStore import kotlinx.serialization.Serializable @@ -18,7 +19,7 @@ suspend fun main() { // val openAI = OpenAI(host = "http://localhost:8081/") val model = openAI.DEFAULT_SERIALIZATION - val scope = Conversation(LocalVectorStore(openAI.DEFAULT_EMBEDDING)) + val scope = Conversation(LocalVectorStore(openAI.DEFAULT_EMBEDDING), LogsMetric()) model .prompt( diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/OpenAIStreamingExample.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/OpenAIStreamingExample.kt index 7a729a697..b3ac22077 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/OpenAIStreamingExample.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/OpenAIStreamingExample.kt @@ -3,13 +3,14 @@ package com.xebia.functional.xef.conversation.streaming import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.llm.Chat +import com.xebia.functional.xef.metrics.LogsMetric import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.store.LocalVectorStore suspend fun main() { val chat: Chat = OpenAI().DEFAULT_CHAT val embeddings = OpenAI().DEFAULT_EMBEDDING - val scope = Conversation(LocalVectorStore(embeddings)) + val scope = Conversation(LocalVectorStore(embeddings), LogsMetric()) chat.promptStreaming(prompt = Prompt("What is the meaning of life?"), scope = scope).collect { print(it) } diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/SpaceCraft.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/SpaceCraft.kt index 3559219e5..e517ec1ef 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/SpaceCraft.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/conversation/streaming/SpaceCraft.kt @@ -5,6 +5,7 @@ import com.xebia.functional.xef.conversation.Description import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.promptStreaming import com.xebia.functional.xef.llm.StreamedFunction +import com.xebia.functional.xef.metrics.LogsMetric import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.store.LocalVectorStore import kotlinx.serialization.Serializable @@ -44,7 +45,7 @@ suspend fun main() { // val openAI = OpenAI(host = "http://localhost:8081/") val model = openAI.DEFAULT_SERIALIZATION - val scope = Conversation(LocalVectorStore(openAI.DEFAULT_EMBEDDING)) + val scope = Conversation(LocalVectorStore(openAI.DEFAULT_EMBEDDING), LogsMetric()) model .promptStreaming( diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Conversation.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Conversation.kt index 1e58d099d..d4fe96f01 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Conversation.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Conversation.kt @@ -1,10 +1,13 @@ package com.xebia.functional.gpt4all import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.metrics.LogsMetric +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore suspend inline fun conversation( store: VectorStore = LocalVectorStore(HuggingFaceLocalEmbeddings.DEFAULT), + metric: Metric = LogsMetric(), noinline block: suspend Conversation.() -> A -): A = block(Conversation(store)) +): A = block(Conversation(store, metric)) diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt index e24fb2c37..c935bfcdd 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt @@ -14,6 +14,8 @@ import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult import com.xebia.functional.xef.llm.models.usage.Usage +import com.xebia.functional.xef.metrics.LogsMetric +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore import kotlinx.coroutines.Dispatchers @@ -45,13 +47,14 @@ interface GPT4All : AutoCloseable, Chat, Completion { @JvmSynthetic suspend fun conversation( block: suspend Conversation.() -> A - ): A = block(conversation(LocalVectorStore(HuggingFaceLocalEmbeddings.DEFAULT))) + ): A = block(conversation()) @JvmStatic @JvmOverloads fun conversation( - store: VectorStore = LocalVectorStore(HuggingFaceLocalEmbeddings.DEFAULT) - ): PlatformConversation = Conversation(store) + store: VectorStore = LocalVectorStore(HuggingFaceLocalEmbeddings.DEFAULT), + metric: Metric = LogsMetric() + ): PlatformConversation = Conversation(store, metric) operator fun invoke( url: String, diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b5bffe887..f94ed8ef2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -45,6 +45,8 @@ suspendApp = "0.4.0" flyway = "9.22.2" resources-kmp = "0.4.0" detekt = "1.23.1" +opentelemetry="1.30.1" +opentelemetry-alpha="1.30.1-alpha" [libraries] arrow-core = { module = "io.arrow-kt:arrow-core", version.ref = "arrow" } @@ -126,6 +128,12 @@ jackson-schema-jakarta = { module = "com.github.victools:jsonschema-module-jakar jakarta-validation = { module = "jakarta.validation:jakarta.validation-api", version.ref = "jakarta" } detekt-api = { module = "io.gitlab.arturbosch.detekt:detekt-api", version.ref = "detekt" } detekt-test = { module = "io.gitlab.arturbosch.detekt:detekt-test", version.ref = "detekt" } +opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } +opentelemetry-exporter-logging = { module = "io.opentelemetry:opentelemetry-exporter-logging", version.ref = "opentelemetry" } +opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version.ref = "opentelemetry" } +opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" } +opentelemetry-semconv = { module = "io.opentelemetry:opentelemetry-semconv", version.ref = "opentelemetry-alpha" } +opentelemetry-extension-kotlin = { module = "io.opentelemetry:opentelemetry-extension-kotlin", version.ref = "opentelemetry" } [bundles] ktor-client = [ diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt index 81ef15a23..817b04f60 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt @@ -9,6 +9,8 @@ import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.gcp.models.GcpChat import com.xebia.functional.xef.gcp.models.GcpEmbeddings import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.metrics.LogsMetric +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore import kotlin.jvm.JvmField @@ -62,8 +64,9 @@ class GCP(projectId: String? = null, location: VertexAIRegion? = null, token: St @JvmSynthetic suspend inline fun conversation( store: VectorStore, + metric: Metric, noinline block: suspend Conversation.() -> A - ): A = block(conversation(store)) + ): A = block(conversation(store, metric)) @JvmSynthetic suspend fun conversation(block: suspend Conversation.() -> A): A = @@ -72,10 +75,11 @@ class GCP(projectId: String? = null, location: VertexAIRegion? = null, token: St @JvmStatic @JvmOverloads fun conversation( - store: VectorStore = LocalVectorStore(FromEnvironment.DEFAULT_EMBEDDING) - ): PlatformConversation = Conversation(store) + store: VectorStore = LocalVectorStore(FromEnvironment.DEFAULT_EMBEDDING), + metric: Metric = LogsMetric(), + ): PlatformConversation = Conversation(store, metric) } } suspend inline fun GCP.conversation(noinline block: suspend Conversation.() -> A): A = - block(Conversation(LocalVectorStore(DEFAULT_EMBEDDING))) + block(Conversation(LocalVectorStore(DEFAULT_EMBEDDING), LogsMetric())) diff --git a/integrations/opentelemetry/build.gradle.kts b/integrations/opentelemetry/build.gradle.kts new file mode 100644 index 000000000..c864765bd --- /dev/null +++ b/integrations/opentelemetry/build.gradle.kts @@ -0,0 +1,53 @@ +plugins { + id(libs.plugins.kotlin.jvm.get().pluginId) + id(libs.plugins.kotlinx.serialization.get().pluginId) + alias(libs.plugins.arrow.gradle.publish) + alias(libs.plugins.semver.gradle) + alias(libs.plugins.detekt) +} + +dependencies { detektPlugins(project(":detekt-rules")) } + +repositories { mavenCentral() } + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + toolchain { languageVersion = JavaLanguageVersion.of(11) } +} + +detekt { + toolVersion = "1.23.1" + source = files("src/main/kotlin") + config.setFrom("../../config/detekt/detekt.yml") + autoCorrect = true +} + +dependencies { + implementation(projects.xefCore) + implementation(libs.opentelemetry.api) + implementation(libs.opentelemetry.exporter.logging) + implementation(libs.opentelemetry.sdk) + implementation(libs.opentelemetry.semconv) + implementation(libs.opentelemetry.extension.kotlin) + implementation(libs.opentelemetry.exporter.otlp) + + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.kotest.property) + testImplementation(libs.kotest.framework) + testImplementation(libs.kotest.assertions) + testRuntimeOnly(libs.kotest.junit5) +} + +tasks { + withType().configureEach { + dependsOn(":detekt-rules:assemble") + autoCorrect = true + } + named("detekt") { + dependsOn(":detekt-rules:assemble") + getByName("build").dependsOn(this) + } + + withType { dependsOn(withType()) } +} diff --git a/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt new file mode 100644 index 000000000..836811c50 --- /dev/null +++ b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryConfig.kt @@ -0,0 +1,50 @@ +package com.xebia.functional.xef.opentelemetry + +import io.opentelemetry.api.OpenTelemetry +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.common.Attributes +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor +import java.util.concurrent.TimeUnit + +data class OpenTelemetryConfig( + val endpointConfig: String, + val defaultScopeName: String, + val serviceName: String +) { + + fun newInstance(): OpenTelemetry { + val jaegerOtlpExporter: OtlpGrpcSpanExporter = OtlpGrpcSpanExporter.builder() + .setEndpoint(endpointConfig) + .setTimeout(30, TimeUnit.SECONDS) + .build() + + val serviceNameResource: Resource = + Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), serviceName)) + + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.builder(jaegerOtlpExporter).build()) + .setResource(Resource.getDefault().merge(serviceNameResource)) + .build() + + + val openTelemetry = OpenTelemetrySdk + .builder() + .setTracerProvider(tracerProvider) + .build() + + Runtime.getRuntime().addShutdownHook(Thread { tracerProvider.close() }) + return openTelemetry + } + + companion object { + val DEFAULT = OpenTelemetryConfig( + endpointConfig = "http://localhost:4317", + defaultScopeName = "io.xef", + serviceName = "xef" + ) + } +} diff --git a/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryMetric.kt b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryMetric.kt new file mode 100644 index 000000000..897323347 --- /dev/null +++ b/integrations/opentelemetry/src/main/kotlin/com/xebia/functional/xef/opentelemetry/OpenTelemetryMetric.kt @@ -0,0 +1,68 @@ +package com.xebia.functional.xef.opentelemetry + +import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.metrics.Metric +import com.xebia.functional.xef.prompt.Prompt +import com.xebia.functional.xef.store.ConversationId +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context + +class OpenTelemetryMetric( + private val config: OpenTelemetryConfig = OpenTelemetryConfig.DEFAULT +) : Metric { + + private val conversations = mutableListOf>() + + private val openTelemetry = config.newInstance() + + override suspend fun promptSpan(conversation: Conversation, prompt: Prompt, block: suspend Metric.() -> A): A { + val cid = conversation.conversationId ?: return block() + + val parentContext = cid.getParentConversation() + + val span = getTracer() + .spanBuilder("Prompt: ${prompt.messages.lastOrNull()?.content ?: "empty"}") + .setParent(parentContext) + .startSpan() + + return try { + val output = block() + span.makeCurrent().use { + span.setAttribute("number-of-messages", prompt.messages.count().toString()) + span.setAttribute("last-message", prompt.messages.lastOrNull()?.content ?: "empty") + } + output + } finally { + span.end() + } + } + + override fun log(conversation: Conversation, message: String) { + val cid = conversation.conversationId ?: return + + val parentContext = cid.getParentConversation() + + val span: Span = getTracer().spanBuilder(message) + .setParent(parentContext) + .startSpan() + span.end() + } + + private fun ConversationId.getParentConversation(): Context { + val parent = conversations.find { it.first == this }?.second + return if (parent == null) { + val newParent = getTracer() + .spanBuilder(value) + .startSpan() + newParent.end() + val newContext = Context.current().with(newParent) + conversations.add(this to newContext) + newContext + } else parent + } + + private fun getTracer(scopeName: String? = null): Tracer = + openTelemetry.getTracer(scopeName ?: config.defaultScopeName) + +} diff --git a/kotlin/src/commonMain/kotlin/com/xebia/functional/xef/conversation/DSLExtensions.kt b/kotlin/src/commonMain/kotlin/com/xebia/functional/xef/conversation/DSLExtensions.kt index 57e01aa7b..c7312d0e7 100644 --- a/kotlin/src/commonMain/kotlin/com/xebia/functional/xef/conversation/DSLExtensions.kt +++ b/kotlin/src/commonMain/kotlin/com/xebia/functional/xef/conversation/DSLExtensions.kt @@ -1,6 +1,8 @@ package com.xebia.functional.xef.conversation import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.metrics.LogsMetric +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore @@ -15,5 +17,6 @@ import com.xebia.functional.xef.store.VectorStore suspend inline fun conversation( embeddings: Embeddings, store: VectorStore = LocalVectorStore(embeddings), + metric: Metric = LogsMetric(), noinline block: suspend Conversation.() -> A -): A = block(Conversation(store)) +): A = block(Conversation(store, metric)) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index dd6a0713e..651945c37 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -16,6 +16,8 @@ import com.xebia.functional.xef.conversation.autoClose import com.xebia.functional.xef.conversation.llm.openai.models.* import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.metrics.LogsMetric +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore import kotlin.jvm.JvmField @@ -182,17 +184,18 @@ class OpenAI( @JvmSynthetic suspend inline fun conversation( store: VectorStore, + metric: Metric, noinline block: suspend Conversation.() -> A - ): A = block(conversation(store)) + ): A = block(conversation(store, metric)) @JvmSynthetic - suspend fun conversation(block: suspend Conversation.() -> A): A = - block(conversation(LocalVectorStore(FromEnvironment.DEFAULT_EMBEDDING))) + suspend fun conversation(block: suspend Conversation.() -> A): A = block(conversation()) @JvmStatic @JvmOverloads fun conversation( - store: VectorStore = LocalVectorStore(FromEnvironment.DEFAULT_EMBEDDING) - ): PlatformConversation = Conversation(store) + store: VectorStore = LocalVectorStore(FromEnvironment.DEFAULT_EMBEDDING), + metric: Metric = LogsMetric() + ): PlatformConversation = Conversation(store, metric) } } diff --git a/scala/src/main/scala/com/xebia/functional/xef/scala/conversation/package.scala b/scala/src/main/scala/com/xebia/functional/xef/scala/conversation/package.scala index bc3b759da..7f4d107c6 100644 --- a/scala/src/main/scala/com/xebia/functional/xef/scala/conversation/package.scala +++ b/scala/src/main/scala/com/xebia/functional/xef/scala/conversation/package.scala @@ -6,6 +6,7 @@ import com.xebia.functional.xef.conversation.{FromJson, JVMConversation} import com.xebia.functional.xef.llm.* import com.xebia.functional.xef.llm.models.images.* import com.xebia.functional.xef.store.{ConversationId, LocalVectorStore, VectorStore} +import com.xebia.functional.xef.metrics.{LogsMetric, Metric} import io.circe.Decoder import io.circe.parser.parse import org.reactivestreams.{Subscriber, Subscription} @@ -15,7 +16,8 @@ import java.util.UUID import java.util.concurrent.LinkedBlockingQueue import scala.jdk.CollectionConverters.* -class ScalaConversation(store: VectorStore, conversationId: Option[ConversationId]) extends JVMConversation(store, conversationId.orNull) +class ScalaConversation(store: VectorStore, metric: Metric, conversationId: Option[ConversationId]) + extends JVMConversation(store, metric, conversationId.orNull) def addContext(context: Array[String])(using conversation: ScalaConversation): Unit = conversation.addContextFromArray(context).join() @@ -79,4 +81,4 @@ def images( def conversation[A]( block: ScalaConversation ?=> A, conversationId: Option[ConversationId] = Some(ConversationId(UUID.randomUUID().toString)) -): A = block(using ScalaConversation(LocalVectorStore(OpenAI.FromEnvironment.DEFAULT_EMBEDDING), conversationId)) +): A = block(using ScalaConversation(LocalVectorStore(OpenAI.FromEnvironment.DEFAULT_EMBEDDING), LogsMetric(), conversationId)) diff --git a/server/docker/opentelemetry/docker-compose.yml b/server/docker/opentelemetry/docker-compose.yml new file mode 100644 index 000000000..49ba02528 --- /dev/null +++ b/server/docker/opentelemetry/docker-compose.yml @@ -0,0 +1,9 @@ +version: "3" +services: + jaeger: + image: jaegertracing/all-in-one:latest + environment: + COLLECTOR_OTLP_ENABLED: true + ports: + - 4317:4317 + - 16686:16686 diff --git a/settings.gradle.kts b/settings.gradle.kts index 4ac46abbe..43ac9bce2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,6 +48,9 @@ project(":xef-sql").projectDir = file("integrations/sql") include("xef-gcp") project(":xef-gcp").projectDir = file("integrations/gcp") + +include("xef-opentelemetry") +project(":xef-opentelemetry").projectDir = file("integrations/opentelemetry") // //