diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/CoreAIScope.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/CoreAIScope.kt index 4ca848e44..7f59a2569 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/CoreAIScope.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/CoreAIScope.kt @@ -99,14 +99,15 @@ class CoreAIScope( functions: List<CFunction>, serializer: (json: String) -> A, promptConfiguration: PromptConfiguration, - ): A = - prompt( + ): A { + return prompt( prompt = Prompt(prompt), context = context, functions = functions, serializer = serializer, promptConfiguration = promptConfiguration, ) + } @AiDsl suspend fun Chat.promptMessage( diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/PromptConfiguration.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/PromptConfiguration.kt index 4bc979405..4886a5ca2 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/PromptConfiguration.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/auto/PromptConfiguration.kt @@ -11,6 +11,7 @@ class PromptConfiguration( val numberOfPredictions: Int = 1, val docsInContext: Int = 20, val minResponseTokens: Int = 500, + val streamToStandardOut: Boolean = false ) { companion object { @@ -21,11 +22,16 @@ class PromptConfiguration( private var numberOfPredictions: Int = 1 private var docsInContext: Int = 20 private var minResponseTokens: Int = 500 + private var streamToStandardOut: Boolean = false fun maxDeserializationAttempts(maxDeserializationAttempts: Int) = apply { this.maxDeserializationAttempts = maxDeserializationAttempts } + fun streamToStandardOut(streamToStandardOut: Boolean) = apply { + this.streamToStandardOut = streamToStandardOut + } + fun user(user: String) = apply { this.user = user } fun temperature(temperature: Double) = apply { this.temperature = temperature } @@ -47,7 +53,8 @@ class PromptConfiguration( temperature, numberOfPredictions, docsInContext, - minResponseTokens + minResponseTokens, + streamToStandardOut ) } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 2df27608e..24ebffd40 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -51,28 +51,33 @@ interface Chat : LLM { return totalLeftTokens } - val userMessage = Message(Role.USER, promptWithContext, Role.USER.name) - fun buildChatRequest(): ChatCompletionRequest = - ChatCompletionRequest( + fun buildChatRequest(): ChatCompletionRequest { + val messages: List<Message> = listOf(Message(Role.USER, promptWithContext, Role.USER.name)) + return ChatCompletionRequest( model = name, user = promptConfiguration.user, - messages = listOf(userMessage), + messages = messages, n = promptConfiguration.numberOfPredictions, temperature = promptConfiguration.temperature, - maxTokens = checkTotalLeftChatTokens(listOf(userMessage)) + maxTokens = checkTotalLeftChatTokens(messages), + streamToStandardOut = promptConfiguration.streamToStandardOut ) + } - fun chatWithFunctionsRequest(): ChatCompletionRequestWithFunctions = - ChatCompletionRequestWithFunctions( + fun chatWithFunctionsRequest(): ChatCompletionRequestWithFunctions { + val firstFnName: String? = functions.firstOrNull()?.name + val messages: List<Message> = listOf(Message(Role.USER, promptWithContext, Role.USER.name)) + return ChatCompletionRequestWithFunctions( model = name, user = promptConfiguration.user, - messages = listOf(userMessage), + messages = messages, n = promptConfiguration.numberOfPredictions, temperature = promptConfiguration.temperature, - maxTokens = checkTotalLeftChatTokens(listOf(userMessage)), + maxTokens = checkTotalLeftChatTokens(messages), functions = functions, - functionCall = mapOf("name" to (functions.firstOrNull()?.name ?: "")) + functionCall = mapOf("name" to (firstFnName ?: "")) ) + } return when (this) { is ChatWithFunctions -> diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index 2de5b177b..8e628e590 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -24,8 +24,8 @@ interface ChatWithFunctions : Chat { functions: List<CFunction>, serializer: (json: String) -> A, promptConfiguration: PromptConfiguration, - ): A = - tryDeserialize(serializer, promptConfiguration.maxDeserializationAttempts) { + ): A { + return tryDeserialize(serializer, promptConfiguration.maxDeserializationAttempts) { promptMessage( prompt = Prompt(prompt), context = context, @@ -33,6 +33,7 @@ interface ChatWithFunctions : Chat { promptConfiguration ) } + } @AiDsl suspend fun <A> prompt( @@ -41,10 +42,11 @@ interface ChatWithFunctions : Chat { functions: List<CFunction>, serializer: (json: String) -> A, promptConfiguration: PromptConfiguration, - ): A = - tryDeserialize(serializer, promptConfiguration.maxDeserializationAttempts) { + ): A { + return tryDeserialize(serializer, promptConfiguration.maxDeserializationAttempts) { promptMessage(prompt = prompt, context = context, functions = functions, promptConfiguration) } + } private suspend fun <A> tryDeserialize( serializer: (json: String) -> A, diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/chat/ChatCompletionRequest.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/chat/ChatCompletionRequest.kt index 4200f6331..a3954a44e 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/chat/ChatCompletionRequest.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/chat/ChatCompletionRequest.kt @@ -12,5 +12,6 @@ data class ChatCompletionRequest( val presencePenalty: Double = 0.0, val frequencyPenalty: Double = 0.0, val logitBias: Map<String, Int> = emptyMap(), - val user: String? + val user: String?, + val streamToStandardOut: Boolean = false, ) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/text/CompletionRequest.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/text/CompletionRequest.kt index e13fb208d..fcc944764 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/text/CompletionRequest.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/text/CompletionRequest.kt @@ -17,4 +17,5 @@ data class CompletionRequest( val frequencyPenalty: Double = 0.0, val bestOf: Int = 1, val logitBias: Map<String, Int> = emptyMap(), + val streamToStandardOut: Boolean = false ) diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/gpt4all/Chat.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/gpt4all/Chat.kt index 8eb84204e..b0c698983 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/gpt4all/Chat.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/gpt4all/Chat.kt @@ -1,49 +1,44 @@ package com.xebia.functional.xef.auto.gpt4all import com.xebia.functional.gpt4all.GPT4All -import com.xebia.functional.gpt4all.LLModel +import com.xebia.functional.gpt4all.Gpt4AllModel import com.xebia.functional.gpt4all.getOrThrow -import com.xebia.functional.gpt4all.huggingFaceUrl import com.xebia.functional.xef.auto.PromptConfiguration import com.xebia.functional.xef.auto.ai -import com.xebia.functional.xef.auto.llm.openai.OpenAI -import com.xebia.functional.xef.pdf.pdf import java.nio.file.Path suspend fun main() { val userDir = System.getProperty("user.dir") - val path = "$userDir/models/gpt4all/ggml-gpt4all-j-v1.3-groovy.bin" - val url = huggingFaceUrl("orel12", "ggml-gpt4all-j-v1.3-groovy", "bin") - val modelType = LLModel.Type.GPTJ - val modelPath: Path = Path.of(path) - val GPT4All = GPT4All(url, modelPath, modelType) + val path = "$userDir/models/gpt4all/ggml-replit-code-v1-3b.bin" - println("🤖 GPT4All loaded: $GPT4All") + val supportedModels = Gpt4AllModel.supportedModels() + supportedModels.forEach { + println("🤖 ${it.name} ${it.url?.let { "- $it" }}") + } - val pdfUrl = "https://www.europarl.europa.eu/RegData/etudes/STUD/2023/740063/IPOL_STU(2023)740063_EN.pdf" + val url = "https://huggingface.co/nomic-ai/ggml-replit-code-v1-3b/resolve/main/ggml-replit-code-v1-3b.bin" + val modelPath: Path = Path.of(path) + val GPT4All = GPT4All(url, modelPath) + println("🤖 GPT4All loaded: $GPT4All") /** * Uses internally [HuggingFaceLocalEmbeddings] default of "sentence-transformers", "msmarco-distilbert-dot-v5" * to provide embeddings for docs in contextScope. */ ai { - println("🤖 Loading PDF: $pdfUrl") - contextScope(pdf(pdfUrl)) { - println("🤖 Context loaded: $context") - GPT4All.use { gpT4All: GPT4All -> - println("🤖 Generating prompt for context") - val prompt = gpT4All.promptMessage( - "Describe in one sentence what the context is about.", + println("🤖 Context loaded: $context") + GPT4All.use { gpT4All: GPT4All -> + println("🤖 Generating prompt for context") + while (true) { + println("🤖 Enter your prompt: ") + val userInput = readlnOrNull() ?: break + gpT4All.promptMessage( + userInput, promptConfiguration = PromptConfiguration { docsInContext(2) + streamToStandardOut(true) }) - println("🤖 Generating images for prompt: \n\n$prompt") - val images = - OpenAI.DEFAULT_IMAGES.images(prompt.joinToString("\n"), promptConfiguration = PromptConfiguration { - docsInContext(1) - }) - println("🤖 Generated images: \n\n${images.data.joinToString("\n") { it.url }}") } } }.getOrThrow() diff --git a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/manual/NoAI.kt b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/manual/NoAI.kt index 0dfcfcc3b..985ba1eb1 100644 --- a/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/manual/NoAI.kt +++ b/examples/kotlin/src/main/kotlin/com/xebia/functional/xef/auto/manual/NoAI.kt @@ -1,16 +1,47 @@ package com.xebia.functional.xef.auto.manual +import com.xebia.functional.gpt4all.GPT4All import com.xebia.functional.gpt4all.HuggingFaceLocalEmbeddings -import com.xebia.functional.xef.auto.llm.openai.OpenAI +import com.xebia.functional.gpt4all.huggingFaceUrl +import com.xebia.functional.xef.auto.PromptConfiguration import com.xebia.functional.xef.pdf.pdf import com.xebia.functional.xef.vectorstores.LocalVectorStore +import java.nio.file.Path suspend fun main() { - val chat = OpenAI.DEFAULT_CHAT - val huggingFaceEmbeddings = HuggingFaceLocalEmbeddings.DEFAULT - val vectorStore = LocalVectorStore(huggingFaceEmbeddings) - val results = pdf("https://www.europarl.europa.eu/RegData/etudes/STUD/2023/740063/IPOL_STU(2023)740063_EN.pdf") + // Choose your base folder for downloaded models + val userDir = System.getProperty("user.dir") + + // Specify the local model path + val modelPath: Path = Path.of("$userDir/models/gpt4all/ggml-gpt4all-j-v1.3-groovy.bin") + + // Specify the Hugging Face URL for the model + val url = huggingFaceUrl("orel12", "ggml-gpt4all-j-v1.3-groovy", "bin") + + // Create an instance of GPT4All with the local model + val gpt4All = GPT4All(url, modelPath) + + // Create an instance of the OPENAI embeddings + val embeddings = HuggingFaceLocalEmbeddings.DEFAULT + + // Create a LocalVectorStore and initialize it with OpenAI Embeddings + val vectorStore = LocalVectorStore(embeddings) + + // Fetch and add texts from a PDF document to the vector store + val results = pdf("https://arxiv.org/pdf/2305.10601.pdf") vectorStore.addTexts(results) - val result: List<String> = chat.promptMessage("What is the content about?", vectorStore) + + // Prompt the GPT4All model with a question and provide the vector store for context + val result: List<String> = gpt4All.use { + it.promptMessage( + question = "What is the Tree of Thoughts framework about?", + context = vectorStore, + promptConfiguration = PromptConfiguration { + docsInContext(5) + } + ) + } + + // Print the response println(result) } diff --git a/gpt4all-kotlin/build.gradle.kts b/gpt4all-kotlin/build.gradle.kts index 12d76721e..1367a3e2d 100644 --- a/gpt4all-kotlin/build.gradle.kts +++ b/gpt4all-kotlin/build.gradle.kts @@ -4,6 +4,7 @@ plugins { alias(libs.plugins.spotless) alias(libs.plugins.arrow.gradle.publish) alias(libs.plugins.semver.gradle) + alias(libs.plugins.kotlinx.serialization) } repositories { @@ -19,7 +20,25 @@ java { } kotlin { - jvm() + jvm { + compilations { + val integrationTest by compilations.creating { + // Create a test task to run the tests produced by this compilation: + tasks.register<Test>("integrationTest") { + description = "Run the integration tests" + group = "verification" + classpath = compileDependencyFiles + runtimeDependencyFiles + output.allOutputs + testClassesDirs = output.classesDirs + + testLogging { + events("passed") + } + } + } + val test by compilations.getting + integrationTest.associateWith(test) + } + } js(IR) { browser() @@ -43,7 +62,7 @@ kotlin { val jvmMain by getting { dependencies { - implementation("net.java.dev.jna:jna-platform:5.13.0") + implementation("com.hexadevlabs:gpt4all-java-binding:+") implementation("ai.djl.huggingface:tokenizers:+") } } @@ -60,6 +79,15 @@ kotlin { } } +tasks.withType<Test>().configureEach { + maxParallelForks = Runtime.getRuntime().availableProcessors() + useJUnitPlatform() + testLogging { + setExceptionFormat("full") + setEvents(listOf("passed", "skipped", "failed", "standardOut", "standardError")) + } +} + tasks.withType<AbstractPublishToMaven> { dependsOn(tasks.withType<Sign>()) } diff --git a/gpt4all-kotlin/src/commonMain/resources/darwin-aarch64/libllama.dylib b/gpt4all-kotlin/src/commonMain/resources/darwin-aarch64/libllama.dylib deleted file mode 100755 index 110d8ae84..000000000 Binary files a/gpt4all-kotlin/src/commonMain/resources/darwin-aarch64/libllama.dylib and /dev/null differ diff --git a/gpt4all-kotlin/src/commonMain/resources/darwin-aarch64/libllmodel.dylib b/gpt4all-kotlin/src/commonMain/resources/darwin-aarch64/libllmodel.dylib deleted file mode 100755 index c2427d138..000000000 Binary files a/gpt4all-kotlin/src/commonMain/resources/darwin-aarch64/libllmodel.dylib and /dev/null differ diff --git a/gpt4all-kotlin/src/commonMain/resources/darwin-x86-64/libllama.dylib b/gpt4all-kotlin/src/commonMain/resources/darwin-x86-64/libllama.dylib deleted file mode 100755 index 110d8ae84..000000000 Binary files a/gpt4all-kotlin/src/commonMain/resources/darwin-x86-64/libllama.dylib and /dev/null differ diff --git a/gpt4all-kotlin/src/commonMain/resources/darwin-x86-64/libllmodel.dylib b/gpt4all-kotlin/src/commonMain/resources/darwin-x86-64/libllmodel.dylib deleted file mode 100755 index c2427d138..000000000 Binary files a/gpt4all-kotlin/src/commonMain/resources/darwin-x86-64/libllmodel.dylib and /dev/null differ diff --git a/gpt4all-kotlin/src/commonMain/resources/linux-x86-64/libllama.so b/gpt4all-kotlin/src/commonMain/resources/linux-x86-64/libllama.so deleted file mode 100755 index 114911861..000000000 Binary files a/gpt4all-kotlin/src/commonMain/resources/linux-x86-64/libllama.so and /dev/null differ diff --git a/gpt4all-kotlin/src/commonMain/resources/linux-x86-64/libllmodel.so b/gpt4all-kotlin/src/commonMain/resources/linux-x86-64/libllmodel.so deleted file mode 100755 index 9935dc42c..000000000 Binary files a/gpt4all-kotlin/src/commonMain/resources/linux-x86-64/libllmodel.so and /dev/null differ diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt index 17a1273ed..cdf78c78b 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt @@ -2,8 +2,7 @@ package com.xebia.functional.gpt4all import ai.djl.training.util.DownloadUtils import ai.djl.training.util.ProgressBar -import com.sun.jna.platform.unix.LibCAPI -import com.xebia.functional.gpt4all.libraries.LLModelContext +import com.hexadevlabs.gpt4all.LLModel import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.llm.Chat @@ -13,16 +12,13 @@ import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult import com.xebia.functional.xef.llm.models.usage.Usage -import java.net.URL import java.nio.file.Files import java.nio.file.Path -import java.nio.file.StandardCopyOption import java.util.* import kotlin.io.path.name -interface GPT4All : AutoCloseable, Chat, Completion { - val gpt4allModel: GPT4AllModel +interface GPT4All : AutoCloseable, Chat, Completion { override fun close() { } @@ -31,22 +27,26 @@ interface GPT4All : AutoCloseable, Chat, Completion { operator fun invoke( url: String, - path: Path, - modelType: LLModel.Type, - generationConfig: GenerationConfig = GenerationConfig(), + path: Path ): GPT4All = object : GPT4All { init { if (!Files.exists(path)) { - DownloadUtils.download(url, path.toFile().absolutePath , ProgressBar()) + DownloadUtils.download(url, path.toFile().absolutePath, ProgressBar()) } } - override val gpt4allModel = GPT4AllModel.invoke(path, modelType) + val llModel = LLModel(path) override suspend fun createCompletion(request: CompletionRequest): CompletionResult = with(request) { - val response: String = gpt4allModel.prompt(prompt, llmModelContext(generationConfig)) + + val config = LLModel.config() + .withTopP(request.topP?.toFloat() ?: 0.4f) + .withTemp(request.temperature?.toFloat() ?: 0f) + .withRepeatPenalty(request.frequencyPenalty.toFloat()) + .build() + val response: String = generateCompletion(prompt, config, request.streamToStandardOut) return CompletionResult( UUID.randomUUID().toString(), path.name, @@ -59,8 +59,13 @@ interface GPT4All : AutoCloseable, Chat, Completion { override suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse = with(request) { - val response: String = - gpt4allModel.prompt(messages.buildPrompt(), llmModelContext(generationConfig)) + val prompt: String = messages.buildPrompt() + val config = LLModel.config() + .withTopP(request.topP.toFloat() ?: 0.4f) + .withTemp(request.temperature.toFloat() ?: 0f) + .withRepeatPenalty(request.frequencyPenalty.toFloat()) + .build() + val response: String = generateCompletion(prompt, config, request.streamToStandardOut) return ChatCompletionResponse( UUID.randomUUID().toString(), path.name, @@ -71,11 +76,13 @@ interface GPT4All : AutoCloseable, Chat, Completion { ) } - override fun tokensFromMessages(messages: List<Message>): Int = 0 + override fun tokensFromMessages(messages: List<Message>): Int { + return 0 + } override val name: String = path.name - override fun close(): Unit = gpt4allModel.close() + override fun close(): Unit = llModel.close() override val modelType: ModelType = ModelType.LocalModel(name, EncodingType.CL100K_BASE, 4096) @@ -90,25 +97,16 @@ interface GPT4All : AutoCloseable, Chat, Completion { return "$messages\n### Response:" } - private fun llmModelContext(generationConfig: GenerationConfig): LLModelContext = - with(generationConfig) { - LLModelContext( - logits_size = LibCAPI.size_t(logitsSize.toLong()), - tokens_size = LibCAPI.size_t(tokensSize.toLong()), - n_past = nPast, - n_ctx = nCtx, - n_predict = nPredict, - top_k = topK, - top_p = topP.toFloat(), - temp = temp.toFloat(), - n_batch = nBatch, - repeat_penalty = repeatPenalty.toFloat(), - repeat_last_n = repeatLastN, - context_erase = contextErase.toFloat() - ) - } + private fun generateCompletion( + prompt: String, + config: LLModel.GenerationConfig, + stream: Boolean, + ): String { + return llModel.generate(prompt, config, stream) + } } } } + diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4AllModel.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4AllModel.kt deleted file mode 100644 index 73c9253e0..000000000 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4AllModel.kt +++ /dev/null @@ -1,39 +0,0 @@ -package com.xebia.functional.gpt4all - -import com.xebia.functional.gpt4all.libraries.LLModelContext -import java.nio.file.Path - -interface GPT4AllModel : AutoCloseable { - val llModel: LLModel - - fun prompt(prompt: String, context: LLModelContext): String - - companion object { - operator fun invoke( - path: Path, - modelType: LLModel.Type - ): GPT4AllModel = object : GPT4AllModel { - override val llModel: LLModel = - when (modelType) { - LLModel.Type.LLAMA -> LlamaModel(path).also { it.loadModel() } - LLModel.Type.GPTJ -> GPTJModel(path).also { it.loadModel() } - } - - override fun prompt(prompt: String, context: LLModelContext): String = - with(llModel) { - val responseBuffer = StringBuffer() - llModelLibrary.llmodel_prompt( - model, - prompt, - { _, _ -> true }, - { _, response -> responseBuffer.append(response).let { true } }, - { _ -> true }, - context - ) - responseBuffer.trim().toString() - } - - override fun close(): Unit = llModel.close() - } - } -} diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt index f2b31980f..e9e165d59 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt @@ -1,7 +1,6 @@ package com.xebia.functional.gpt4all import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer -import com.xebia.functional.xef.embeddings.Embedding as XefEmbedding import com.xebia.functional.xef.embeddings.Embeddings import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest @@ -16,20 +15,30 @@ class HuggingFaceLocalEmbeddings(name: String, artifact: String) : com.xebia.fun override val name: String = HuggingFaceLocalEmbeddings::class.java.canonicalName override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { - val embeddings = tokenizer.batchEncode(request.input).mapIndexed { ix, em -> - Embedding("embedding", em.ids.map { it.toFloat() }, ix) - } - return EmbeddingResult(embeddings, Usage.ZERO) + val embedings = tokenizer.batchEncode(request.input) + return EmbeddingResult( + data = embedings.mapIndexed { n, em -> Embedding("embedding", em.ids.map { it.toFloat() }, n) }, + usage = Usage.ZERO + ) } override suspend fun embedDocuments( texts: List<String>, chunkSize: Int?, requestConfig: RequestConfig - ): List<XefEmbedding> = - tokenizer.batchEncode(texts).map { em -> XefEmbedding(em.ids.map { it.toFloat() }) } + ): List<com.xebia.functional.xef.embeddings.Embedding> { + val encodings = tokenizer.batchEncode(texts) + return encodings.mapIndexed { n, em -> + com.xebia.functional.xef.embeddings.Embedding( + em.ids.map { it.toFloat() }, + ) + } + } - override suspend fun embedQuery(text: String, requestConfig: RequestConfig): List<XefEmbedding> = + override suspend fun embedQuery( + text: String, + requestConfig: RequestConfig + ): List<com.xebia.functional.xef.embeddings.Embedding> = embedDocuments(listOf(text), null, requestConfig) companion object { diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/LLModel.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/LLModel.kt deleted file mode 100644 index 681bc048b..000000000 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/LLModel.kt +++ /dev/null @@ -1,71 +0,0 @@ -package com.xebia.functional.gpt4all - -import com.sun.jna.Library -import com.sun.jna.Native -import com.sun.jna.Pointer -import com.xebia.functional.gpt4all.libraries.LLModelLibrary -import com.xebia.functional.gpt4all.libraries.LlamaLibrary -import java.nio.file.Path - -sealed interface LLModel : AutoCloseable { - val llamaLibrary: LlamaLibrary - val llModelLibrary: LLModelLibrary - val model: Pointer? - val name: String - - enum class Type { - LLAMA, - GPTJ - } - - fun loadModel(): Boolean - - fun type(): Type = - when (this) { - is LlamaModel -> Type.LLAMA - is GPTJModel -> Type.GPTJ - } -} - -interface LlamaModel : LLModel { - companion object { - operator fun invoke( - path: Path - ): LlamaModel = object : LlamaModel { - override val llamaLibrary: LlamaLibrary = loadLlamaLibrary() - override val llModelLibrary: LLModelLibrary.Llama = loadLLModelLibrary() - override val model: Pointer? = llModelLibrary.llmodel_llama_create() - override val name: String = path.getModelName() - override fun loadModel(): Boolean = llModelLibrary.llmodel_loadModel(model, path.toString()) - override fun close(): Unit = llModelLibrary.llmodel_llama_destroy(model) - } - } -} - -interface GPTJModel : LLModel { - companion object { - operator fun invoke( - path: Path - ): GPTJModel = object : GPTJModel { - override val llamaLibrary: LlamaLibrary = loadLlamaLibrary() - override val llModelLibrary: LLModelLibrary.GPTJ = loadLLModelLibrary() - override val model: Pointer? = llModelLibrary.llmodel_gptj_create() - override val name: String = path.getModelName() - override fun loadModel(): Boolean = llModelLibrary.llmodel_loadModel(model, path.toString()) - override fun close(): Unit = llModelLibrary.llmodel_gptj_destroy(model) - } - } -} - -private fun loadLlamaLibrary(): LlamaLibrary = - load<LlamaLibrary>(from = "llama") - -private inline fun <reified T : Library> loadLLModelLibrary(): T = - load<T>(from = "llmodel") - -private fun Path.getModelName(): String = - toFile().name.split( - "\\.(?=[^.]+$)".toRegex() - ).dropLastWhile { it.isEmpty() }.toTypedArray()[0] - -private inline fun <reified T : Library> load(from: String): T = Native.load(from, T::class.java) diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Models.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Models.kt new file mode 100644 index 000000000..ce6fb933e --- /dev/null +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/Models.kt @@ -0,0 +1,32 @@ +package com.xebia.functional.gpt4all + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import java.net.URL + +@Serializable +data class Gpt4AllModel( + val order: String, + val md5sum: String, + val name: String, + val filename: String, + val filesize: String, + val requires: String? = null, + val ramrequired: String, + val parameters: String, + val quant: String, + val type: String, + val description: String, + val disableGUI: String? = null, + val url: String? = null +) { + companion object { + private val url = "https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json" + fun supportedModels(): List<Gpt4AllModel> { + // fetch the content as string from https://raw.githubusercontent.com/nomic-ai/gpt4all/main/gpt4all-chat/metadata/models.json + val json = URL(url).readText() + // parse the json string into a list of Model objects + return Json.decodeFromString<List<Gpt4AllModel>>(json) + } + } +} diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/libraries/LLModelLibrary.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/libraries/LLModelLibrary.kt deleted file mode 100644 index c8b8dde4e..000000000 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/libraries/LLModelLibrary.kt +++ /dev/null @@ -1,85 +0,0 @@ -package com.xebia.functional.gpt4all.libraries - -import com.sun.jna.Callback -import com.sun.jna.Library -import com.sun.jna.Pointer -import com.sun.jna.Structure -import com.sun.jna.platform.unix.LibCAPI - -@Structure.FieldOrder( - "logits", - "logits_size", - "tokens", - "tokens_size", - "n_past", - "n_ctx", - "n_predict", - "top_k", - "top_p", - "temp", - "n_batch", - "repeat_penalty", - "repeat_last_n", - "context_erase" -) -open class LLModelContext( - @JvmField - var logits: Pointer? = null, - @JvmField - var logits_size: LibCAPI.size_t? = null, - @JvmField - var tokens: Pointer? = null, - @JvmField - var tokens_size: LibCAPI.size_t? = null, - @JvmField - var n_past: Int = 0, - @JvmField - var n_ctx: Int = 0, - @JvmField - var n_predict: Int = 0, - @JvmField - var top_k: Int = 0, - @JvmField - var top_p: Float = 0f, - @JvmField - var temp: Float = 0f, - @JvmField - var n_batch: Int = 0, - @JvmField - var repeat_penalty: Float = 0f, - @JvmField - var repeat_last_n: Int = 0, - @JvmField - var context_erase: Float = 0f -) : Structure() - -sealed interface LLModelLibrary : Library { - fun llmodel_loadModel(model: Pointer?, modelPath: String?): Boolean - fun llmodel_isModelLoaded(model: Pointer?): Boolean - fun llmodel_prompt( - model: Pointer?, - prompt: String?, - promptCallback: LLModelResponseCallback, - responseCallback: LLModelResponseCallback, - recalculateCallback: LLModelRecalculateCallback, - context: LLModelContext? - ) - - interface GPTJ : LLModelLibrary { - fun llmodel_gptj_create(): Pointer? - fun llmodel_gptj_destroy(model: Pointer?) - } - - interface Llama : LLModelLibrary { - fun llmodel_llama_create(): Pointer? - fun llmodel_llama_destroy(model: Pointer?) - } - - fun interface LLModelResponseCallback : Callback { - fun callback(tokenId: Int, response: String?): Boolean - } - - fun interface LLModelRecalculateCallback : Callback { - fun callback(isRecalculating: Boolean): Boolean - } -} diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/libraries/LlamaLibrary.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/libraries/LlamaLibrary.kt deleted file mode 100644 index cc3b7250a..000000000 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/libraries/LlamaLibrary.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.xebia.functional.gpt4all.libraries - -import com.sun.jna.Library -import com.sun.jna.Pointer - -interface LlamaLibrary : Library { - fun llama_n_embd(context: Pointer): Int - fun llama_get_embeddings(context: Pointer): Pointer -} diff --git a/gpt4all-kotlin/src/jvmTest/kotlin/com/xebia/functional/xef/tests/GPT4ALLModelSpec.kt b/gpt4all-kotlin/src/jvmTest/kotlin/com/xebia/functional/xef/tests/GPT4ALLModelSpec.kt new file mode 100644 index 000000000..cb80517e6 --- /dev/null +++ b/gpt4all-kotlin/src/jvmTest/kotlin/com/xebia/functional/xef/tests/GPT4ALLModelSpec.kt @@ -0,0 +1,12 @@ +package com.xebia.functional.xef.tests + +import com.xebia.functional.gpt4all.Gpt4AllModel +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.ints.shouldBeGreaterThan + +class GPT4ALLModelSpec : + StringSpec({ + "should return a list of supported models by GPT4ALL" { + Gpt4AllModel.supportedModels().size shouldBeGreaterThan 0 + } + }) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/DeserializerLLMAgent.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/DeserializerLLMAgent.kt index f2c8275f8..156d48cc3 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/DeserializerLLMAgent.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/DeserializerLLMAgent.kt @@ -61,14 +61,16 @@ suspend fun <A> CoreAIScope.prompt( isLenient = true }, promptConfiguration: PromptConfiguration = PromptConfiguration.DEFAULTS, -): A = - model.prompt( +): A { + val functions = generateCFunction(serializer.descriptor) + return model.prompt( prompt, context, - generateCFunction(serializer.descriptor), + functions, { json.decodeFromString(serializer, it) }, promptConfiguration ) +} @OptIn(ExperimentalSerializationApi::class) private fun generateCFunction(descriptor: SerialDescriptor): List<CFunction> { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIClient.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIClient.kt index 7b9022089..9f8f68cb3 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIClient.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIClient.kt @@ -8,6 +8,7 @@ import com.aallam.openai.api.completion.CompletionRequest as OpenAICompletionReq import com.aallam.openai.api.completion.TextCompletion import com.aallam.openai.api.completion.completionRequest import com.aallam.openai.api.core.Usage as OpenAIUsage +import com.aallam.openai.api.embedding.EmbeddingRequest as OpenAIEmbeddingRequest import com.aallam.openai.api.embedding.EmbeddingResponse import com.aallam.openai.api.embedding.embeddingRequest import com.aallam.openai.api.image.ImageCreation @@ -51,14 +52,7 @@ class OpenAIModel( request: ChatCompletionRequest ): ChatCompletionResponse { val response = client.chatCompletion(toChatCompletionRequest(request)) - return ChatCompletionResponse( - id = response.id, - `object` = response.model.id, - created = response.created, - model = response.model.id, - choices = response.choices.map { chatCompletionChoice(it) }, - usage = usage(response.usage) - ) + return chatCompletionResult(response) } @OptIn(BetaOpenAI::class) @@ -66,45 +60,19 @@ class OpenAIModel( request: ChatCompletionRequestWithFunctions ): ChatCompletionResponseWithFunctions { val response = client.chatCompletion(toChatCompletionRequestWithFunctions(request)) - - fun chatCompletionChoiceWithFunctions(choice: ChatChoice): ChoiceWithFunctions = - ChoiceWithFunctions( - message = - choice.message?.let { - MessageWithFunctionCall( - role = it.role.role, - content = it.content, - name = it.name, - functionCall = it.functionCall?.let { FnCall(it.name, it.arguments) } - ) - }, - finishReason = choice.finishReason, - index = choice.index, - ) - - return ChatCompletionResponseWithFunctions( - id = response.id, - `object` = response.model.id, - created = response.created, - model = response.model.id, - choices = response.choices.map { chatCompletionChoiceWithFunctions(it) }, - usage = usage(response.usage) - ) + return chatCompletionResultWithFunctions(response) } override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { - val openAIRequest = embeddingRequest { - model = ModelId(request.model) - input = request.input - user = request.user - } - - return embeddingResult(client.embeddings(openAIRequest)) + val response = client.embeddings(toEmbeddingRequest(request)) + return embeddingResult(response) } @OptIn(BetaOpenAI::class) - override suspend fun createImages(request: ImagesGenerationRequest): ImagesGenerationResponse = - imageResult(client.imageURL(toImageCreationRequest(request))) + override suspend fun createImages(request: ImagesGenerationRequest): ImagesGenerationResponse { + val response = client.imageURL(toImageCreationRequest(request)) + return imageResult(response) + } private fun toCompletionRequest(request: CompletionRequest): OpenAICompletionRequest = completionRequest { @@ -150,6 +118,46 @@ class OpenAIModel( totalTokens = usage?.totalTokens, ) + @OptIn(BetaOpenAI::class) + private fun chatCompletionResult(response: ChatCompletion): ChatCompletionResponse = + ChatCompletionResponse( + id = response.id, + `object` = response.model.id, + created = response.created, + model = response.model.id, + choices = response.choices.map { chatCompletionChoice(it) }, + usage = usage(response.usage) + ) + + @OptIn(BetaOpenAI::class) + private fun chatCompletionResultWithFunctions( + response: ChatCompletion + ): ChatCompletionResponseWithFunctions = + ChatCompletionResponseWithFunctions( + id = response.id, + `object` = response.model.id, + created = response.created, + model = response.model.id, + choices = response.choices.map { chatCompletionChoiceWithFunctions(it) }, + usage = usage(response.usage) + ) + + @OptIn(BetaOpenAI::class) + private fun chatCompletionChoiceWithFunctions(choice: ChatChoice): ChoiceWithFunctions = + ChoiceWithFunctions( + message = + choice.message?.let { + MessageWithFunctionCall( + role = it.role.role, + content = it.content, + name = it.name, + functionCall = it.functionCall?.let { FnCall(it.name, it.arguments) } + ) + }, + finishReason = choice.finishReason, + index = choice.index, + ) + @OptIn(BetaOpenAI::class) private fun chatCompletionChoice(choice: ChatChoice): Choice = Choice( @@ -250,6 +258,13 @@ class OpenAIModel( usage = usage(response.usage) ) + private fun toEmbeddingRequest(request: EmbeddingRequest): OpenAIEmbeddingRequest = + embeddingRequest { + model = ModelId(request.model) + input = request.input + user = request.user + } + @OptIn(BetaOpenAI::class) private fun imageResult(response: List<ImageURL>): ImagesGenerationResponse = ImagesGenerationResponse(data = response.map { ImageGenerationUrl(it.url) }) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIEmbeddings.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIEmbeddings.kt index c7017b1a5..7be6c4355 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIEmbeddings.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/auto/llm/openai/OpenAIEmbeddings.kt @@ -5,9 +5,7 @@ import com.xebia.functional.xef.embeddings.Embedding import com.xebia.functional.xef.embeddings.Embeddings import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest import com.xebia.functional.xef.llm.models.embeddings.RequestConfig -import kotlin.time.ExperimentalTime -@ExperimentalTime class OpenAIEmbeddings(private val oaiClient: com.xebia.functional.xef.llm.Embeddings) : Embeddings {