diff --git a/build.gradle.kts b/build.gradle.kts index 58e774517..49de77cc2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,6 +47,7 @@ kotlin { commonMain { dependencies { implementation(libs.arrow.fx) + implementation(libs.arrow.resilience) implementation(libs.kotlinx.serialization.json) implementation(libs.bundles.ktor.client) implementation(libs.okio) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 95488e116..60bd1c084 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,6 +11,7 @@ kotest-arrow = "1.3.0" [libraries] arrow-fx = { module = "io.arrow-kt:arrow-fx-coroutines", version.ref = "arrow" } +arrow-resilience = { module = "io.arrow-kt:arrow-resilience", version.ref = "arrow" } open-ai = { module = "com.theokanning.openai-gpt3-java:service", version.ref = "openai" } kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinx-json" } ktor-client = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } diff --git a/src/commonMain/kotlin/com/xebia/functional/env/config.kt b/src/commonMain/kotlin/com/xebia/functional/env/config.kt new file mode 100644 index 000000000..9cd8e6897 --- /dev/null +++ b/src/commonMain/kotlin/com/xebia/functional/env/config.kt @@ -0,0 +1,52 @@ +package com.xebia.functional.env + +import arrow.core.NonEmptyList +import arrow.core.raise.Raise +import arrow.core.raise.catch +import arrow.core.raise.recover +import arrow.core.raise.zipOrAccumulate +import io.ktor.http.Url as KUrl +import arrow.resilience.Schedule +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +data class InvalidConfig(val message: String) + +data class Env(val openAI: OpenAIConfig, val huggingFace: HuggingFaceConfig) + +data class OpenAIConfig(val token: String, val chunkSize: Int, val retryConfig: RetryConfig) + +data class RetryConfig(val backoff: Duration, val maxRetries: Long) { + fun schedule(): Schedule = + Schedule.recurs(maxRetries) + .and(Schedule.exponential(backoff)) + .jittered(0.75, 1.25) + .map { } +} + +data class HuggingFaceConfig(val token: String, val baseUrl: KUrl) + +fun Raise.Env(): Env = + recover({ + zipOrAccumulate( + { OpenAIConfig() }, + { HuggingFaceConfig() } + ) { openAI, huggingFace -> Env(openAI, huggingFace) } + }) { nel -> raise(InvalidConfig(nel.joinToString(separator = "\n"))) } + +fun Raise>.OpenAIConfig(token: String? = null) = + zipOrAccumulate( + { token ?: env("OPENAI_TOKEN") }, + { env("OPENAI_CHUNK_SIZE", default = 1000) { it.toIntOrNull() } }, + { env("OPENAI_BACKOFF", default = 5.seconds) { it.toIntOrNull()?.seconds } }, + { env("OPENAI_MAX_RETRIES", default = 5) { it.toLongOrNull() } }, + ) { token2, chunkSize, backoff, maxRetries -> OpenAIConfig(token2, chunkSize, RetryConfig(backoff, maxRetries)) } + +fun Raise>.HuggingFaceConfig(token: String? = null) = + zipOrAccumulate( + { token ?: env("HF_TOKEN") }, + { env("HF_BASE_URI", default = Url("https://api-inference.huggingface.co")) { Url(it) } } + ) { token2, baseUrl -> HuggingFaceConfig(token2, baseUrl) } + +fun Raise.Url(urlString: String): KUrl = + catch({ KUrl(urlString) }) { raise(it.message ?: "Invalid url: $it") } diff --git a/src/commonMain/kotlin/com/xebia/functional/env/env.kt b/src/commonMain/kotlin/com/xebia/functional/env/env.kt new file mode 100644 index 000000000..4df2dcbb4 --- /dev/null +++ b/src/commonMain/kotlin/com/xebia/functional/env/env.kt @@ -0,0 +1,48 @@ +package com.xebia.functional.env + +import arrow.core.raise.Raise +import arrow.core.raise.ensureNotNull + +/** + * Get an env variable by [name], or fallback to an _optional_ default value. + * In case no env value is found or no default value is provided then it raises an error message. + */ +fun Raise.env( + name: String, + default: String? = null +): String = + ensureNotNull(getenv(name) ?: default) { "\"$name\" configuration missing" } + +/** + * Get an env variable by [name] and [transform] it. + * If no env variable is found, it raises an error message. + * The [transform] function can also raise a custom error message. + */ +fun Raise.env( + name: String, + transform: Raise.(String) -> A? +): A { + val string = getenv(name)?.let { transform(it) } + return ensureNotNull(string) { "\"$name\" configuration found with $string" } +} + +/** + * Get an env variable by [name] and [transform] it, or fallback to a default value. + * The [transform] function can raise a custom error message. + */ +fun Raise.env( + name: String, + default: A, + transform: Raise.(String) -> A? +): A = getenv(name)?.let { transform(it) } ?: default + +/** + * A function that reads the configuration from the environment. + * This only works on JVM, Native and NodeJS. + * + * In the browser, we default to `null` so either rely on the default values, + * or provide construct the values explicitly. + * + * We might be able to support browser through webpack. + */ +expect fun getenv(name: String): String? diff --git a/src/commonMain/kotlin/com/xebia/functional/ktor.kt b/src/commonMain/kotlin/com/xebia/functional/ktor.kt index 72046fb8a..2969db275 100644 --- a/src/commonMain/kotlin/com/xebia/functional/ktor.kt +++ b/src/commonMain/kotlin/com/xebia/functional/ktor.kt @@ -1,8 +1,14 @@ package com.xebia.functional +import arrow.core.Either +import arrow.core.nonFatalOrThrow import arrow.fx.coroutines.ResourceScope +import arrow.resilience.Schedule +import arrow.resilience.ScheduleStep +import com.xebia.functional.env.RetryConfig import io.ktor.client.HttpClient import io.ktor.client.engine.HttpClientEngine +import io.ktor.client.plugins.HttpRequestRetry import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.client.request.HttpRequestBuilder import io.ktor.client.request.header @@ -10,6 +16,7 @@ import io.ktor.client.request.setBody import io.ktor.http.ContentType import io.ktor.http.contentType import io.ktor.serialization.kotlinx.json.json +import kotlin.time.Duration inline fun HttpRequestBuilder.configure(token: String, request: A): Unit { header("Authorization", "Bearer $token") @@ -22,4 +29,4 @@ suspend fun ResourceScope.httpClient(engine: HttpClientEngine): HttpClient = HttpClient(engine) { install(ContentNegotiation) { json() } } - }) { client, _ -> client.close() } + }) { client, _ -> client.close() } \ No newline at end of file diff --git a/src/commonMain/kotlin/com/xebia/functional/llm/huggingface/HuggingFaceClient.kt b/src/commonMain/kotlin/com/xebia/functional/llm/huggingface/HuggingFaceClient.kt index a9aab59bb..d7b203f08 100644 --- a/src/commonMain/kotlin/com/xebia/functional/llm/huggingface/HuggingFaceClient.kt +++ b/src/commonMain/kotlin/com/xebia/functional/llm/huggingface/HuggingFaceClient.kt @@ -2,6 +2,7 @@ package com.xebia.functional.llm.huggingface import arrow.fx.coroutines.ResourceScope import com.xebia.functional.configure +import com.xebia.functional.env.HuggingFaceConfig import com.xebia.functional.httpClient import io.ktor.client.HttpClient import io.ktor.client.call.body @@ -11,8 +12,10 @@ import io.ktor.client.request.HttpRequestBuilder import io.ktor.client.request.header import io.ktor.client.request.post import io.ktor.client.request.setBody +import io.ktor.client.request.url import io.ktor.http.ContentType import io.ktor.http.contentType +import io.ktor.http.path import io.ktor.serialization.kotlinx.json.json interface HuggingFaceClient { @@ -21,20 +24,18 @@ interface HuggingFaceClient { suspend fun ResourceScope.KtorHuggingFaceClient( engine: HttpClientEngine, - token: String -): HuggingFaceClient = KtorHuggingFaceClient(httpClient(engine), token) + config: HuggingFaceConfig +): HuggingFaceClient = KtorHuggingFaceClient(httpClient(engine), config) private class KtorHuggingFaceClient( private val httpClient: HttpClient, - private val token: String + private val config: HuggingFaceConfig ) : HuggingFaceClient { - // TODO move to config - private val baseUrl = "https://api-inference.huggingface.co" - override suspend fun generate(request: InferenceRequest, model: Model): List { - val response = httpClient.post("$baseUrl/models/${model.name}") { - configure(token, request) + val response = httpClient.post(config.baseUrl) { + url { path("models", model.name) } + configure(config.token, request) } return response.body() } diff --git a/src/commonMain/kotlin/com/xebia/functional/llm/openai/OpenAIClient.kt b/src/commonMain/kotlin/com/xebia/functional/llm/openai/OpenAIClient.kt index 5e76fd17e..9b57f3478 100644 --- a/src/commonMain/kotlin/com/xebia/functional/llm/openai/OpenAIClient.kt +++ b/src/commonMain/kotlin/com/xebia/functional/llm/openai/OpenAIClient.kt @@ -1,12 +1,10 @@ -package llm.openai +package com.xebia.functional.llm.openai import arrow.fx.coroutines.ResourceScope +import arrow.resilience.retry import com.xebia.functional.configure +import com.xebia.functional.env.OpenAIConfig import com.xebia.functional.httpClient -import com.xebia.functional.llm.openai.CompletionChoice -import com.xebia.functional.llm.openai.CompletionRequest -import com.xebia.functional.llm.openai.EmbeddingRequest -import com.xebia.functional.llm.openai.EmbeddingResult import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.engine.HttpClientEngine @@ -19,23 +17,27 @@ interface OpenAIClient { suspend fun ResourceScope.KtorOpenAIClient( engine: HttpClientEngine, - token: String -): OpenAIClient = KtorOpenAIClient(httpClient(engine), token) + config: OpenAIConfig +): OpenAIClient = KtorOpenAIClient(httpClient(engine), config) private class KtorOpenAIClient( private val httpClient: HttpClient, - private val token: String + private val config: OpenAIConfig ) : OpenAIClient { private val baseUrl = "https://api.openai.com/v1" override suspend fun createCompletion(request: CompletionRequest): List { - val response = httpClient.post("$baseUrl/completions") { configure(token, request) } + val response = config.retryConfig.schedule().retry { + httpClient.post("$baseUrl/completions") { configure(config.token, request) } + } return response.body() } override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { - val response = httpClient.post("$baseUrl/embeddings") { configure(token, request) } + val response = config.retryConfig.schedule().retry { + httpClient.post("$baseUrl/embeddings") { configure(config.token, request) } + } return response.body() } } diff --git a/src/jsMain/kotlin/com/xebia/functional/env/getenv.kt b/src/jsMain/kotlin/com/xebia/functional/env/getenv.kt new file mode 100644 index 000000000..1dea60583 --- /dev/null +++ b/src/jsMain/kotlin/com/xebia/functional/env/getenv.kt @@ -0,0 +1,11 @@ +package com.xebia.functional.env + +external val process: dynamic + +/** + * We wrap it in runCatching because this only works in NodeJS. + * In the browser, we get a ReferenceError: "process" is not defined. + */ +actual fun getenv(name: String): String? = runCatching { + process.env[name] as String? +}.getOrNull() diff --git a/src/jvmMain/kotlin/com/xebia/functional/env/getenv.kt b/src/jvmMain/kotlin/com/xebia/functional/env/getenv.kt new file mode 100644 index 000000000..a017ea317 --- /dev/null +++ b/src/jvmMain/kotlin/com/xebia/functional/env/getenv.kt @@ -0,0 +1,4 @@ +package com.xebia.functional.env + +actual fun getenv(name: String): String? = + System.getenv(name) diff --git a/src/nativeMain/kotlin/com/xebia/functional/env/getenv.kt b/src/nativeMain/kotlin/com/xebia/functional/env/getenv.kt new file mode 100644 index 000000000..f8719427f --- /dev/null +++ b/src/nativeMain/kotlin/com/xebia/functional/env/getenv.kt @@ -0,0 +1,6 @@ +package com.xebia.functional.env + +import kotlinx.cinterop.toKString + +actual fun getenv(name: String): String? = + platform.posix.getenv(name)?.toKString()