Move TokenTextSplitter to commonMain (#85)

nomisRev · web-flow · commit 382c27ffadf0 · 2023-05-22T13:02:32.000+02:00
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/loaders/BaseLoader.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/loaders/BaseLoader.kt
@@ -1,9 +1,9 @@
 package com.xebia.functional.xef.loaders
 
-import com.xebia.functional.xef.textsplitters.BaseTextSplitter
+import com.xebia.functional.xef.textsplitters.TextSplitter
 
 interface BaseLoader {
   suspend fun load(): List<String>
-  suspend fun loadAndSplit(textSplitter: BaseTextSplitter): List<String> =
+  suspend fun loadAndSplit(textSplitter: TextSplitter): List<String> =
     textSplitter.splitDocuments(documents = load())
 }
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/CharacterTextSplitter.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/CharacterTextSplitter.kt
@@ -1,7 +1,7 @@
 package com.xebia.functional.xef.textsplitters
 
-fun CharacterTextSplitter(separator: String): BaseTextSplitter =
-  object : BaseTextSplitter {
+fun CharacterTextSplitter(separator: String): TextSplitter =
+  object : TextSplitter {
 
     override suspend fun splitText(text: String): List<String> = text.split(separator)
 
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TextSplitter.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TextSplitter.kt
@@ -1,6 +1,6 @@
 package com.xebia.functional.xef.textsplitters
 
-interface BaseTextSplitter {
+interface TextSplitter {
   suspend fun splitText(text: String): List<String>
   suspend fun splitDocuments(documents: List<String>): List<String>
   suspend fun splitTextInDocuments(text: String): List<String>
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt
@@ -3,14 +3,14 @@ package com.xebia.functional.xef.textsplitters
 import com.xebia.functional.tokenizer.Encoding
 import com.xebia.functional.tokenizer.ModelType
 
-fun TokenTextSplitter(modelType: ModelType, chunkSize: Int, chunkOverlap: Int): BaseTextSplitter =
+fun TokenTextSplitter(modelType: ModelType, chunkSize: Int, chunkOverlap: Int): TextSplitter =
   TokenTextSplitterImpl(modelType.encoding, chunkSize, chunkOverlap)
 
 private class TokenTextSplitterImpl(
   private val tokenizer: Encoding,
   private val chunkSize: Int,
   private val chunkOverlap: Int
-) : BaseTextSplitter {
+) : TextSplitter {
 
   override suspend fun splitText(text: String): List<String> {
     val inputIds = tokenizer.encode(text)
diff --git a/core/src/jvmMain/kotlin/com/xebia/functional/xef/agents/BingSearch.kt b/core/src/jvmMain/kotlin/com/xebia/functional/xef/agents/BingSearch.kt
@@ -4,15 +4,15 @@ import arrow.core.flatten
 import arrow.fx.coroutines.parMap
 import com.apptasticsoftware.rssreader.Item
 import com.apptasticsoftware.rssreader.RssReader
-import com.xebia.functional.xef.textsplitters.BaseTextSplitter
+import com.xebia.functional.xef.textsplitters.TextSplitter
 import io.ktor.http.*
 import java.util.stream.Collectors
 import kotlin.jvm.optionals.toList
 import kotlinx.coroutines.Dispatchers
 
 suspend fun bingSearch(
   search: String,
-  splitter: BaseTextSplitter,
+  splitter: TextSplitter,
   url: String = "https://www.bing.com/news/search?q=${search.encodeURLParameter()}&format=rss",
   maxLinks: Int = 10
 ): List<String> {
diff --git a/core/src/jvmMain/kotlin/com/xebia/functional/xef/agents/ScrapeUrlContent.kt b/core/src/jvmMain/kotlin/com/xebia/functional/xef/agents/ScrapeUrlContent.kt
@@ -1,7 +1,7 @@
 package com.xebia.functional.xef.agents
 
 import com.xebia.functional.xef.loaders.ScrapeURLTextLoader
-import com.xebia.functional.xef.textsplitters.BaseTextSplitter
+import com.xebia.functional.xef.textsplitters.TextSplitter
 
-suspend fun scrapeUrlContent(url: String, splitter: BaseTextSplitter): List<String> =
+suspend fun scrapeUrlContent(url: String, splitter: TextSplitter): List<String> =
   ScrapeURLTextLoader(url).loadAndSplit(splitter)
diff --git a/filesystem/src/commonMain/kotlin/com/xebia/functional/xef/loaders/TextLoader.kt b/filesystem/src/commonMain/kotlin/com/xebia/functional/xef/loaders/TextLoader.kt
@@ -1,7 +1,6 @@
 package com.xebia.functional.xef.loaders
 
 import com.xebia.functional.xef.io.DEFAULT
-import com.xebia.functional.xef.textsplitters.BaseTextSplitter
 import okio.FileSystem
 import okio.Path
 
diff --git a/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt b/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt
@@ -2,7 +2,7 @@ package com.xebia.functional.xef.pdf
 
 import com.xebia.functional.tokenizer.ModelType
 import com.xebia.functional.xef.loaders.BaseLoader
-import com.xebia.functional.xef.textsplitters.BaseTextSplitter
+import com.xebia.functional.xef.textsplitters.TextSplitter
 import com.xebia.functional.xef.textsplitters.TokenTextSplitter
 import io.ktor.client.*
 import io.ktor.client.request.*
@@ -15,7 +15,7 @@ import java.io.File
 
 suspend fun pdf(
   url: String,
-  splitter: BaseTextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
+  splitter: TextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
 ): List<String> =
   HttpClient().use {
     val response = it.get(url)
@@ -29,7 +29,7 @@ suspend fun pdf(
 
 suspend fun pdf(
   file: File,
-  splitter: BaseTextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
+  splitter: TextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
 ): List<String> {
   val loader = PDFLoader(file)
   return loader.loadAndSplit(splitter)
