Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move TokenTextSplitter to commonMain #85

Merged
merged 1 commit into from
May 22, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package com.xebia.functional.xef.loaders

import com.xebia.functional.xef.textsplitters.BaseTextSplitter
import com.xebia.functional.xef.textsplitters.TextSplitter

interface BaseLoader {
suspend fun load(): List<String>
suspend fun loadAndSplit(textSplitter: BaseTextSplitter): List<String> =
suspend fun loadAndSplit(textSplitter: TextSplitter): List<String> =
textSplitter.splitDocuments(documents = load())
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.xebia.functional.xef.textsplitters

fun CharacterTextSplitter(separator: String): BaseTextSplitter =
object : BaseTextSplitter {
fun CharacterTextSplitter(separator: String): TextSplitter =
object : TextSplitter {

override suspend fun splitText(text: String): List<String> = text.split(separator)

Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.xebia.functional.xef.textsplitters

interface BaseTextSplitter {
interface TextSplitter {
suspend fun splitText(text: String): List<String>
suspend fun splitDocuments(documents: List<String>): List<String>
suspend fun splitTextInDocuments(text: String): List<String>
Original file line number Diff line number Diff line change
@@ -3,14 +3,14 @@ package com.xebia.functional.xef.textsplitters
import com.xebia.functional.tokenizer.Encoding
import com.xebia.functional.tokenizer.ModelType

fun TokenTextSplitter(modelType: ModelType, chunkSize: Int, chunkOverlap: Int): BaseTextSplitter =
fun TokenTextSplitter(modelType: ModelType, chunkSize: Int, chunkOverlap: Int): TextSplitter =
TokenTextSplitterImpl(modelType.encoding, chunkSize, chunkOverlap)

private class TokenTextSplitterImpl(
private val tokenizer: Encoding,
private val chunkSize: Int,
private val chunkOverlap: Int
) : BaseTextSplitter {
) : TextSplitter {

override suspend fun splitText(text: String): List<String> {
val inputIds = tokenizer.encode(text)
Original file line number Diff line number Diff line change
@@ -4,15 +4,15 @@ import arrow.core.flatten
import arrow.fx.coroutines.parMap
import com.apptasticsoftware.rssreader.Item
import com.apptasticsoftware.rssreader.RssReader
import com.xebia.functional.xef.textsplitters.BaseTextSplitter
import com.xebia.functional.xef.textsplitters.TextSplitter
import io.ktor.http.*
import java.util.stream.Collectors
import kotlin.jvm.optionals.toList
import kotlinx.coroutines.Dispatchers

suspend fun bingSearch(
search: String,
splitter: BaseTextSplitter,
splitter: TextSplitter,
url: String = "https://www.bing.com/news/search?q=${search.encodeURLParameter()}&format=rss",
maxLinks: Int = 10
): List<String> {
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.xebia.functional.xef.agents

import com.xebia.functional.xef.loaders.ScrapeURLTextLoader
import com.xebia.functional.xef.textsplitters.BaseTextSplitter
import com.xebia.functional.xef.textsplitters.TextSplitter

suspend fun scrapeUrlContent(url: String, splitter: BaseTextSplitter): List<String> =
suspend fun scrapeUrlContent(url: String, splitter: TextSplitter): List<String> =
ScrapeURLTextLoader(url).loadAndSplit(splitter)
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.xebia.functional.xef.loaders

import com.xebia.functional.xef.io.DEFAULT
import com.xebia.functional.xef.textsplitters.BaseTextSplitter
import okio.FileSystem
import okio.Path

Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ package com.xebia.functional.xef.pdf

import com.xebia.functional.tokenizer.ModelType
import com.xebia.functional.xef.loaders.BaseLoader
import com.xebia.functional.xef.textsplitters.BaseTextSplitter
import com.xebia.functional.xef.textsplitters.TextSplitter
import com.xebia.functional.xef.textsplitters.TokenTextSplitter
import io.ktor.client.*
import io.ktor.client.request.*
@@ -15,7 +15,7 @@ import java.io.File

suspend fun pdf(
url: String,
splitter: BaseTextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
splitter: TextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
): List<String> =
HttpClient().use {
val response = it.get(url)
@@ -29,7 +29,7 @@ suspend fun pdf(

suspend fun pdf(
file: File,
splitter: BaseTextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
splitter: TextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
): List<String> {
val loader = PDFLoader(file)
return loader.loadAndSplit(splitter)