Remote PDF Loading (#75)

* Remote PDF Loading

* Remove unused import

* Update integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt

Co-authored-by: Simon Vergauwen <[email protected]>

* Fix merge from main

---------

Co-authored-by: Simon Vergauwen <[email protected]>

Author: raulraja

example/src/main/kotlin/com/xebia/functional/xef/auto/PDFDocument.kt

@@ -2,14 +2,14 @@ package com.xebia.functional.xef.auto
 
 import com.xebia.functional.xef.pdf.pdf
 import kotlinx.serialization.Serializable
-import java.io.File
 
 @Serializable
 data class AIResponse(val answer: String, val source: String)
 
+const val pdfUrl = "https://people.cs.ksu.edu/~schmidt/705a/Scala/Programming-in-Scala.pdf"
+
 suspend fun main() = ai {
-  val file = AIResponse::class.java.getResource("/documents/doc.pdf").file
-  contextScope(pdf(file = File(file))) {
+  contextScope(pdf(url = pdfUrl)) {
     while (true) {
       print("Enter your question: ")
       val line = readlnOrNull() ?: break

example/src/main/resources/documents/doc.pdf

@@ -4,10 +4,29 @@ import com.xebia.functional.tokenizer.ModelType
 import com.xebia.functional.xef.loaders.BaseLoader
 import com.xebia.functional.xef.textsplitters.BaseTextSplitter
 import com.xebia.functional.xef.textsplitters.TokenTextSplitter
+import io.ktor.client.*
+import io.ktor.client.request.*
+import io.ktor.client.statement.*
+import io.ktor.util.cio.*
+import io.ktor.utils.io.*
 import org.apache.pdfbox.pdmodel.PDDocument
 import org.apache.pdfbox.text.PDFTextStripper
 import java.io.File
 
+suspend fun pdf(
+  url: String,
+  splitter: BaseTextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)
+): List<String> =
+  HttpClient().use {
+    val response = it.get(url)
+    val file = File.createTempFile("pdf", ".pdf")
+    file.writeChannel().use {
+      response.bodyAsChannel().copyAndClose(this)
+    }
+    pdf(file, splitter)
+  }
+
+
 suspend fun pdf(
   file: File,
   splitter: BaseTextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50)