Update core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitt…

…ers/TokenTextSplitter.kt Co-authored-by: Simon Vergauwen <[email protected]>
xebia-functional · Jun 23, 2023 · 5a71dd0 · 5a71dd0
1 parent 5f83cb8
commit 5a71dd0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt
@@ -19,7 +19,7 @@ private class TokenTextSplitterImpl(
       val end = minOf(startIdx + chunkSize, inputIds.size)
       return tokenizer.decode(inputIds.subList(startIdx, end))
     }
-    val segments = IntProgression.fromClosedRange(0, inputIds.size - 1, chunkSize - chunkOverlap)
+    val segments = inputIds.indices step (chunkSize - chunkOverlap)
     return segments.toList().map { decodeSegment(it) }
   }