Skip to content

Commit 83434d4

Browse files
committed
PI-2526 Update chunk size to match the value used by data science evals
1 parent f138a2e commit 83434d4

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

projects/person-search-index-from-delius/container/pipelines/contact/index/ingest-pipeline.tpl.json

+8-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
"text_chunking": {
2626
"algorithm": {
2727
"fixed_token_length": {
28-
"token_limit": 10,
29-
"overlap_rate": 0.1,
28+
"token_limit": 64,
29+
"overlap_rate": 0.125,
3030
"tokenizer": "standard"
3131
}
3232
},
@@ -35,6 +35,12 @@
3535
}
3636
}
3737
},
38+
{
39+
"foreach": {
40+
"field": "textChunks",
41+
"processor"
42+
}
43+
},
3844
{
3945
"text_embedding": {
4046
"model_id": "${model_id}",

0 commit comments

Comments
 (0)