fix the issue that len(tokenizer(prompt)["input_ids"]) > prompt_len (v…

…llm-project#10524) Signed-off-by: Wang, Yi A <[email protected]>
neuralmagic · Nov 21, 2024 · 8a93a59 · 8a93a59
1 parent 1cfde82
commit 8a93a59
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py
@@ -54,6 +54,7 @@ async def async_request_tgi(
             "do_sample": True,
             "temperature": 0.01,  # TGI does not accept 0.0 temperature.
             "top_p": 0.99,  # TGI does not accept 1.0 top_p.
+            "truncate": request_func_input.prompt_len,
             # TGI does not accept ignore_eos flag.
         }
         payload = {