fix(llm): withhold hidden stop-sequence prefixes split across tokens

lib/llm/benches/request_trace_finish_metadata.rs

@@ -115,6 +115,8 @@ fn backend_outputs(count: usize) -> Vec<BackendOutput> {
             worker_trace_link: None,
             engine_data: None,
             routing_data: None,
+            encoder_result: None,
+            jailed_text: None,
         })
         .collect()
 }

lib/llm/benches/tokenizer_simple.rs

@@ -62,7 +62,7 @@ pub fn decode(c: &mut Criterion) {
                 let tokenizer: Arc<dyn Tokenizer> =
                     Arc::new(HuggingFaceTokenizer::from_file(TEST_TOKENIZER).unwrap());
                 let ds = DecodeStream::new(tokenizer, &[], false);
-                Decoder::new(ds, StopConditions::default(), false, None)
+                Decoder::new(ds, StopConditions::default(), false, None, None)
             },
             |mut decoder| {
                 for tok in black_box(TEST_TOKS) {
@@ -86,7 +86,7 @@ pub fn decode_big(c: &mut Criterion) {
                 let tokenizer: Arc<dyn Tokenizer> =
                     Arc::new(HuggingFaceTokenizer::from_file(TEST_TOKENIZER).unwrap());
                 let ds = DecodeStream::new(tokenizer, &[], false);
-                Decoder::new(ds, StopConditions::default(), false, None)
+                Decoder::new(ds, StopConditions::default(), false, None, None)
             },
             |mut decoder| {
                 for tok in black_box(&BIG_TEST_TOKS) {
@@ -127,7 +127,7 @@ pub fn tiktoken_decode(c: &mut Criterion) {
                 let tokenizer: Arc<dyn Tokenizer> =
                     Arc::new(TikTokenTokenizer::from_file_auto(TEST_TIKTOKEN).unwrap());
                 let ds = DecodeStream::new(tokenizer, &[], false);
-                Decoder::new(ds, StopConditions::default(), false, None)
+                Decoder::new(ds, StopConditions::default(), false, None, None)
             },
             |mut decoder| {
                 for tok in black_box(&toks) {