diff --git a/crates/mesh-native-serving-plugin-host/src/lib.rs b/crates/mesh-native-serving-plugin-host/src/lib.rs index 70864694a2..6d3a56dad2 100644 --- a/crates/mesh-native-serving-plugin-host/src/lib.rs +++ b/crates/mesh-native-serving-plugin-host/src/lib.rs @@ -5,6 +5,7 @@ mod plugin_dispatch; mod test_support; use std::{ + collections::HashMap, ffi::{c_char, c_void}, path::{Path, PathBuf}, ptr::NonNull, @@ -125,6 +126,7 @@ impl ModelServingHooksFactory for NativeServingPluginFactory { definition: Arc::clone(&self.definition), instance: Some(instance), proposal_token_buffer: Mutex::new(vec![0; MAX_NATIVE_PLUGIN_PROPOSAL_TOKENS]), + committed_generated_tokens: Mutex::new(HashMap::new()), }; let driver = Arc::new(PluginDriver::spawn(active)?); let lifecycle: Arc = Arc::new(NativeLifecycleIngress { @@ -270,6 +272,7 @@ struct ActivePlugin { definition: Arc, instance: Option>, proposal_token_buffer: Mutex>, + committed_generated_tokens: Mutex>, } // SAFETY: activation succeeds only for plugins implementing the ABI's @@ -306,19 +309,81 @@ impl ActivePlugin { prompt_token_ids: abi::TokenSlice::from_tokens(&start.prompt_token_ids), }; let status = unsafe { (self.definition.api().begin_generation)(self.instance()?, &event) }; - self.call_status("begin generation", status) + self.call_status("begin generation", status)?; + self.committed_generated_tokens + .lock() + .map_err(|_| anyhow!("native serving plugin commit state lock poisoned"))? + .insert((start.request_id, start.session_id), 0); + Ok(()) } fn committed(&self, commit: &GenerationCommit) -> Result<()> { + self.commit_tokens( + commit.request_id, + commit.session_id, + commit.generated_token_count, + &commit.token_ids, + ) + } + + fn commit_tokens( + &self, + request_id: u64, + session_id: u64, + generated_token_count: usize, + token_ids: &[i32], + ) -> Result<()> { + if token_ids.is_empty() { + return Ok(()); + } + let mut committed_generated_tokens = self + .committed_generated_tokens + .lock() + .map_err(|_| anyhow!("native serving plugin commit state lock poisoned"))?; + self.commit_tokens_locked( + &mut committed_generated_tokens, + request_id, + session_id, + generated_token_count, + token_ids, + ) + } + + fn commit_tokens_locked( + &self, + committed_generated_tokens: &mut HashMap<(u64, u64), usize>, + request_id: u64, + session_id: u64, + generated_token_count: usize, + token_ids: &[i32], + ) -> Result<()> { + if token_ids.is_empty() { + return Ok(()); + } + let key = (request_id, session_id); + let prior_generated_token_count = committed_generated_tokens + .get(&key) + .copied() + .unwrap_or_else(|| generated_token_count.saturating_sub(token_ids.len())); + let expected_generated_token_count = prior_generated_token_count + .checked_add(token_ids.len()) + .context("native serving plugin generated-token count overflow")?; + if generated_token_count != expected_generated_token_count { + bail!( + "native serving plugin commit does not extend the tracked generated-token prefix" + ); + } let event = abi::GenerationCommit { struct_size: size_of::(), - request_id: commit.request_id, - session_id: commit.session_id, - generated_token_count: u64::try_from(commit.generated_token_count)?, - token_ids: abi::TokenSlice::from_tokens(&commit.token_ids), + request_id, + session_id, + generated_token_count: u64::try_from(generated_token_count)?, + token_ids: abi::TokenSlice::from_tokens(token_ids), }; let status = unsafe { (self.definition.api().commit_generation)(self.instance()?, &event) }; - self.call_status("commit generation", status) + self.call_status("commit generation", status)?; + committed_generated_tokens.insert(key, generated_token_count); + Ok(()) } fn abort(&self, abort: &GenerationAbort) -> Result<()> { @@ -328,10 +393,21 @@ impl ActivePlugin { session_id: abort.session_id, }; let status = unsafe { (self.definition.api().abort_generation)(self.instance()?, &event) }; - self.call_status("abort generation", status) + let result = self.call_status("abort generation", status); + self.committed_generated_tokens + .lock() + .map_err(|_| anyhow!("native serving plugin commit state lock poisoned"))? + .remove(&(abort.request_id, abort.session_id)); + result } fn finish(&self, receipt: &GenerationReceipt) -> Result<()> { + let key = (receipt.request_id, receipt.session_id); + self.commit_final_suffix( + receipt.request_id, + receipt.session_id, + &receipt.generated_token_ids, + )?; let request_to_first_token_us = receipt.request_to_first_token_us.unwrap_or_default(); let event = abi::GenerationFinish { struct_size: size_of::(), @@ -351,10 +427,56 @@ impl ActivePlugin { ), }; let status = unsafe { (self.definition.api().finish_generation)(self.instance()?, &event) }; - self.call_status("finish generation", status) + let result = self.call_status("finish generation", status); + self.committed_generated_tokens + .lock() + .map_err(|_| anyhow!("native serving plugin commit state lock poisoned"))? + .remove(&key); + result + } + + fn commit_final_suffix( + &self, + request_id: u64, + session_id: u64, + generated_token_ids: &[i32], + ) -> Result<()> { + let mut committed_generated_tokens = self + .committed_generated_tokens + .lock() + .map_err(|_| anyhow!("native serving plugin commit state lock poisoned"))?; + let key = (request_id, session_id); + let committed = committed_generated_tokens + .get(&key) + .copied() + .unwrap_or_default(); + if committed > generated_token_ids.len() { + bail!("native serving plugin committed beyond the final generation receipt"); + } + if committed < generated_token_ids.len() { + self.commit_tokens_locked( + &mut committed_generated_tokens, + request_id, + session_id, + generated_token_ids.len(), + &generated_token_ids[committed..], + )?; + } + Ok(()) } fn propose(&self, query: LinearProposalQuery) -> Result> { + if !query.pending_token_ids.is_empty() { + let generated_token_count = query + .committed_token_count + .saturating_sub(query.prompt_token_count); + self.commit_tokens( + query.request_id, + query.session_id, + generated_token_count, + &query.pending_token_ids, + )?; + } let event = abi::ProposalQuery { struct_size: size_of::(), request_id: query.request_id, @@ -446,6 +568,11 @@ impl ActivePlugin { } fn report(&self, receipt: &LinearProposalReceipt) -> Result<()> { + self.commit_proposal_tokens( + receipt.request_id, + receipt.session_id, + &receipt.committed_tokens, + )?; let event = abi::ProposalOutcome { struct_size: size_of::(), decision_id: abi::ByteSlice::from_bytes(receipt.decision_id.as_bytes()), @@ -469,6 +596,32 @@ impl ActivePlugin { self.call_status("report proposal", status) } + fn commit_proposal_tokens( + &self, + request_id: u64, + session_id: u64, + committed_tokens: &[i32], + ) -> Result<()> { + let mut committed_generated_tokens = self + .committed_generated_tokens + .lock() + .map_err(|_| anyhow!("native serving plugin commit state lock poisoned"))?; + let key = (request_id, session_id); + let generated_token_count = committed_generated_tokens + .get(&key) + .copied() + .unwrap_or_default() + .checked_add(committed_tokens.len()) + .context("native serving plugin generated-token count overflow")?; + self.commit_tokens_locked( + &mut committed_generated_tokens, + request_id, + session_id, + generated_token_count, + committed_tokens, + ) + } + fn discard(&self, decision_id: &[u8], reason: LinearProposalDiscardReason) -> Result<()> { let event = abi::ProposalDiscard { struct_size: size_of::(), diff --git a/crates/mesh-native-serving-plugin-host/src/plugin_dispatch/tests.rs b/crates/mesh-native-serving-plugin-host/src/plugin_dispatch/tests.rs index a5c3de06e7..221b17d9f9 100644 --- a/crates/mesh-native-serving-plugin-host/src/plugin_dispatch/tests.rs +++ b/crates/mesh-native-serving-plugin-host/src/plugin_dispatch/tests.rs @@ -243,6 +243,73 @@ fn lifecycle_ingress_shares_plugin_queue_order_with_proposals() { assert_eq!(*events.lock().unwrap(), ["begin", "commit", "proposal"]); } +#[test] +fn proposal_applies_pending_tokens_before_lookup() { + let (active, events) = fake_active_with_events(Duration::ZERO); + active + .begin(&GenerationStart { + request_id: 1, + session_id: 2, + agent_session_id: None, + prompt_token_ids: Arc::from([3]), + }) + .unwrap(); + let result = active + .propose( + LinearProposalQuery::new( + 1, + 2, + 1, + 3, + 2, + 8, + Instant::now() + Duration::from_millis(100), + ) + .with_pending_token_ids(vec![4, 5].into_boxed_slice()), + ) + .unwrap(); + + assert!(result.is_none()); + assert_eq!(*events.lock().unwrap(), ["begin", "commit", "proposal"]); +} + +#[test] +fn blocking_commit_cannot_extend_the_proposal_deadline() { + let (active, _, _) = fake_active_with_timing( + Duration::ZERO, + Duration::from_millis(250), + Duration::ZERO, + false, + ); + let driver = Arc::new(PluginDriver::spawn(active).unwrap()); + let ingress = NativeLifecycleIngress { + driver: Arc::clone(&driver), + }; + ingress + .try_submit(GenerationLifecycleObservation::Started(GenerationStart { + request_id: 1, + session_id: 2, + agent_session_id: None, + prompt_token_ids: Arc::from([3]), + })) + .unwrap(); + + let started = Instant::now(); + let result = driver + .propose( + LinearProposalQuery::new(1, 2, 1, 2, 1, 8, started + Duration::from_millis(5)) + .with_pending_token_ids(vec![4].into_boxed_slice()), + ) + .unwrap(); + + assert!(result.proposal.unwrap().is_none()); + assert!( + started.elapsed() < Duration::from_millis(150), + "proposal waited {:?} for a blocking commit", + started.elapsed() + ); +} + #[test] fn lifecycle_callback_failure_is_observed_without_poisoning_the_driver() { let (active, _, _) = fake_active_with_options(Duration::ZERO, true); @@ -333,12 +400,12 @@ fn dropping_the_driver_drains_its_backlog_and_shuts_the_plugin_down() { ); let observations = fake_observations(&active); let driver = PluginDriver::spawn(active).unwrap(); - for _ in 0..20 { + for generated_token_count in 1..=20 { driver .enqueue(PluginCommand::Committed(GenerationCommit { request_id: 1, session_id: 2, - generated_token_count: 1, + generated_token_count, token_ids: vec![4].into_boxed_slice(), })) .unwrap(); diff --git a/crates/mesh-native-serving-plugin-host/src/test_support.rs b/crates/mesh-native-serving-plugin-host/src/test_support.rs index 791a4831ee..a420dd69e6 100644 --- a/crates/mesh-native-serving-plugin-host/src/test_support.rs +++ b/crates/mesh-native-serving-plugin-host/src/test_support.rs @@ -1,6 +1,7 @@ //! Shared fake native plugin used by host and dispatch unit tests. use std::{ + collections::HashMap, ffi::{c_char, c_void}, ptr::NonNull, sync::{ @@ -257,6 +258,7 @@ pub(crate) fn fake_active_with_timing( definition, instance: NonNull::new(Box::into_raw(state).cast::()), proposal_token_buffer: Mutex::new(vec![0; MAX_NATIVE_PLUGIN_PROPOSAL_TOKENS]), + committed_generated_tokens: Mutex::new(HashMap::new()), }, events, abort_count, diff --git a/crates/skippy-server/src/frontend/generation_receipt.rs b/crates/skippy-server/src/frontend/generation_receipt.rs index a2876ea1a3..fc81df0c5c 100644 --- a/crates/skippy-server/src/frontend/generation_receipt.rs +++ b/crates/skippy-server/src/frontend/generation_receipt.rs @@ -115,7 +115,8 @@ pub struct GenerationCommit { pub trait GenerationReceiptSink: Send + Sync { fn begin(&self, start: &GenerationStart) -> Result<()>; - /// Observes canonical target tokens submitted before the next proposal lookup. + /// Optionally observes canonical target-token deltas for integrations that + /// need lifecycle notifications outside the local proposal hot path. fn committed(&self, commit: &GenerationCommit) -> Result<()>; fn abort(&self, abort: &GenerationAbort) -> Result<()>; diff --git a/crates/skippy-server/src/frontend/linear_proposal.rs b/crates/skippy-server/src/frontend/linear_proposal.rs index b202431f1d..76df69e88c 100644 --- a/crates/skippy-server/src/frontend/linear_proposal.rs +++ b/crates/skippy-server/src/frontend/linear_proposal.rs @@ -69,12 +69,11 @@ impl LinearProposal { /// Bounded, target-authoritative state supplied to a proposal source. /// -/// The proposal path deliberately does not pass a full context buffer. A -/// lifecycle observer receives canonical committed token deltas separately; -/// the request path carries only O(1) identity and position data so a source -/// can honor its absolute deadline without copying or hashing an arbitrarily -/// large prompt. -#[derive(Clone, Copy, Debug)] +/// The proposal path deliberately does not pass a full context buffer. The +/// bounded canonical token delta since the previous proposal boundary travels +/// with this query, so a source can update its state synchronously without a +/// separate lifecycle event or copying or hashing an arbitrarily large prompt. +#[derive(Clone, Debug)] #[non_exhaustive] pub struct LinearProposalQuery { /// OpenAI request identity. @@ -88,13 +87,19 @@ pub struct LinearProposalQuery { /// that may already have been committed during prefill. pub prompt_token_count: usize, /// Total canonical tokens committed at this query boundary, including the - /// prompt and every target token previously delivered to the lifecycle - /// observer. + /// prompt and every target token previously delivered to the proposal + /// source. pub committed_token_count: usize, /// Number of target tokens already generated. pub decode_step: usize, /// Maximum proposal width Skippy will accept for this query. pub max_proposal_tokens: usize, + /// Canonical target tokens emitted since the previous proposal boundary. + /// + /// Proposal sources apply this delta before producing the proposal. Keeping + /// the delta on the synchronous query avoids a separate lifecycle event on + /// the local-generation hot path. + pub pending_token_ids: Box<[i32]>, /// Advisory deadline the synchronous source must honor. pub deadline: Instant, } @@ -118,9 +123,16 @@ impl LinearProposalQuery { committed_token_count, decode_step, max_proposal_tokens, + pending_token_ids: Vec::new().into_boxed_slice(), deadline, } } + + #[must_use] + pub fn with_pending_token_ids(mut self, pending_token_ids: Box<[i32]>) -> Self { + self.pending_token_ids = pending_token_ids; + self + } } /// Why Skippy rejected a source decision without producing a receipt. @@ -159,6 +171,10 @@ pub enum LinearProposalDisposition { #[derive(Clone, Debug, PartialEq)] #[non_exhaustive] pub struct LinearProposalReceipt { + /// OpenAI request identity for the proposal's generation. + pub request_id: u64, + /// OpenAI session identity for the proposal's generation. + pub session_id: u64, /// Source identity copied from the proposal. pub decision_id: OpaqueProposalDecisionId, /// Target-authoritative verification outcome. @@ -464,6 +480,9 @@ pub(crate) struct QueriedLinearProposal { } pub(crate) enum LinearProposalQueryOutcome { + /// No proposal command was submitted, so any pending token delta remains + /// owned by the caller. + Skipped, NoProposal { source_telemetry: Option, }, @@ -474,7 +493,7 @@ pub(crate) enum LinearProposalQueryOutcome { Ready(QueriedLinearProposal), } -#[derive(Clone, Copy)] +#[derive(Clone)] pub(crate) struct LinearProposalQueryParams { pub(crate) request_id: u64, pub(crate) session_id: u64, @@ -483,6 +502,7 @@ pub(crate) struct LinearProposalQueryParams { pub(crate) committed_token_count: usize, pub(crate) remaining_new_tokens: usize, pub(crate) runtime_max_proposal_tokens: usize, + pub(crate) pending_token_ids: Box<[i32]>, } pub(crate) fn query_linear_proposal( @@ -503,9 +523,7 @@ pub(crate) fn query_linear_proposal( .min(params.runtime_max_proposal_tokens) .min(config.max_proposal_tokens()); if max_proposal_tokens == 0 { - return Ok(LinearProposalQueryOutcome::NoProposal { - source_telemetry: None, - }); + return Ok(LinearProposalQueryOutcome::Skipped); } let operation_started = Instant::now(); let deadline = operation_started @@ -514,15 +532,18 @@ pub(crate) fn query_linear_proposal( let proposal_started = Instant::now(); let response = config .source() - .propose(LinearProposalQuery::new( - params.request_id, - params.session_id, - params.prompt_token_count, - params.committed_token_count, - params.decode_step, - max_proposal_tokens, - deadline, - )) + .propose( + LinearProposalQuery::new( + params.request_id, + params.session_id, + params.prompt_token_count, + params.committed_token_count, + params.decode_step, + max_proposal_tokens, + deadline, + ) + .with_pending_token_ids(params.pending_token_ids), + ) .map_err(openai_backend_error)?; let proposal_elapsed_us = elapsed_us(proposal_started); let (proposal, source_telemetry) = response.into_parts(); @@ -652,6 +673,7 @@ mod tests { committed_token_count, remaining_new_tokens, runtime_max_proposal_tokens, + pending_token_ids: Vec::new().into_boxed_slice(), } } @@ -662,12 +684,17 @@ mod tests { discard_fails: Mutex, report_fails: Mutex, queries: Mutex>, + pending_token_batches: Mutex>>, receipts: Mutex>, discards: Mutex>, } impl LinearProposalIngress for FakeIngress { fn propose(&self, query: LinearProposalQuery) -> Result { + self.pending_token_batches + .lock() + .unwrap() + .push(query.pending_token_ids.clone()); self.queries.lock().unwrap().push(RecordedQuery { request_id: query.request_id, session_id: query.session_id, @@ -793,6 +820,8 @@ mod tests { let config = LinearProposalIngressConfig::new(source.clone(), Duration::from_secs(1), 4).unwrap(); let receipt = LinearProposalReceipt { + request_id: 1, + session_id: 2, decision_id: OpaqueProposalDecisionId::new(vec![90]).unwrap(), disposition: LinearProposalDisposition::FullAccept, proposal_token_count: 1, @@ -895,6 +924,39 @@ mod tests { assert!(source.discards.lock().unwrap().is_empty()); } + #[test] + fn query_forwards_pending_tokens_to_the_proposal_source() { + let source = Arc::new(FakeIngress::default()); + let config = + LinearProposalIngressConfig::new(source.clone(), Duration::from_secs(1), 4).unwrap(); + let mut params = query_params(7, 8, 2, 1, 3, 5, 4); + params.pending_token_ids = vec![31, 32].into_boxed_slice(); + + assert!(matches!( + query_linear_proposal(&config, params).unwrap(), + LinearProposalQueryOutcome::NoProposal { .. } + )); + assert_eq!( + source.pending_token_batches.lock().unwrap().as_slice(), + &[vec![31, 32].into_boxed_slice()] + ); + } + + #[test] + fn query_does_not_accept_pending_tokens_without_proposal_capacity() { + let source = Arc::new(FakeIngress::default()); + let config = + LinearProposalIngressConfig::new(source.clone(), Duration::from_secs(1), 4).unwrap(); + let mut params = query_params(7, 8, 2, 1, 3, 1, 4); + params.pending_token_ids = vec![31].into_boxed_slice(); + + assert!(matches!( + query_linear_proposal(&config, params).unwrap(), + LinearProposalQueryOutcome::Skipped + )); + assert!(source.pending_token_batches.lock().unwrap().is_empty()); + } + #[test] fn query_rejects_an_inconsistent_prompt_decode_boundary_before_ingress() { let source = Arc::new(FakeIngress::default()); @@ -981,6 +1043,8 @@ mod tests { let source = FakeIngress::default(); let id = OpaqueProposalDecisionId::new(vec![7, 8, 9]).unwrap(); let receipt = LinearProposalReceipt { + request_id: 1, + session_id: 2, decision_id: id.clone(), disposition: LinearProposalDisposition::FirstMismatch, proposal_token_count: 4, @@ -1031,6 +1095,8 @@ mod tests { fn receipt_telemetry_excludes_source_ids_tokens_and_error_text() { let secret = "private-decision-/Users/nick/prompt.txt"; let receipt = LinearProposalReceipt { + request_id: 1, + session_id: 2, decision_id: OpaqueProposalDecisionId::new(secret.as_bytes().to_vec()).unwrap(), disposition: LinearProposalDisposition::FullAccept, proposal_token_count: 1, diff --git a/crates/skippy-server/src/frontend/linear_proposal/execution.rs b/crates/skippy-server/src/frontend/linear_proposal/execution.rs index fccc66bd00..64bc28ae6d 100644 --- a/crates/skippy-server/src/frontend/linear_proposal/execution.rs +++ b/crates/skippy-server/src/frontend/linear_proposal/execution.rs @@ -26,6 +26,8 @@ struct LinearProposalExecution { #[derive(Clone, Copy)] pub(crate) struct LinearProposalExecutionParams<'a> { + pub(crate) request_id: u64, + pub(crate) request_session_id: u64, pub(crate) session_id: &'a str, pub(crate) current: i32, pub(crate) base_position: u64, @@ -104,6 +106,8 @@ impl StageOpenAiBackend { }); let total_elapsed_us = elapsed_us(queried.operation_started); Ok(Some(LinearProposalReceipt { + request_id: params.request_id, + session_id: params.request_session_id, decision_id: queried.proposal.decision_id, disposition, proposal_token_count, diff --git a/crates/skippy-server/src/frontend/local_generation/linear_decode.rs b/crates/skippy-server/src/frontend/local_generation/linear_decode.rs index b1139b119f..9d14bdd110 100644 --- a/crates/skippy-server/src/frontend/local_generation/linear_decode.rs +++ b/crates/skippy-server/src/frontend/local_generation/linear_decode.rs @@ -63,42 +63,67 @@ impl StageOpenAiBackend { .saturating_add(state.decoded_tokens), remaining_new_tokens, runtime_max_proposal_tokens: state.linear_proposal_max_tokens, + pending_token_ids: state + .pending_linear_proposal_tokens + .clone() + .into_boxed_slice(), }, - )? { - LinearProposalQueryOutcome::NoProposal { source_telemetry } => { - self.emit_linear_proposal_source_telemetry( - source_telemetry, - state.emit_token_debug, - ); - None + ) { + Ok(LinearProposalQueryOutcome::Skipped) => { + return Ok(LinearProposalProgress::NotUsed); } - LinearProposalQueryOutcome::DeadlineExceeded { - proposal_elapsed_us, - source_telemetry, - } => { - self.emit_linear_proposal_source_telemetry( - source_telemetry, - state.emit_token_debug, - ); - let mut attrs = BTreeMap::new(); - attrs.insert( - "llama_stage.linear_proposal.discard_reason".to_string(), - json!("deadline_exceeded"), - ); - attrs.insert( - "llama_stage.linear_proposal.proposal_us".to_string(), - json!(proposal_elapsed_us), - ); - self.telemetry - .emit("stage.openai_linear_proposal_late", attrs); - None + Ok(outcome) => { + state.pending_linear_proposal_tokens.clear(); + match outcome { + LinearProposalQueryOutcome::Skipped => unreachable!("handled above"), + LinearProposalQueryOutcome::NoProposal { source_telemetry } => { + self.emit_linear_proposal_source_telemetry( + source_telemetry, + state.emit_token_debug, + ); + None + } + LinearProposalQueryOutcome::DeadlineExceeded { + proposal_elapsed_us, + source_telemetry, + } => { + self.emit_linear_proposal_source_telemetry( + source_telemetry, + state.emit_token_debug, + ); + let mut attrs = BTreeMap::new(); + attrs.insert( + "llama_stage.linear_proposal.discard_reason".to_string(), + json!("deadline_exceeded"), + ); + attrs.insert( + "llama_stage.linear_proposal.proposal_us".to_string(), + json!(proposal_elapsed_us), + ); + self.telemetry + .emit("stage.openai_linear_proposal_late", attrs); + None + } + LinearProposalQueryOutcome::Ready(queried) => { + self.emit_linear_proposal_source_telemetry( + queried.source_telemetry, + state.emit_token_debug, + ); + Some(queried) + } + } } - LinearProposalQueryOutcome::Ready(queried) => { - self.emit_linear_proposal_source_telemetry( - queried.source_telemetry, - state.emit_token_debug, + Err(error) => { + state.linear_context_tokens = None; + state.linear_proposal_max_tokens = 0; + self.telemetry.emit( + "stage.openai_linear_proposal_disabled", + BTreeMap::from([( + "llama_stage.linear_proposal.error".to_string(), + json!(error.to_string()), + )]), ); - Some(queried) + return Ok(LinearProposalProgress::NotUsed); } }; if request @@ -114,6 +139,8 @@ impl StageOpenAiBackend { let receipt = execute_linear_proposal_with_terminal_discard(config, &decision_id, || { self.execute_local_linear_proposal( LinearProposalExecutionParams { + request_id: request.ids.request_id, + request_session_id: request.ids.session_id, session_id, current: state.current, base_position, diff --git a/crates/skippy-server/src/frontend/local_generation/tests.rs b/crates/skippy-server/src/frontend/local_generation/tests.rs index b7523d4a1c..b0817e2f43 100644 --- a/crates/skippy-server/src/frontend/local_generation/tests.rs +++ b/crates/skippy-server/src/frontend/local_generation/tests.rs @@ -30,6 +30,7 @@ use crate::telemetry::{Telemetry, TelemetryLevel}; #[derive(Default)] struct RecordingReceiptSink { receipts: Mutex>, + commits: Mutex>, fail: AtomicBool, } @@ -38,7 +39,8 @@ impl GenerationReceiptSink for RecordingReceiptSink { Ok(()) } - fn committed(&self, _commit: &GenerationCommit) -> Result<()> { + fn committed(&self, commit: &GenerationCommit) -> Result<()> { + self.commits.lock().unwrap().push(commit.clone()); Ok(()) } @@ -189,6 +191,9 @@ fn local_generation_eventually_delivers_receipts_and_cleanup_survives_sink_error )?; wait_for_receipts(&sink, 1); + let commits = sink.commits.lock().unwrap(); + assert!(commits.is_empty()); + drop(commits); let receipts = sink.receipts.lock().unwrap(); assert_eq!(receipts.len(), 1); assert_eq!(receipts[0].request_id, ids.request_id); diff --git a/crates/skippy-server/src/frontend/local_generation/token_generation.rs b/crates/skippy-server/src/frontend/local_generation/token_generation.rs index 4bd95073cb..ff55faf608 100644 --- a/crates/skippy-server/src/frontend/local_generation/token_generation.rs +++ b/crates/skippy-server/src/frontend/local_generation/token_generation.rs @@ -60,6 +60,7 @@ pub(super) struct DecodeState { pub(super) generation_hooks_active: bool, pub(super) linear_proposal_max_tokens: usize, pub(super) linear_context_tokens: Option>, + pub(super) pending_linear_proposal_tokens: Vec, pub(super) emit_token_debug: bool, pub(super) native_mtp_options: NativeMtpDecodeOptions, pub(super) native_mtp: NativeMtpVerifier, @@ -747,6 +748,7 @@ impl StageOpenAiBackend { .last() .expect("checked non-empty prompt"); let mut stopped = false; + let mut pending_linear_proposal_tokens = Vec::new(); if let Some(predicted) = prompt_prefill_sample { if request .cancellation @@ -756,6 +758,7 @@ impl StageOpenAiBackend { } current = predicted; decoded_tokens += 1; + pending_linear_proposal_tokens.push(current); stopped = emit_token(current)? == TokenControl::Stop; } let hook_request = request.hook_request.take(); @@ -785,6 +788,9 @@ impl StageOpenAiBackend { } tokens }); + if linear_proposal_max_tokens == 0 { + pending_linear_proposal_tokens.clear(); + } Ok(DecodeState { decoded_tokens, current, @@ -801,6 +807,7 @@ impl StageOpenAiBackend { generation_hooks_active, linear_proposal_max_tokens, linear_context_tokens, + pending_linear_proposal_tokens, emit_token_debug: self.telemetry.is_debug_enabled(), native_mtp_options: NativeMtpDecodeOptions::from_config(request.speculative), native_mtp: NativeMtpVerifier::default(), @@ -834,9 +841,11 @@ impl StageOpenAiBackend { LinearProposalProgress::Stop => break, LinearProposalProgress::NotUsed => {} } - if self.decode_one_token(request, session_id, &mut state, emit_token)? - == TokenControl::Stop - { + let control = self.decode_one_token(request, session_id, &mut state, emit_token)?; + if state.linear_proposal_max_tokens > 0 { + state.pending_linear_proposal_tokens.push(state.current); + } + if control == TokenControl::Stop { break; } }