From e762e781b653ad7880a5fa03b632dae49e511702 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sat, 8 Aug 2026 14:56:57 +1000 Subject: [PATCH 1/4] fix(skippy): emit local generation lifecycle commits --- .../src/frontend/local_generation/tests.rs | 47 ++++++++++++++++++- .../local_generation/token_generation.rs | 31 +++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/crates/skippy-server/src/frontend/local_generation/tests.rs b/crates/skippy-server/src/frontend/local_generation/tests.rs index b7523d4a1c..d6106f2510 100644 --- a/crates/skippy-server/src/frontend/local_generation/tests.rs +++ b/crates/skippy-server/src/frontend/local_generation/tests.rs @@ -17,6 +17,7 @@ use crate::frontend::generation::{ LocalGeneration, OpenAiBackendMode, OpenAiCacheHints, OpenAiGenerationIds, StageOpenAiBackend, TokenControl, }; +use crate::frontend::local_generation::token_generation::commit_local_generation_token; use crate::frontend::local_generation::{ native_mtp_dispatch_counts_for_test, prompt_fits_single_prefill_sample, }; @@ -30,6 +31,7 @@ use crate::telemetry::{Telemetry, TelemetryLevel}; #[derive(Default)] struct RecordingReceiptSink { receipts: Mutex>, + commits: Mutex>, fail: AtomicBool, } @@ -38,7 +40,8 @@ impl GenerationReceiptSink for RecordingReceiptSink { Ok(()) } - fn committed(&self, _commit: &GenerationCommit) -> Result<()> { + fn committed(&self, commit: &GenerationCommit) -> Result<()> { + self.commits.lock().unwrap().push(commit.clone()); Ok(()) } @@ -65,6 +68,16 @@ fn wait_for_receipts(sink: &RecordingReceiptSink, expected: usize) { panic!("timed out waiting for {expected} generation receipts"); } +fn wait_for_commits(sink: &RecordingReceiptSink, expected: usize) { + for _ in 0..100 { + if sink.commits.lock().unwrap().len() >= expected { + return; + } + thread::sleep(Duration::from_millis(1)); + } + panic!("timed out waiting for {expected} generation commits"); +} + #[test] fn single_prefill_sample_requires_prompt_to_fit_session_batch() { assert!(!prompt_fits_single_prefill_sample(0, 2048)); @@ -73,6 +86,28 @@ fn single_prefill_sample_requires_prompt_to_fit_session_batch() { assert!(!prompt_fits_single_prefill_sample(2049, 2048)); } +#[test] +fn local_generation_commits_advance_before_the_next_proposal_boundary() { + let sink = Arc::new(RecordingReceiptSink::default()); + let config = GenerationReceiptConfig::new(sink.clone()); + let mut generated_token_count = 0; + + commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 33); + commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 44); + + wait_for_commits(&sink, 2); + let commits = sink.commits.lock().unwrap(); + assert_eq!(commits.len(), 2); + assert_eq!(commits[0].request_id, 11); + assert_eq!(commits[0].session_id, 22); + assert_eq!(commits[0].generated_token_count, 1); + assert_eq!(commits[0].token_ids.as_ref(), [33]); + assert_eq!(commits[1].request_id, 11); + assert_eq!(commits[1].session_id, 22); + assert_eq!(commits[1].generated_token_count, 2); + assert_eq!(commits[1].token_ids.as_ref(), [44]); +} + #[test] fn local_native_mtp_decode_uses_non_frame_runtime_api() { let (sampled_calls, frame_calls) = native_mtp_dispatch_counts_for_test(); @@ -188,7 +223,17 @@ fn local_generation_eventually_delivers_receipts_and_cleanup_survives_sink_error }, )?; + wait_for_commits(&sink, emitted.len()); wait_for_receipts(&sink, 1); + let commits = sink.commits.lock().unwrap(); + assert_eq!(commits.len(), emitted.len()); + for (index, (commit, token_id)) in commits.iter().zip(&emitted).enumerate() { + assert_eq!(commit.request_id, ids.request_id); + assert_eq!(commit.session_id, ids.session_id); + assert_eq!(commit.generated_token_count, index + 1); + assert_eq!(commit.token_ids.as_ref(), [*token_id]); + } + drop(commits); let receipts = sink.receipts.lock().unwrap(); assert_eq!(receipts.len(), 1); assert_eq!(receipts[0].request_id, ids.request_id); diff --git a/crates/skippy-server/src/frontend/local_generation/token_generation.rs b/crates/skippy-server/src/frontend/local_generation/token_generation.rs index 4bd95073cb..9da357439e 100644 --- a/crates/skippy-server/src/frontend/local_generation/token_generation.rs +++ b/crates/skippy-server/src/frontend/local_generation/token_generation.rs @@ -7,7 +7,9 @@ use crate::frontend::generation::OpenAiGenerationIds; use crate::frontend::generation::PhaseTimer; use crate::frontend::generation::StageOpenAiBackend; use crate::frontend::generation::TokenControl; -use crate::frontend::generation_receipt::{GenerationStart, complete_generation_before_cleanup}; +use crate::frontend::generation_receipt::{ + GenerationCommit, GenerationStart, complete_generation_before_cleanup, +}; use crate::frontend::linear_proposal::greedy_linear_proposal_admitted; use crate::frontend::util::openai_backend_error; use crate::frontend::util::saturating_u32; @@ -29,6 +31,25 @@ use std::time::Duration; use super::{LocalGenerationReceiptFinalization, prompt_fits_single_prefill_sample}; +pub(super) fn commit_local_generation_token( + config: Option<&crate::frontend::GenerationReceiptConfig>, + request_id: u64, + session_id: u64, + generated_token_count: &mut usize, + token_id: i32, +) { + let Some(config) = config else { + return; + }; + *generated_token_count = generated_token_count.saturating_add(1); + config.committed(GenerationCommit { + request_id, + session_id, + generated_token_count: *generated_token_count, + token_ids: vec![token_id].into_boxed_slice(), + }); +} + struct PromptPrefillResult { prompt_prefill_sample: Option, chat_sampling_configured: bool, @@ -109,12 +130,20 @@ impl StageOpenAiBackend { let mut receipt_cancelled = false; let mut receipt_model_generation_elapsed = None; let mut cache_stats = GenerationCacheStats::default(); + let mut lifecycle_committed_token_count = 0usize; let mut emit_token = |token_id| { if let Some(observation) = receipt_observation.as_ref() && let Some(observation) = observation.borrow_mut().as_mut() { observation.record_token(token_id, request.ids.request_started_at.elapsed()); } + commit_local_generation_token( + self.generation_receipt.as_ref(), + receipt_request_id, + receipt_session_id, + &mut lifecycle_committed_token_count, + token_id, + ); let control = on_token(token_id)?; if control == TokenControl::Stop && let Some(observation) = receipt_observation.as_ref() From 512b725b06f337b180726ff7b37f40cfe0e8855d Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sat, 8 Aug 2026 15:20:33 +1000 Subject: [PATCH 2/4] fix(skippy): order local commits before proposals --- .../src/frontend/generation_receipt.rs | 69 +++++++++++++++++-- .../src/frontend/local_generation/tests.rs | 3 +- .../local_generation/token_generation.rs | 2 +- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/crates/skippy-server/src/frontend/generation_receipt.rs b/crates/skippy-server/src/frontend/generation_receipt.rs index a2876ea1a3..c9d8481f8b 100644 --- a/crates/skippy-server/src/frontend/generation_receipt.rs +++ b/crates/skippy-server/src/frontend/generation_receipt.rs @@ -161,29 +161,56 @@ impl GenerationLifecycleObservation { pub trait GenerationLifecycleIngress: Send + Sync { fn try_submit(&self, observation: GenerationLifecycleObservation) -> Result<()>; + /// Submits an observation and waits until it has been delivered. Direct + /// ingresses can use their ordered submission path; queued adapters + /// override this to avoid dropping a commit when their queue is full. + fn submit_ordered(&self, observation: GenerationLifecycleObservation) -> Result<()> { + self.try_submit(observation)?; + self.flush() + } + + /// Waits until all observations submitted before this call have been + /// delivered. Direct ingresses are already ordered at submission time; + /// queued adapters override this to provide an explicit boundary. + fn flush(&self) -> Result<()> { + Ok(()) + } + /// Asynchronous delivery failures observed after a successful submission. fn delivery_failures(&self) -> u64 { 0 } } +enum QueuedGenerationLifecycleMessage { + Observation(GenerationLifecycleObservation), + Barrier(SyncSender>), +} + struct QueuedGenerationReceiptSink { - sender: SyncSender, + sender: SyncSender, delivery_failures: Arc, } impl QueuedGenerationReceiptSink { fn new(sink: Arc) -> Self { let (sender, receiver) = - sync_channel::(GENERATION_RECEIPT_QUEUE_CAPACITY); + sync_channel::(GENERATION_RECEIPT_QUEUE_CAPACITY); let delivery_failures = Arc::new(AtomicU64::new(0)); let worker_delivery_failures = Arc::clone(&delivery_failures); std::thread::Builder::new() .name("skippy-generation-receipts".into()) .spawn(move || { - while let Ok(observation) = receiver.recv() { - if observation.deliver_to(sink.as_ref()).is_err() { - worker_delivery_failures.fetch_add(1, Ordering::Relaxed); + while let Ok(message) = receiver.recv() { + match message { + QueuedGenerationLifecycleMessage::Observation(observation) => { + if observation.deliver_to(sink.as_ref()).is_err() { + worker_delivery_failures.fetch_add(1, Ordering::Relaxed); + } + } + QueuedGenerationLifecycleMessage::Barrier(completion) => { + let _ = completion.send(Ok(())); + } } } }) @@ -198,7 +225,7 @@ impl QueuedGenerationReceiptSink { impl GenerationLifecycleIngress for QueuedGenerationReceiptSink { fn try_submit(&self, observation: GenerationLifecycleObservation) -> Result<()> { self.sender - .try_send(observation) + .try_send(QueuedGenerationLifecycleMessage::Observation(observation)) .map_err(|error| match error { TrySendError::Full(_) => anyhow::anyhow!("generation lifecycle queue is full"), TrySendError::Disconnected(_) => { @@ -210,6 +237,23 @@ impl GenerationLifecycleIngress for QueuedGenerationReceiptSink { fn delivery_failures(&self) -> u64 { self.delivery_failures.load(Ordering::Relaxed) } + + fn flush(&self) -> Result<()> { + let (completion, completed) = sync_channel(0); + self.sender + .send(QueuedGenerationLifecycleMessage::Barrier(completion)) + .map_err(|_| anyhow::anyhow!("generation lifecycle delivery worker is unavailable"))?; + completed + .recv() + .map_err(|_| anyhow::anyhow!("generation lifecycle delivery worker is unavailable"))? + } + + fn submit_ordered(&self, observation: GenerationLifecycleObservation) -> Result<()> { + self.sender + .send(QueuedGenerationLifecycleMessage::Observation(observation)) + .map_err(|_| anyhow::anyhow!("generation lifecycle delivery worker is unavailable"))?; + self.flush() + } } /// Optional local-generation observation. @@ -284,6 +328,19 @@ impl GenerationReceiptConfig { self.enqueue(GenerationLifecycleObservation::Committed(commit)); } + /// Commits a token and waits until the lifecycle ingress has delivered it. + /// Local generation uses this before the next proposal lookup so consumers + /// cannot observe a stale canonical position. + pub(crate) fn committed_before_proposal(&self, commit: GenerationCommit) { + if self + .ingress + .submit_ordered(GenerationLifecycleObservation::Committed(commit)) + .is_err() + { + self.submission_failures.fetch_add(1, Ordering::Relaxed); + } + } + pub(crate) fn abort(&self, abort: GenerationAbort) { self.enqueue(GenerationLifecycleObservation::Aborted(abort)); } diff --git a/crates/skippy-server/src/frontend/local_generation/tests.rs b/crates/skippy-server/src/frontend/local_generation/tests.rs index d6106f2510..3f62a5456d 100644 --- a/crates/skippy-server/src/frontend/local_generation/tests.rs +++ b/crates/skippy-server/src/frontend/local_generation/tests.rs @@ -87,7 +87,7 @@ fn single_prefill_sample_requires_prompt_to_fit_session_batch() { } #[test] -fn local_generation_commits_advance_before_the_next_proposal_boundary() { +fn local_generation_commit_helper_preserves_ordered_counts() { let sink = Arc::new(RecordingReceiptSink::default()); let config = GenerationReceiptConfig::new(sink.clone()); let mut generated_token_count = 0; @@ -95,7 +95,6 @@ fn local_generation_commits_advance_before_the_next_proposal_boundary() { commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 33); commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 44); - wait_for_commits(&sink, 2); let commits = sink.commits.lock().unwrap(); assert_eq!(commits.len(), 2); assert_eq!(commits[0].request_id, 11); diff --git a/crates/skippy-server/src/frontend/local_generation/token_generation.rs b/crates/skippy-server/src/frontend/local_generation/token_generation.rs index 9da357439e..239c51c46d 100644 --- a/crates/skippy-server/src/frontend/local_generation/token_generation.rs +++ b/crates/skippy-server/src/frontend/local_generation/token_generation.rs @@ -42,7 +42,7 @@ pub(super) fn commit_local_generation_token( return; }; *generated_token_count = generated_token_count.saturating_add(1); - config.committed(GenerationCommit { + config.committed_before_proposal(GenerationCommit { request_id, session_id, generated_token_count: *generated_token_count, From b54f3d7c03f0d63c1e57c67d9e5d7f223c24daf5 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sat, 8 Aug 2026 16:42:13 +1000 Subject: [PATCH 3/4] fix(skippy): bound ordered lifecycle delivery --- .../src/frontend/generation_receipt.rs | 142 +++++++++++++++--- .../src/frontend/local_generation/tests.rs | 4 +- .../local_generation/token_generation.rs | 20 +-- 3 files changed, 133 insertions(+), 33 deletions(-) diff --git a/crates/skippy-server/src/frontend/generation_receipt.rs b/crates/skippy-server/src/frontend/generation_receipt.rs index c9d8481f8b..507337a291 100644 --- a/crates/skippy-server/src/frontend/generation_receipt.rs +++ b/crates/skippy-server/src/frontend/generation_receipt.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::mpsc::{SyncSender, TrySendError, sync_channel}; -use std::time::Duration; +use std::sync::mpsc::{RecvTimeoutError, SyncSender, TrySendError, sync_channel}; +use std::time::{Duration, Instant}; use anyhow::Result; use openai_frontend::{OpenAiError, OpenAiResult}; @@ -11,6 +11,7 @@ use crate::runtime_state::RuntimeState; const TOKEN_ID_DIGEST_DOMAIN: &[u8] = b"skippy-generation-token-ids-v1\0"; const GENERATION_RECEIPT_QUEUE_CAPACITY: usize = 1_024; +const GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT: Duration = Duration::from_millis(250); /// Why a successful local generation stopped. #[non_exhaustive] @@ -201,15 +202,25 @@ impl QueuedGenerationReceiptSink { std::thread::Builder::new() .name("skippy-generation-receipts".into()) .spawn(move || { + let mut failures_since_barrier = 0u64; while let Ok(message) = receiver.recv() { match message { QueuedGenerationLifecycleMessage::Observation(observation) => { if observation.deliver_to(sink.as_ref()).is_err() { worker_delivery_failures.fetch_add(1, Ordering::Relaxed); + failures_since_barrier = failures_since_barrier.saturating_add(1); } } QueuedGenerationLifecycleMessage::Barrier(completion) => { - let _ = completion.send(Ok(())); + let result = if failures_since_barrier == 0 { + Ok(()) + } else { + failures_since_barrier = 0; + Err(anyhow::anyhow!( + "generation lifecycle observation delivery failed" + )) + }; + let _ = completion.send(result); } } } @@ -239,20 +250,60 @@ impl GenerationLifecycleIngress for QueuedGenerationReceiptSink { } fn flush(&self) -> Result<()> { - let (completion, completed) = sync_channel(0); - self.sender - .send(QueuedGenerationLifecycleMessage::Barrier(completion)) - .map_err(|_| anyhow::anyhow!("generation lifecycle delivery worker is unavailable"))?; - completed - .recv() - .map_err(|_| anyhow::anyhow!("generation lifecycle delivery worker is unavailable"))? + self.flush_until(Instant::now() + GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT) } fn submit_ordered(&self, observation: GenerationLifecycleObservation) -> Result<()> { - self.sender - .send(QueuedGenerationLifecycleMessage::Observation(observation)) - .map_err(|_| anyhow::anyhow!("generation lifecycle delivery worker is unavailable"))?; - self.flush() + let deadline = Instant::now() + GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT; + self.send_until( + QueuedGenerationLifecycleMessage::Observation(observation), + deadline, + )?; + self.flush_until(deadline) + } +} + +impl QueuedGenerationReceiptSink { + fn send_until( + &self, + mut message: QueuedGenerationLifecycleMessage, + deadline: Instant, + ) -> Result<()> { + loop { + match self.sender.try_send(message) { + Ok(()) => return Ok(()), + Err(TrySendError::Disconnected(_)) => { + return Err(anyhow::anyhow!( + "generation lifecycle delivery worker is unavailable" + )); + } + Err(TrySendError::Full(returned_message)) => { + if Instant::now() >= deadline { + return Err(anyhow::anyhow!("generation lifecycle delivery timed out")); + } + message = returned_message; + std::thread::yield_now(); + } + } + } + } + + fn flush_until(&self, deadline: Instant) -> Result<()> { + let (completion, completed) = sync_channel(0); + self.send_until( + QueuedGenerationLifecycleMessage::Barrier(completion), + deadline, + )?; + let timeout = deadline.saturating_duration_since(Instant::now()); + match completed.recv_timeout(timeout) { + Ok(result) => result, + Err(RecvTimeoutError::Timeout) => { + Err(anyhow::anyhow!("generation lifecycle delivery timed out")) + } + Err(RecvTimeoutError::Disconnected) => Err(anyhow::anyhow!( + "generation lifecycle delivery worker is unavailable" + )), + } } } @@ -331,14 +382,12 @@ impl GenerationReceiptConfig { /// Commits a token and waits until the lifecycle ingress has delivered it. /// Local generation uses this before the next proposal lookup so consumers /// cannot observe a stale canonical position. - pub(crate) fn committed_before_proposal(&self, commit: GenerationCommit) { - if self - .ingress + pub(crate) fn committed_before_proposal(&self, commit: GenerationCommit) -> Result<()> { + self.ingress .submit_ordered(GenerationLifecycleObservation::Committed(commit)) - .is_err() - { - self.submission_failures.fetch_add(1, Ordering::Relaxed); - } + .inspect_err(|_| { + self.submission_failures.fetch_add(1, Ordering::Relaxed); + }) } pub(crate) fn abort(&self, abort: GenerationAbort) { @@ -581,6 +630,7 @@ fn duration_us(duration: Duration) -> u64 { #[cfg(test)] mod tests { use std::sync::Mutex; + use std::sync::atomic::AtomicBool; use std::thread; use super::*; @@ -608,6 +658,8 @@ mod tests { struct RecordingSink { receipts: Mutex>, error: Option<&'static str>, + fail_commits: AtomicBool, + commit_delay: Duration, } #[derive(Default)] @@ -630,7 +682,14 @@ mod tests { } fn committed(&self, _commit: &GenerationCommit) -> Result<()> { - Ok(()) + if !self.commit_delay.is_zero() { + thread::sleep(self.commit_delay); + } + if self.fail_commits.load(Ordering::Relaxed) { + Err(anyhow::anyhow!("commit failed")) + } else { + Ok(()) + } } fn abort(&self, _abort: &GenerationAbort) -> Result<()> { @@ -654,6 +713,44 @@ mod tests { panic!("timed out waiting for {expected} generation receipts"); } + fn test_commit() -> GenerationCommit { + GenerationCommit { + request_id: 1, + session_id: 2, + generated_token_count: 1, + token_ids: vec![3].into_boxed_slice(), + } + } + + #[test] + fn queued_barrier_reports_delivery_failure_and_resets_interval() { + let sink = Arc::new(RecordingSink::default()); + sink.fail_commits.store(true, Ordering::Relaxed); + let config = GenerationReceiptConfig::new(sink.clone()); + + assert!(config.committed_before_proposal(test_commit()).is_err()); + assert_eq!(config.delivery_failures(), 2); + + sink.fail_commits.store(false, Ordering::Relaxed); + assert!(config.committed_before_proposal(test_commit()).is_ok()); + assert_eq!(config.delivery_failures(), 2); + } + + #[test] + fn queued_ordered_delivery_times_out_when_sink_stalls() { + let sink = Arc::new(RecordingSink { + commit_delay: GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT.saturating_mul(4), + ..Default::default() + }); + let config = GenerationReceiptConfig::new(sink); + let started = Instant::now(); + + let error = config.committed_before_proposal(test_commit()).unwrap_err(); + + assert!(error.to_string().contains("timed out")); + assert!(started.elapsed() < GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT.saturating_mul(2)); + } + #[test] fn token_digest_is_stable_and_order_sensitive() { let digest = generation_token_id_digest(&[-1, 0, 1, i32::MAX]); @@ -881,6 +978,7 @@ mod tests { let failing_sink = Arc::new(RecordingSink { receipts: Mutex::new(Vec::new()), error: Some("sink failed"), + ..Default::default() }); let failing_config = GenerationReceiptConfig::new(failing_sink); let mut observation = test_observation(0); diff --git a/crates/skippy-server/src/frontend/local_generation/tests.rs b/crates/skippy-server/src/frontend/local_generation/tests.rs index 3f62a5456d..03d6a9c776 100644 --- a/crates/skippy-server/src/frontend/local_generation/tests.rs +++ b/crates/skippy-server/src/frontend/local_generation/tests.rs @@ -92,8 +92,8 @@ fn local_generation_commit_helper_preserves_ordered_counts() { let config = GenerationReceiptConfig::new(sink.clone()); let mut generated_token_count = 0; - commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 33); - commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 44); + commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 33).unwrap(); + commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 44).unwrap(); let commits = sink.commits.lock().unwrap(); assert_eq!(commits.len(), 2); diff --git a/crates/skippy-server/src/frontend/local_generation/token_generation.rs b/crates/skippy-server/src/frontend/local_generation/token_generation.rs index 239c51c46d..0b5017bbf3 100644 --- a/crates/skippy-server/src/frontend/local_generation/token_generation.rs +++ b/crates/skippy-server/src/frontend/local_generation/token_generation.rs @@ -37,17 +37,19 @@ pub(super) fn commit_local_generation_token( session_id: u64, generated_token_count: &mut usize, token_id: i32, -) { +) -> OpenAiResult<()> { let Some(config) = config else { - return; + return Ok(()); }; *generated_token_count = generated_token_count.saturating_add(1); - config.committed_before_proposal(GenerationCommit { - request_id, - session_id, - generated_token_count: *generated_token_count, - token_ids: vec![token_id].into_boxed_slice(), - }); + config + .committed_before_proposal(GenerationCommit { + request_id, + session_id, + generated_token_count: *generated_token_count, + token_ids: vec![token_id].into_boxed_slice(), + }) + .map_err(openai_backend_error) } struct PromptPrefillResult { @@ -143,7 +145,7 @@ impl StageOpenAiBackend { receipt_session_id, &mut lifecycle_committed_token_count, token_id, - ); + )?; let control = on_token(token_id)?; if control == TokenControl::Stop && let Some(observation) = receipt_observation.as_ref() From 52b077ed023939b64e31ab49bd33ceb6bc46d258 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Sat, 8 Aug 2026 18:13:35 +1000 Subject: [PATCH 4/4] fix(skippy): keep proposal commits off lifecycle hot path --- .../src/lib.rs | 351 ++++++++++++++++-- .../src/frontend/generation_receipt.rs | 176 +-------- .../src/frontend/linear_proposal.rs | 110 +++++- .../src/frontend/linear_proposal/execution.rs | 4 + .../local_generation/linear_decode.rs | 61 ++- .../src/frontend/local_generation/tests.rs | 41 +- .../local_generation/token_generation.rs | 48 +-- 7 files changed, 488 insertions(+), 303 deletions(-) diff --git a/crates/mesh-native-serving-plugin-host/src/lib.rs b/crates/mesh-native-serving-plugin-host/src/lib.rs index a84c2ad056..75456712a0 100644 --- a/crates/mesh-native-serving-plugin-host/src/lib.rs +++ b/crates/mesh-native-serving-plugin-host/src/lib.rs @@ -1,6 +1,7 @@ //! Loads one native serving plugin and adapts its stable ABI to Skippy hooks. use std::{ + collections::HashMap, ffi::{c_char, c_void}, path::{Path, PathBuf}, ptr::NonNull, @@ -125,6 +126,7 @@ impl ModelServingHooksFactory for NativeServingPluginFactory { definition: Arc::clone(&self.definition), instance: Some(instance), proposal_token_buffer: vec![0; MAX_NATIVE_PLUGIN_PROPOSAL_TOKENS], + committed_generated_tokens: HashMap::new(), }; let driver = Arc::new(PluginDriver::spawn(active)?); let lifecycle: Arc = Arc::new(NativeLifecycleIngress { @@ -270,6 +272,7 @@ struct ActivePlugin { definition: Arc, instance: Option>, proposal_token_buffer: Vec, + committed_generated_tokens: HashMap<(u64, u64), usize>, } // SAFETY: activation succeeds only for plugins implementing the ABI's @@ -296,7 +299,7 @@ impl ActivePlugin { )) } - fn begin(&self, start: &GenerationStart) -> Result<()> { + fn begin(&mut self, start: &GenerationStart) -> Result<()> { let agent_session_id = start.agent_session_id.as_deref().unwrap_or_default(); let event = abi::GenerationStart { struct_size: size_of::(), @@ -306,32 +309,79 @@ impl ActivePlugin { prompt_token_ids: abi::TokenSlice::from_tokens(&start.prompt_token_ids), }; let status = unsafe { (self.definition.api().begin_generation)(self.instance()?, &event) }; - self.call_status("begin generation", status) + self.call_status("begin generation", status)?; + self.committed_generated_tokens + .insert((start.request_id, start.session_id), 0); + Ok(()) } - fn committed(&self, commit: &GenerationCommit) -> Result<()> { + fn committed(&mut self, commit: &GenerationCommit) -> Result<()> { + self.commit_tokens( + commit.request_id, + commit.session_id, + commit.generated_token_count, + &commit.token_ids, + ) + } + + fn commit_tokens( + &mut self, + request_id: u64, + session_id: u64, + generated_token_count: usize, + token_ids: &[i32], + ) -> Result<()> { + if token_ids.is_empty() { + return Ok(()); + } + let key = (request_id, session_id); + let prior_generated_token_count = self + .committed_generated_tokens + .get(&key) + .copied() + .unwrap_or_else(|| generated_token_count.saturating_sub(token_ids.len())); + let expected_generated_token_count = prior_generated_token_count + .checked_add(token_ids.len()) + .context("native serving plugin generated-token count overflow")?; + if generated_token_count != expected_generated_token_count { + bail!( + "native serving plugin commit does not extend the tracked generated-token prefix" + ); + } let event = abi::GenerationCommit { struct_size: size_of::(), - request_id: commit.request_id, - session_id: commit.session_id, - generated_token_count: u64::try_from(commit.generated_token_count)?, - token_ids: abi::TokenSlice::from_tokens(&commit.token_ids), + request_id, + session_id, + generated_token_count: u64::try_from(generated_token_count)?, + token_ids: abi::TokenSlice::from_tokens(token_ids), }; let status = unsafe { (self.definition.api().commit_generation)(self.instance()?, &event) }; - self.call_status("commit generation", status) + self.call_status("commit generation", status)?; + self.committed_generated_tokens + .insert(key, generated_token_count); + Ok(()) } - fn abort(&self, abort: &GenerationAbort) -> Result<()> { + fn abort(&mut self, abort: &GenerationAbort) -> Result<()> { let event = abi::GenerationAbort { struct_size: size_of::(), request_id: abort.request_id, session_id: abort.session_id, }; let status = unsafe { (self.definition.api().abort_generation)(self.instance()?, &event) }; - self.call_status("abort generation", status) + let result = self.call_status("abort generation", status); + self.committed_generated_tokens + .remove(&(abort.request_id, abort.session_id)); + result } - fn finish(&self, receipt: &GenerationReceipt) -> Result<()> { + fn finish(&mut self, receipt: &GenerationReceipt) -> Result<()> { + let key = (receipt.request_id, receipt.session_id); + self.commit_final_suffix( + receipt.request_id, + receipt.session_id, + &receipt.generated_token_ids, + )?; let request_to_first_token_us = receipt.request_to_first_token_us.unwrap_or_default(); let event = abi::GenerationFinish { struct_size: size_of::(), @@ -351,10 +401,49 @@ impl ActivePlugin { ), }; let status = unsafe { (self.definition.api().finish_generation)(self.instance()?, &event) }; - self.call_status("finish generation", status) + let result = self.call_status("finish generation", status); + self.committed_generated_tokens.remove(&key); + result + } + + fn commit_final_suffix( + &mut self, + request_id: u64, + session_id: u64, + generated_token_ids: &[i32], + ) -> Result<()> { + let key = (request_id, session_id); + let committed = self + .committed_generated_tokens + .get(&key) + .copied() + .unwrap_or_default(); + if committed > generated_token_ids.len() { + bail!("native serving plugin committed beyond the final generation receipt"); + } + if committed < generated_token_ids.len() { + self.commit_tokens( + request_id, + session_id, + generated_token_ids.len(), + &generated_token_ids[committed..], + )?; + } + Ok(()) } fn propose(&mut self, query: LinearProposalQuery) -> Result> { + if !query.pending_token_ids.is_empty() { + let generated_token_count = query + .committed_token_count + .saturating_sub(query.prompt_token_count); + self.commit_tokens( + query.request_id, + query.session_id, + generated_token_count, + &query.pending_token_ids, + )?; + } let event = abi::ProposalQuery { struct_size: size_of::(), request_id: query.request_id, @@ -445,7 +534,12 @@ impl ActivePlugin { Ok(None) } - fn report(&self, receipt: &LinearProposalReceipt) -> Result<()> { + fn report(&mut self, receipt: &LinearProposalReceipt) -> Result<()> { + self.commit_proposal_tokens( + receipt.request_id, + receipt.session_id, + &receipt.committed_tokens, + )?; let event = abi::ProposalOutcome { struct_size: size_of::(), decision_id: abi::ByteSlice::from_bytes(receipt.decision_id.as_bytes()), @@ -469,6 +563,23 @@ impl ActivePlugin { self.call_status("report proposal", status) } + fn commit_proposal_tokens( + &mut self, + request_id: u64, + session_id: u64, + committed_tokens: &[i32], + ) -> Result<()> { + let key = (request_id, session_id); + let generated_token_count = self + .committed_generated_tokens + .get(&key) + .copied() + .unwrap_or_default() + .checked_add(committed_tokens.len()) + .context("native serving plugin generated-token count overflow")?; + self.commit_tokens(key.0, key.1, generated_token_count, committed_tokens) + } + fn discard(&self, decision_id: &[u8], reason: LinearProposalDiscardReason) -> Result<()> { let event = abi::ProposalDiscard { struct_size: size_of::(), @@ -628,7 +739,9 @@ impl PluginDriver { let (reply, response) = sync_channel(1); match self.sender.try_send(PluginCommand::Proposal(query, reply)) { Ok(()) => {} - Err(TrySendError::Full(_)) => return Ok(None), + Err(TrySendError::Full(_)) => { + bail!("native serving plugin command queue is full before proposal handoff") + } Err(TrySendError::Disconnected(_)) => { bail!("native serving plugin worker stopped") } @@ -821,7 +934,6 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; static FAKE_NAME: &[u8] = b"test-serving-plugin"; - static CANCEL_COUNT: AtomicUsize = AtomicUsize::new(0); #[test] fn tokenizer_inventory_view_borrows_host_owned_bytes_for_activation() { @@ -851,11 +963,20 @@ mod tests { struct FakeState { start_delay: Duration, + commit_delay: Duration, begin_fails: bool, events: Arc>>, abort_count: Arc, + cancel_count: Arc, } + type FakeActiveWithCounters = ( + ActivePlugin, + Arc>>, + Arc, + Arc, + ); + unsafe extern "C" fn fake_activate( _context: *const abi::ActivationContext, _activation: *mut abi::PluginActivation, @@ -889,6 +1010,7 @@ mod tests { ) -> abi::PluginStatus { let state = unsafe { &*instance.cast::() }; state.events.lock().unwrap().push("commit"); + thread::sleep(state.commit_delay); abi::PluginStatus::OK } @@ -902,9 +1024,11 @@ mod tests { } unsafe extern "C" fn fake_finish( - _instance: abi::PluginInstance, + instance: abi::PluginInstance, _event: *const abi::GenerationFinish, ) -> abi::PluginStatus { + let state = unsafe { &*instance.cast::() }; + state.events.lock().unwrap().push("finish"); abi::PluginStatus::OK } @@ -929,16 +1053,19 @@ mod tests { } unsafe extern "C" fn fake_cancel_proposal( - _instance: abi::PluginInstance, + instance: abi::PluginInstance, _operation: abi::ProposalOperation, ) { - CANCEL_COUNT.fetch_add(1, Ordering::SeqCst); + let state = unsafe { &*instance.cast::() }; + state.cancel_count.fetch_add(1, Ordering::SeqCst); } unsafe extern "C" fn fake_report_proposal( - _instance: abi::PluginInstance, + instance: abi::PluginInstance, _event: *const abi::ProposalOutcome, ) -> abi::PluginStatus { + let state = unsafe { &*instance.cast::() }; + state.events.lock().unwrap().push("report"); abi::PluginStatus::OK } @@ -977,10 +1104,6 @@ mod tests { } } - fn fake_active(start_delay: Duration) -> ActivePlugin { - fake_active_with_events(start_delay).0 - } - fn fake_active_with_events( start_delay: Duration, ) -> (ActivePlugin, Arc>>) { @@ -1006,6 +1129,28 @@ mod tests { Arc>>, Arc, ) { + fake_active_with_delays(start_delay, Duration::ZERO, begin_fails) + } + + fn fake_active_with_delays( + start_delay: Duration, + commit_delay: Duration, + begin_fails: bool, + ) -> ( + ActivePlugin, + Arc>>, + Arc, + ) { + let (active, events, abort_count, _) = + fake_active_with_counters(start_delay, commit_delay, begin_fails); + (active, events, abort_count) + } + + fn fake_active_with_counters( + start_delay: Duration, + commit_delay: Duration, + begin_fails: bool, + ) -> FakeActiveWithCounters { let table = Box::leak(Box::new(fake_table())); let definition = Arc::new(LoadedDefinition { _library: None, @@ -1014,20 +1159,25 @@ mod tests { }); let events = Arc::new(Mutex::new(Vec::new())); let abort_count = Arc::new(AtomicUsize::new(0)); + let cancel_count = Arc::new(AtomicUsize::new(0)); let state = Box::new(FakeState { start_delay, + commit_delay, begin_fails, events: Arc::clone(&events), abort_count: Arc::clone(&abort_count), + cancel_count: Arc::clone(&cancel_count), }); ( ActivePlugin { definition, instance: NonNull::new(Box::into_raw(state).cast::()), proposal_token_buffer: vec![0; MAX_NATIVE_PLUGIN_PROPOSAL_TOKENS], + committed_generated_tokens: HashMap::new(), }, events, abort_count, + cancel_count, ) } @@ -1085,8 +1235,9 @@ mod tests { #[test] fn blocking_plugin_cannot_extend_the_decode_deadline() { - CANCEL_COUNT.store(0, Ordering::SeqCst); - let driver = PluginDriver::spawn(fake_active(Duration::from_millis(250))).unwrap(); + let (active, _, _, cancel_count) = + fake_active_with_counters(Duration::from_millis(250), Duration::ZERO, false); + let driver = PluginDriver::spawn(active).unwrap(); let started = Instant::now(); let result = driver .propose(LinearProposalQuery::new( @@ -1107,7 +1258,39 @@ mod tests { "decode waited {elapsed:?} for a blocking plugin" ); drop(driver); - assert_eq!(CANCEL_COUNT.load(Ordering::SeqCst), 1); + assert_eq!(cancel_count.load(Ordering::SeqCst), 1); + } + + #[test] + fn blocking_commit_cannot_extend_the_decode_deadline() { + let (active, _, _) = + fake_active_with_delays(Duration::ZERO, Duration::from_millis(250), false); + let driver = Arc::new(PluginDriver::spawn(active).unwrap()); + let ingress = NativeLifecycleIngress { + driver: Arc::clone(&driver), + }; + ingress + .try_submit(GenerationLifecycleObservation::Started(GenerationStart { + request_id: 1, + session_id: 2, + agent_session_id: None, + prompt_token_ids: Arc::from([3]), + })) + .unwrap(); + let started = Instant::now(); + let result = driver + .propose( + LinearProposalQuery::new(1, 2, 1, 2, 1, 8, started + Duration::from_millis(5)) + .with_pending_token_ids(vec![4].into_boxed_slice()), + ) + .unwrap(); + let elapsed = started.elapsed(); + + assert!(result.is_none()); + assert!( + elapsed < Duration::from_millis(150), + "decode waited {elapsed:?} for a blocking commit" + ); } #[test] @@ -1150,6 +1333,122 @@ mod tests { assert_eq!(*events.lock().unwrap(), ["begin", "commit", "proposal"]); } + #[test] + fn proposal_applies_pending_tokens_before_lookup() { + let (active, events) = fake_active_with_events(Duration::ZERO); + let mut active = active; + active + .begin(&GenerationStart { + request_id: 1, + session_id: 2, + agent_session_id: None, + prompt_token_ids: Arc::from([3]), + }) + .unwrap(); + let result = active + .propose( + LinearProposalQuery::new( + 1, + 2, + 1, + 3, + 2, + 8, + Instant::now() + Duration::from_millis(100), + ) + .with_pending_token_ids(vec![4, 5].into_boxed_slice()), + ) + .unwrap(); + + assert!(result.is_none()); + assert_eq!(*events.lock().unwrap(), ["begin", "commit", "proposal"]); + } + + #[test] + fn proposal_report_commits_verified_tokens_before_outcome() { + let (active, events) = fake_active_with_events(Duration::ZERO); + let mut active = active; + active + .begin(&GenerationStart { + request_id: 1, + session_id: 2, + agent_session_id: None, + prompt_token_ids: Arc::from([3]), + }) + .unwrap(); + active.commit_proposal_tokens(1, 2, &[4, 5]).unwrap(); + let status = unsafe { + (active.definition.api().report_proposal)(active.instance().unwrap(), std::ptr::null()) + }; + assert_eq!(status, abi::PluginStatus::OK); + active.commit_final_suffix(1, 2, &[4, 5]).unwrap(); + + assert_eq!(*events.lock().unwrap(), ["begin", "commit", "report"]); + } + + #[test] + fn finish_commits_the_remaining_generated_suffix() { + let (active, events) = fake_active_with_events(Duration::ZERO); + let mut active = active; + active + .begin(&GenerationStart { + request_id: 1, + session_id: 2, + agent_session_id: None, + prompt_token_ids: Arc::from([3]), + }) + .unwrap(); + active + .committed(&GenerationCommit { + request_id: 1, + session_id: 2, + generated_token_count: 1, + token_ids: vec![4].into_boxed_slice(), + }) + .unwrap(); + active.commit_final_suffix(1, 2, &[4, 5, 6]).unwrap(); + let status = unsafe { + (active.definition.api().finish_generation)( + active.instance().unwrap(), + std::ptr::null(), + ) + }; + assert_eq!(status, abi::PluginStatus::OK); + + assert_eq!( + *events.lock().unwrap(), + ["begin", "commit", "commit", "finish"] + ); + } + + #[test] + fn saturated_plugin_queue_rejects_proposal_handoff() { + let (sender, _receiver) = sync_channel(0); + let driver = PluginDriver { + sender, + fatal_error: Arc::new(Mutex::new(None)), + lifecycle_delivery_failures: Arc::new(AtomicU64::new(0)), + worker: Mutex::new(None), + }; + + let error = driver + .propose( + LinearProposalQuery::new( + 1, + 2, + 1, + 2, + 1, + 8, + Instant::now() + Duration::from_millis(100), + ) + .with_pending_token_ids(vec![4].into_boxed_slice()), + ) + .unwrap_err(); + + assert!(error.to_string().contains("before proposal handoff")); + } + #[test] fn lifecycle_callback_failure_is_observed_without_poisoning_the_driver() { let (active, _, _) = fake_active_with_options(Duration::ZERO, true); diff --git a/crates/skippy-server/src/frontend/generation_receipt.rs b/crates/skippy-server/src/frontend/generation_receipt.rs index 507337a291..fc81df0c5c 100644 --- a/crates/skippy-server/src/frontend/generation_receipt.rs +++ b/crates/skippy-server/src/frontend/generation_receipt.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::mpsc::{RecvTimeoutError, SyncSender, TrySendError, sync_channel}; -use std::time::{Duration, Instant}; +use std::sync::mpsc::{SyncSender, TrySendError, sync_channel}; +use std::time::Duration; use anyhow::Result; use openai_frontend::{OpenAiError, OpenAiResult}; @@ -11,7 +11,6 @@ use crate::runtime_state::RuntimeState; const TOKEN_ID_DIGEST_DOMAIN: &[u8] = b"skippy-generation-token-ids-v1\0"; const GENERATION_RECEIPT_QUEUE_CAPACITY: usize = 1_024; -const GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT: Duration = Duration::from_millis(250); /// Why a successful local generation stopped. #[non_exhaustive] @@ -116,7 +115,8 @@ pub struct GenerationCommit { pub trait GenerationReceiptSink: Send + Sync { fn begin(&self, start: &GenerationStart) -> Result<()>; - /// Observes canonical target tokens submitted before the next proposal lookup. + /// Optionally observes canonical target-token deltas for integrations that + /// need lifecycle notifications outside the local proposal hot path. fn committed(&self, commit: &GenerationCommit) -> Result<()>; fn abort(&self, abort: &GenerationAbort) -> Result<()>; @@ -162,66 +162,29 @@ impl GenerationLifecycleObservation { pub trait GenerationLifecycleIngress: Send + Sync { fn try_submit(&self, observation: GenerationLifecycleObservation) -> Result<()>; - /// Submits an observation and waits until it has been delivered. Direct - /// ingresses can use their ordered submission path; queued adapters - /// override this to avoid dropping a commit when their queue is full. - fn submit_ordered(&self, observation: GenerationLifecycleObservation) -> Result<()> { - self.try_submit(observation)?; - self.flush() - } - - /// Waits until all observations submitted before this call have been - /// delivered. Direct ingresses are already ordered at submission time; - /// queued adapters override this to provide an explicit boundary. - fn flush(&self) -> Result<()> { - Ok(()) - } - /// Asynchronous delivery failures observed after a successful submission. fn delivery_failures(&self) -> u64 { 0 } } -enum QueuedGenerationLifecycleMessage { - Observation(GenerationLifecycleObservation), - Barrier(SyncSender>), -} - struct QueuedGenerationReceiptSink { - sender: SyncSender, + sender: SyncSender, delivery_failures: Arc, } impl QueuedGenerationReceiptSink { fn new(sink: Arc) -> Self { let (sender, receiver) = - sync_channel::(GENERATION_RECEIPT_QUEUE_CAPACITY); + sync_channel::(GENERATION_RECEIPT_QUEUE_CAPACITY); let delivery_failures = Arc::new(AtomicU64::new(0)); let worker_delivery_failures = Arc::clone(&delivery_failures); std::thread::Builder::new() .name("skippy-generation-receipts".into()) .spawn(move || { - let mut failures_since_barrier = 0u64; - while let Ok(message) = receiver.recv() { - match message { - QueuedGenerationLifecycleMessage::Observation(observation) => { - if observation.deliver_to(sink.as_ref()).is_err() { - worker_delivery_failures.fetch_add(1, Ordering::Relaxed); - failures_since_barrier = failures_since_barrier.saturating_add(1); - } - } - QueuedGenerationLifecycleMessage::Barrier(completion) => { - let result = if failures_since_barrier == 0 { - Ok(()) - } else { - failures_since_barrier = 0; - Err(anyhow::anyhow!( - "generation lifecycle observation delivery failed" - )) - }; - let _ = completion.send(result); - } + while let Ok(observation) = receiver.recv() { + if observation.deliver_to(sink.as_ref()).is_err() { + worker_delivery_failures.fetch_add(1, Ordering::Relaxed); } } }) @@ -236,7 +199,7 @@ impl QueuedGenerationReceiptSink { impl GenerationLifecycleIngress for QueuedGenerationReceiptSink { fn try_submit(&self, observation: GenerationLifecycleObservation) -> Result<()> { self.sender - .try_send(QueuedGenerationLifecycleMessage::Observation(observation)) + .try_send(observation) .map_err(|error| match error { TrySendError::Full(_) => anyhow::anyhow!("generation lifecycle queue is full"), TrySendError::Disconnected(_) => { @@ -248,63 +211,6 @@ impl GenerationLifecycleIngress for QueuedGenerationReceiptSink { fn delivery_failures(&self) -> u64 { self.delivery_failures.load(Ordering::Relaxed) } - - fn flush(&self) -> Result<()> { - self.flush_until(Instant::now() + GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT) - } - - fn submit_ordered(&self, observation: GenerationLifecycleObservation) -> Result<()> { - let deadline = Instant::now() + GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT; - self.send_until( - QueuedGenerationLifecycleMessage::Observation(observation), - deadline, - )?; - self.flush_until(deadline) - } -} - -impl QueuedGenerationReceiptSink { - fn send_until( - &self, - mut message: QueuedGenerationLifecycleMessage, - deadline: Instant, - ) -> Result<()> { - loop { - match self.sender.try_send(message) { - Ok(()) => return Ok(()), - Err(TrySendError::Disconnected(_)) => { - return Err(anyhow::anyhow!( - "generation lifecycle delivery worker is unavailable" - )); - } - Err(TrySendError::Full(returned_message)) => { - if Instant::now() >= deadline { - return Err(anyhow::anyhow!("generation lifecycle delivery timed out")); - } - message = returned_message; - std::thread::yield_now(); - } - } - } - } - - fn flush_until(&self, deadline: Instant) -> Result<()> { - let (completion, completed) = sync_channel(0); - self.send_until( - QueuedGenerationLifecycleMessage::Barrier(completion), - deadline, - )?; - let timeout = deadline.saturating_duration_since(Instant::now()); - match completed.recv_timeout(timeout) { - Ok(result) => result, - Err(RecvTimeoutError::Timeout) => { - Err(anyhow::anyhow!("generation lifecycle delivery timed out")) - } - Err(RecvTimeoutError::Disconnected) => Err(anyhow::anyhow!( - "generation lifecycle delivery worker is unavailable" - )), - } - } } /// Optional local-generation observation. @@ -379,17 +285,6 @@ impl GenerationReceiptConfig { self.enqueue(GenerationLifecycleObservation::Committed(commit)); } - /// Commits a token and waits until the lifecycle ingress has delivered it. - /// Local generation uses this before the next proposal lookup so consumers - /// cannot observe a stale canonical position. - pub(crate) fn committed_before_proposal(&self, commit: GenerationCommit) -> Result<()> { - self.ingress - .submit_ordered(GenerationLifecycleObservation::Committed(commit)) - .inspect_err(|_| { - self.submission_failures.fetch_add(1, Ordering::Relaxed); - }) - } - pub(crate) fn abort(&self, abort: GenerationAbort) { self.enqueue(GenerationLifecycleObservation::Aborted(abort)); } @@ -630,7 +525,6 @@ fn duration_us(duration: Duration) -> u64 { #[cfg(test)] mod tests { use std::sync::Mutex; - use std::sync::atomic::AtomicBool; use std::thread; use super::*; @@ -658,8 +552,6 @@ mod tests { struct RecordingSink { receipts: Mutex>, error: Option<&'static str>, - fail_commits: AtomicBool, - commit_delay: Duration, } #[derive(Default)] @@ -682,14 +574,7 @@ mod tests { } fn committed(&self, _commit: &GenerationCommit) -> Result<()> { - if !self.commit_delay.is_zero() { - thread::sleep(self.commit_delay); - } - if self.fail_commits.load(Ordering::Relaxed) { - Err(anyhow::anyhow!("commit failed")) - } else { - Ok(()) - } + Ok(()) } fn abort(&self, _abort: &GenerationAbort) -> Result<()> { @@ -713,44 +598,6 @@ mod tests { panic!("timed out waiting for {expected} generation receipts"); } - fn test_commit() -> GenerationCommit { - GenerationCommit { - request_id: 1, - session_id: 2, - generated_token_count: 1, - token_ids: vec![3].into_boxed_slice(), - } - } - - #[test] - fn queued_barrier_reports_delivery_failure_and_resets_interval() { - let sink = Arc::new(RecordingSink::default()); - sink.fail_commits.store(true, Ordering::Relaxed); - let config = GenerationReceiptConfig::new(sink.clone()); - - assert!(config.committed_before_proposal(test_commit()).is_err()); - assert_eq!(config.delivery_failures(), 2); - - sink.fail_commits.store(false, Ordering::Relaxed); - assert!(config.committed_before_proposal(test_commit()).is_ok()); - assert_eq!(config.delivery_failures(), 2); - } - - #[test] - fn queued_ordered_delivery_times_out_when_sink_stalls() { - let sink = Arc::new(RecordingSink { - commit_delay: GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT.saturating_mul(4), - ..Default::default() - }); - let config = GenerationReceiptConfig::new(sink); - let started = Instant::now(); - - let error = config.committed_before_proposal(test_commit()).unwrap_err(); - - assert!(error.to_string().contains("timed out")); - assert!(started.elapsed() < GENERATION_RECEIPT_ORDERED_DELIVERY_TIMEOUT.saturating_mul(2)); - } - #[test] fn token_digest_is_stable_and_order_sensitive() { let digest = generation_token_id_digest(&[-1, 0, 1, i32::MAX]); @@ -978,7 +825,6 @@ mod tests { let failing_sink = Arc::new(RecordingSink { receipts: Mutex::new(Vec::new()), error: Some("sink failed"), - ..Default::default() }); let failing_config = GenerationReceiptConfig::new(failing_sink); let mut observation = test_observation(0); diff --git a/crates/skippy-server/src/frontend/linear_proposal.rs b/crates/skippy-server/src/frontend/linear_proposal.rs index ee0702f8a6..baa3113a4a 100644 --- a/crates/skippy-server/src/frontend/linear_proposal.rs +++ b/crates/skippy-server/src/frontend/linear_proposal.rs @@ -69,12 +69,11 @@ impl LinearProposal { /// Bounded, target-authoritative state supplied to a proposal source. /// -/// The proposal path deliberately does not pass a full context buffer. A -/// lifecycle observer receives canonical committed token deltas separately; -/// the request path carries only O(1) identity and position data so a source -/// can honor its absolute deadline without copying or hashing an arbitrarily -/// large prompt. -#[derive(Clone, Copy, Debug)] +/// The proposal path deliberately does not pass a full context buffer. The +/// bounded canonical token delta since the previous proposal boundary travels +/// with this query, so a source can update its state synchronously without a +/// separate lifecycle event or copying or hashing an arbitrarily large prompt. +#[derive(Clone, Debug)] #[non_exhaustive] pub struct LinearProposalQuery { /// OpenAI request identity. @@ -88,13 +87,19 @@ pub struct LinearProposalQuery { /// that may already have been committed during prefill. pub prompt_token_count: usize, /// Total canonical tokens committed at this query boundary, including the - /// prompt and every target token previously delivered to the lifecycle - /// observer. + /// prompt and every target token previously delivered to the proposal + /// source. pub committed_token_count: usize, /// Number of target tokens already generated. pub decode_step: usize, /// Maximum proposal width Skippy will accept for this query. pub max_proposal_tokens: usize, + /// Canonical target tokens emitted since the previous proposal boundary. + /// + /// Proposal sources apply this delta before producing the proposal. Keeping + /// the delta on the synchronous query avoids a separate lifecycle event on + /// the local-generation hot path. + pub pending_token_ids: Box<[i32]>, /// Advisory deadline the synchronous source must honor. pub deadline: Instant, } @@ -118,9 +123,16 @@ impl LinearProposalQuery { committed_token_count, decode_step, max_proposal_tokens, + pending_token_ids: Vec::new().into_boxed_slice(), deadline, } } + + #[must_use] + pub fn with_pending_token_ids(mut self, pending_token_ids: Box<[i32]>) -> Self { + self.pending_token_ids = pending_token_ids; + self + } } /// Why Skippy rejected a source decision without producing a receipt. @@ -159,6 +171,10 @@ pub enum LinearProposalDisposition { #[derive(Clone, Debug, PartialEq)] #[non_exhaustive] pub struct LinearProposalReceipt { + /// OpenAI request identity for the proposal's generation. + pub request_id: u64, + /// OpenAI session identity for the proposal's generation. + pub session_id: u64, /// Source identity copied from the proposal. pub decision_id: OpaqueProposalDecisionId, /// Target-authoritative verification outcome. @@ -356,12 +372,17 @@ pub(crate) struct QueriedLinearProposal { } pub(crate) enum LinearProposalQueryOutcome { + /// No proposal command was submitted, so any pending token delta remains + /// owned by the caller. + Skipped, NoProposal, - DeadlineExceeded { proposal_elapsed_us: u64 }, + DeadlineExceeded { + proposal_elapsed_us: u64, + }, Ready(QueriedLinearProposal), } -#[derive(Clone, Copy)] +#[derive(Clone)] pub(crate) struct LinearProposalQueryParams { pub(crate) request_id: u64, pub(crate) session_id: u64, @@ -370,6 +391,7 @@ pub(crate) struct LinearProposalQueryParams { pub(crate) committed_token_count: usize, pub(crate) remaining_new_tokens: usize, pub(crate) runtime_max_proposal_tokens: usize, + pub(crate) pending_token_ids: Box<[i32]>, } pub(crate) fn query_linear_proposal( @@ -390,7 +412,7 @@ pub(crate) fn query_linear_proposal( .min(params.runtime_max_proposal_tokens) .min(config.max_proposal_tokens()); if max_proposal_tokens == 0 { - return Ok(LinearProposalQueryOutcome::NoProposal); + return Ok(LinearProposalQueryOutcome::Skipped); } let operation_started = Instant::now(); let deadline = operation_started @@ -399,15 +421,18 @@ pub(crate) fn query_linear_proposal( let proposal_started = Instant::now(); let proposal = config .source() - .propose(LinearProposalQuery::new( - params.request_id, - params.session_id, - params.prompt_token_count, - params.committed_token_count, - params.decode_step, - max_proposal_tokens, - deadline, - )) + .propose( + LinearProposalQuery::new( + params.request_id, + params.session_id, + params.prompt_token_count, + params.committed_token_count, + params.decode_step, + max_proposal_tokens, + deadline, + ) + .with_pending_token_ids(params.pending_token_ids), + ) .map_err(openai_backend_error)?; let proposal_elapsed_us = elapsed_us(proposal_started); let Some(proposal) = proposal else { @@ -534,6 +559,7 @@ mod tests { committed_token_count, remaining_new_tokens, runtime_max_proposal_tokens, + pending_token_ids: Vec::new().into_boxed_slice(), } } @@ -544,12 +570,17 @@ mod tests { discard_fails: Mutex, report_fails: Mutex, queries: Mutex>, + pending_token_batches: Mutex>>, receipts: Mutex>, discards: Mutex>, } impl LinearProposalIngress for FakeIngress { fn propose(&self, query: LinearProposalQuery) -> Result> { + self.pending_token_batches + .lock() + .unwrap() + .push(query.pending_token_ids.clone()); self.queries.lock().unwrap().push(RecordedQuery { request_id: query.request_id, session_id: query.session_id, @@ -673,6 +704,8 @@ mod tests { let config = LinearProposalIngressConfig::new(source.clone(), Duration::from_secs(1), 4).unwrap(); let receipt = LinearProposalReceipt { + request_id: 1, + session_id: 2, decision_id: OpaqueProposalDecisionId::new(vec![90]).unwrap(), disposition: LinearProposalDisposition::FullAccept, proposal_token_count: 1, @@ -775,6 +808,39 @@ mod tests { assert!(source.discards.lock().unwrap().is_empty()); } + #[test] + fn query_forwards_pending_tokens_to_the_proposal_source() { + let source = Arc::new(FakeIngress::default()); + let config = + LinearProposalIngressConfig::new(source.clone(), Duration::from_secs(1), 4).unwrap(); + let mut params = query_params(7, 8, 2, 1, 3, 5, 4); + params.pending_token_ids = vec![31, 32].into_boxed_slice(); + + assert!(matches!( + query_linear_proposal(&config, params).unwrap(), + LinearProposalQueryOutcome::NoProposal + )); + assert_eq!( + source.pending_token_batches.lock().unwrap().as_slice(), + &[vec![31, 32].into_boxed_slice()] + ); + } + + #[test] + fn query_does_not_accept_pending_tokens_without_proposal_capacity() { + let source = Arc::new(FakeIngress::default()); + let config = + LinearProposalIngressConfig::new(source.clone(), Duration::from_secs(1), 4).unwrap(); + let mut params = query_params(7, 8, 2, 1, 3, 1, 4); + params.pending_token_ids = vec![31].into_boxed_slice(); + + assert!(matches!( + query_linear_proposal(&config, params).unwrap(), + LinearProposalQueryOutcome::Skipped + )); + assert!(source.pending_token_batches.lock().unwrap().is_empty()); + } + #[test] fn query_rejects_an_inconsistent_prompt_decode_boundary_before_ingress() { let source = Arc::new(FakeIngress::default()); @@ -860,6 +926,8 @@ mod tests { let source = FakeIngress::default(); let id = OpaqueProposalDecisionId::new(vec![7, 8, 9]).unwrap(); let receipt = LinearProposalReceipt { + request_id: 1, + session_id: 2, decision_id: id.clone(), disposition: LinearProposalDisposition::FirstMismatch, proposal_token_count: 4, @@ -910,6 +978,8 @@ mod tests { fn receipt_telemetry_excludes_source_ids_tokens_and_error_text() { let secret = "private-decision-/Users/nick/prompt.txt"; let receipt = LinearProposalReceipt { + request_id: 1, + session_id: 2, decision_id: OpaqueProposalDecisionId::new(secret.as_bytes().to_vec()).unwrap(), disposition: LinearProposalDisposition::FullAccept, proposal_token_count: 1, diff --git a/crates/skippy-server/src/frontend/linear_proposal/execution.rs b/crates/skippy-server/src/frontend/linear_proposal/execution.rs index fccc66bd00..64bc28ae6d 100644 --- a/crates/skippy-server/src/frontend/linear_proposal/execution.rs +++ b/crates/skippy-server/src/frontend/linear_proposal/execution.rs @@ -26,6 +26,8 @@ struct LinearProposalExecution { #[derive(Clone, Copy)] pub(crate) struct LinearProposalExecutionParams<'a> { + pub(crate) request_id: u64, + pub(crate) request_session_id: u64, pub(crate) session_id: &'a str, pub(crate) current: i32, pub(crate) base_position: u64, @@ -104,6 +106,8 @@ impl StageOpenAiBackend { }); let total_elapsed_us = elapsed_us(queried.operation_started); Ok(Some(LinearProposalReceipt { + request_id: params.request_id, + session_id: params.request_session_id, decision_id: queried.proposal.decision_id, disposition, proposal_token_count, diff --git a/crates/skippy-server/src/frontend/local_generation/linear_decode.rs b/crates/skippy-server/src/frontend/local_generation/linear_decode.rs index 8e21123d05..7b22188b41 100644 --- a/crates/skippy-server/src/frontend/local_generation/linear_decode.rs +++ b/crates/skippy-server/src/frontend/local_generation/linear_decode.rs @@ -63,26 +63,51 @@ impl StageOpenAiBackend { .saturating_add(state.decoded_tokens), remaining_new_tokens, runtime_max_proposal_tokens: state.linear_proposal_max_tokens, + pending_token_ids: state + .pending_linear_proposal_tokens + .clone() + .into_boxed_slice(), }, - )? { - LinearProposalQueryOutcome::NoProposal => None, - LinearProposalQueryOutcome::DeadlineExceeded { - proposal_elapsed_us, - } => { - let mut attrs = BTreeMap::new(); - attrs.insert( - "llama_stage.linear_proposal.discard_reason".to_string(), - json!("deadline_exceeded"), - ); - attrs.insert( - "llama_stage.linear_proposal.proposal_us".to_string(), - json!(proposal_elapsed_us), + ) { + Ok(LinearProposalQueryOutcome::Skipped) => { + return Ok(LinearProposalProgress::NotUsed); + } + Ok(outcome) => { + state.pending_linear_proposal_tokens.clear(); + match outcome { + LinearProposalQueryOutcome::Skipped => unreachable!("handled above"), + LinearProposalQueryOutcome::NoProposal => None, + LinearProposalQueryOutcome::DeadlineExceeded { + proposal_elapsed_us, + } => { + let mut attrs = BTreeMap::new(); + attrs.insert( + "llama_stage.linear_proposal.discard_reason".to_string(), + json!("deadline_exceeded"), + ); + attrs.insert( + "llama_stage.linear_proposal.proposal_us".to_string(), + json!(proposal_elapsed_us), + ); + self.telemetry + .emit("stage.openai_linear_proposal_late", attrs); + None + } + LinearProposalQueryOutcome::Ready(queried) => Some(queried), + } + } + Err(error) => { + state.linear_context_tokens = None; + state.linear_proposal_max_tokens = 0; + self.telemetry.emit( + "stage.openai_linear_proposal_disabled", + BTreeMap::from([( + "llama_stage.linear_proposal.error".to_string(), + json!(error.to_string()), + )]), ); - self.telemetry - .emit("stage.openai_linear_proposal_late", attrs); - None + return Ok(LinearProposalProgress::NotUsed); } - LinearProposalQueryOutcome::Ready(queried) => Some(queried), }; if request .cancellation @@ -97,6 +122,8 @@ impl StageOpenAiBackend { let receipt = execute_linear_proposal_with_terminal_discard(config, &decision_id, || { self.execute_local_linear_proposal( LinearProposalExecutionParams { + request_id: request.ids.request_id, + request_session_id: request.ids.session_id, session_id, current: state.current, base_position, diff --git a/crates/skippy-server/src/frontend/local_generation/tests.rs b/crates/skippy-server/src/frontend/local_generation/tests.rs index 03d6a9c776..b0817e2f43 100644 --- a/crates/skippy-server/src/frontend/local_generation/tests.rs +++ b/crates/skippy-server/src/frontend/local_generation/tests.rs @@ -17,7 +17,6 @@ use crate::frontend::generation::{ LocalGeneration, OpenAiBackendMode, OpenAiCacheHints, OpenAiGenerationIds, StageOpenAiBackend, TokenControl, }; -use crate::frontend::local_generation::token_generation::commit_local_generation_token; use crate::frontend::local_generation::{ native_mtp_dispatch_counts_for_test, prompt_fits_single_prefill_sample, }; @@ -68,16 +67,6 @@ fn wait_for_receipts(sink: &RecordingReceiptSink, expected: usize) { panic!("timed out waiting for {expected} generation receipts"); } -fn wait_for_commits(sink: &RecordingReceiptSink, expected: usize) { - for _ in 0..100 { - if sink.commits.lock().unwrap().len() >= expected { - return; - } - thread::sleep(Duration::from_millis(1)); - } - panic!("timed out waiting for {expected} generation commits"); -} - #[test] fn single_prefill_sample_requires_prompt_to_fit_session_batch() { assert!(!prompt_fits_single_prefill_sample(0, 2048)); @@ -86,27 +75,6 @@ fn single_prefill_sample_requires_prompt_to_fit_session_batch() { assert!(!prompt_fits_single_prefill_sample(2049, 2048)); } -#[test] -fn local_generation_commit_helper_preserves_ordered_counts() { - let sink = Arc::new(RecordingReceiptSink::default()); - let config = GenerationReceiptConfig::new(sink.clone()); - let mut generated_token_count = 0; - - commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 33).unwrap(); - commit_local_generation_token(Some(&config), 11, 22, &mut generated_token_count, 44).unwrap(); - - let commits = sink.commits.lock().unwrap(); - assert_eq!(commits.len(), 2); - assert_eq!(commits[0].request_id, 11); - assert_eq!(commits[0].session_id, 22); - assert_eq!(commits[0].generated_token_count, 1); - assert_eq!(commits[0].token_ids.as_ref(), [33]); - assert_eq!(commits[1].request_id, 11); - assert_eq!(commits[1].session_id, 22); - assert_eq!(commits[1].generated_token_count, 2); - assert_eq!(commits[1].token_ids.as_ref(), [44]); -} - #[test] fn local_native_mtp_decode_uses_non_frame_runtime_api() { let (sampled_calls, frame_calls) = native_mtp_dispatch_counts_for_test(); @@ -222,16 +190,9 @@ fn local_generation_eventually_delivers_receipts_and_cleanup_survives_sink_error }, )?; - wait_for_commits(&sink, emitted.len()); wait_for_receipts(&sink, 1); let commits = sink.commits.lock().unwrap(); - assert_eq!(commits.len(), emitted.len()); - for (index, (commit, token_id)) in commits.iter().zip(&emitted).enumerate() { - assert_eq!(commit.request_id, ids.request_id); - assert_eq!(commit.session_id, ids.session_id); - assert_eq!(commit.generated_token_count, index + 1); - assert_eq!(commit.token_ids.as_ref(), [*token_id]); - } + assert!(commits.is_empty()); drop(commits); let receipts = sink.receipts.lock().unwrap(); assert_eq!(receipts.len(), 1); diff --git a/crates/skippy-server/src/frontend/local_generation/token_generation.rs b/crates/skippy-server/src/frontend/local_generation/token_generation.rs index 0b5017bbf3..ff55faf608 100644 --- a/crates/skippy-server/src/frontend/local_generation/token_generation.rs +++ b/crates/skippy-server/src/frontend/local_generation/token_generation.rs @@ -7,9 +7,7 @@ use crate::frontend::generation::OpenAiGenerationIds; use crate::frontend::generation::PhaseTimer; use crate::frontend::generation::StageOpenAiBackend; use crate::frontend::generation::TokenControl; -use crate::frontend::generation_receipt::{ - GenerationCommit, GenerationStart, complete_generation_before_cleanup, -}; +use crate::frontend::generation_receipt::{GenerationStart, complete_generation_before_cleanup}; use crate::frontend::linear_proposal::greedy_linear_proposal_admitted; use crate::frontend::util::openai_backend_error; use crate::frontend::util::saturating_u32; @@ -31,27 +29,6 @@ use std::time::Duration; use super::{LocalGenerationReceiptFinalization, prompt_fits_single_prefill_sample}; -pub(super) fn commit_local_generation_token( - config: Option<&crate::frontend::GenerationReceiptConfig>, - request_id: u64, - session_id: u64, - generated_token_count: &mut usize, - token_id: i32, -) -> OpenAiResult<()> { - let Some(config) = config else { - return Ok(()); - }; - *generated_token_count = generated_token_count.saturating_add(1); - config - .committed_before_proposal(GenerationCommit { - request_id, - session_id, - generated_token_count: *generated_token_count, - token_ids: vec![token_id].into_boxed_slice(), - }) - .map_err(openai_backend_error) -} - struct PromptPrefillResult { prompt_prefill_sample: Option, chat_sampling_configured: bool, @@ -83,6 +60,7 @@ pub(super) struct DecodeState { pub(super) generation_hooks_active: bool, pub(super) linear_proposal_max_tokens: usize, pub(super) linear_context_tokens: Option>, + pub(super) pending_linear_proposal_tokens: Vec, pub(super) emit_token_debug: bool, pub(super) native_mtp_options: NativeMtpDecodeOptions, pub(super) native_mtp: NativeMtpVerifier, @@ -132,20 +110,12 @@ impl StageOpenAiBackend { let mut receipt_cancelled = false; let mut receipt_model_generation_elapsed = None; let mut cache_stats = GenerationCacheStats::default(); - let mut lifecycle_committed_token_count = 0usize; let mut emit_token = |token_id| { if let Some(observation) = receipt_observation.as_ref() && let Some(observation) = observation.borrow_mut().as_mut() { observation.record_token(token_id, request.ids.request_started_at.elapsed()); } - commit_local_generation_token( - self.generation_receipt.as_ref(), - receipt_request_id, - receipt_session_id, - &mut lifecycle_committed_token_count, - token_id, - )?; let control = on_token(token_id)?; if control == TokenControl::Stop && let Some(observation) = receipt_observation.as_ref() @@ -778,6 +748,7 @@ impl StageOpenAiBackend { .last() .expect("checked non-empty prompt"); let mut stopped = false; + let mut pending_linear_proposal_tokens = Vec::new(); if let Some(predicted) = prompt_prefill_sample { if request .cancellation @@ -787,6 +758,7 @@ impl StageOpenAiBackend { } current = predicted; decoded_tokens += 1; + pending_linear_proposal_tokens.push(current); stopped = emit_token(current)? == TokenControl::Stop; } let hook_request = request.hook_request.take(); @@ -816,6 +788,9 @@ impl StageOpenAiBackend { } tokens }); + if linear_proposal_max_tokens == 0 { + pending_linear_proposal_tokens.clear(); + } Ok(DecodeState { decoded_tokens, current, @@ -832,6 +807,7 @@ impl StageOpenAiBackend { generation_hooks_active, linear_proposal_max_tokens, linear_context_tokens, + pending_linear_proposal_tokens, emit_token_debug: self.telemetry.is_debug_enabled(), native_mtp_options: NativeMtpDecodeOptions::from_config(request.speculative), native_mtp: NativeMtpVerifier::default(), @@ -865,9 +841,11 @@ impl StageOpenAiBackend { LinearProposalProgress::Stop => break, LinearProposalProgress::NotUsed => {} } - if self.decode_one_token(request, session_id, &mut state, emit_token)? - == TokenControl::Stop - { + let control = self.decode_one_token(request, session_id, &mut state, emit_token)?; + if state.linear_proposal_max_tokens > 0 { + state.pending_linear_proposal_tokens.push(state.current); + } + if control == TokenControl::Stop { break; } }