diff --git a/crates/mesh-llm-config/src/model.rs b/crates/mesh-llm-config/src/model.rs index b5a326d2e..6a02709d6 100644 --- a/crates/mesh-llm-config/src/model.rs +++ b/crates/mesh-llm-config/src/model.rs @@ -560,6 +560,7 @@ pub struct SpeculativeConfig { pub ngram_min: Option, pub ngram_max: Option, pub ngram_proposer: Option, + pub ngram_fallback: Option, pub ngram_max_proposal_tokens: Option, pub extension_initial_tokens: Option, pub extension_max_tokens: Option, @@ -611,6 +612,7 @@ impl SpeculativeConfig { ngram_min: pick!(ngram_min), ngram_max: pick!(ngram_max), ngram_proposer: pick!(ngram_proposer), + ngram_fallback: pick!(ngram_fallback), ngram_max_proposal_tokens: pick!(ngram_max_proposal_tokens), extension_initial_tokens: pick!(extension_initial_tokens), extension_max_tokens: pick!(extension_max_tokens), @@ -680,6 +682,8 @@ struct SpeculativeConfigRaw { #[serde(default)] ngram_proposer: Option, #[serde(default)] + ngram_fallback: Option, + #[serde(default)] ngram_max_proposal_tokens: Option, #[serde(default)] extension_initial_tokens: Option, @@ -736,6 +740,7 @@ impl<'de> Deserialize<'de> for SpeculativeConfig { ngram_min: raw.ngram_min, ngram_max: raw.ngram_max, ngram_proposer: raw.ngram_proposer, + ngram_fallback: raw.ngram_fallback, ngram_max_proposal_tokens: raw.ngram_max_proposal_tokens, extension_initial_tokens: raw.extension_initial_tokens, extension_max_tokens: raw.extension_max_tokens, @@ -759,7 +764,7 @@ impl Serialize for SpeculativeConfig { { use serde::ser::SerializeMap; - let mut map = serializer.serialize_map(Some(32))?; + let mut map = serializer.serialize_map(Some(33))?; map.serialize_entry("strategy", &self.strategy)?; map.serialize_entry("mode", &self.mode)?; if self.legacy_draft_model_path_used { @@ -788,6 +793,7 @@ impl Serialize for SpeculativeConfig { map.serialize_entry("ngram_min", &self.ngram_min)?; map.serialize_entry("ngram_max", &self.ngram_max)?; map.serialize_entry("ngram_proposer", &self.ngram_proposer)?; + map.serialize_entry("ngram_fallback", &self.ngram_fallback)?; map.serialize_entry("ngram_max_proposal_tokens", &self.ngram_max_proposal_tokens)?; map.serialize_entry("extension_initial_tokens", &self.extension_initial_tokens)?; map.serialize_entry("extension_max_tokens", &self.extension_max_tokens)?; diff --git a/crates/mesh-llm-config/src/model/built_in_schema.rs b/crates/mesh-llm-config/src/model/built_in_schema.rs index 614874b08..91b457add 100644 --- a/crates/mesh-llm-config/src/model/built_in_schema.rs +++ b/crates/mesh-llm-config/src/model/built_in_schema.rs @@ -670,6 +670,7 @@ fn speculative_settings(prefix: &str) -> Vec { &format!("{prefix}.ngram_proposer"), string_enum(["simple", "cache", "suffix"]), ), + basic_setting(&format!("{prefix}.ngram_fallback"), string_enum(["simple"])), basic_setting( &format!("{prefix}.ngram_max_proposal_tokens"), ConfigValueSchema::Integer, diff --git a/crates/mesh-llm-config/src/model/built_in_schema/control_behavior/speculative.rs b/crates/mesh-llm-config/src/model/built_in_schema/control_behavior/speculative.rs index e05b28fae..ae19995a7 100644 --- a/crates/mesh-llm-config/src/model/built_in_schema/control_behavior/speculative.rs +++ b/crates/mesh-llm-config/src/model/built_in_schema/control_behavior/speculative.rs @@ -68,7 +68,7 @@ pub(super) fn apply_speculative_behavior( push_range_constraint(setting, Some(format!("{prefix}.ngram_min")), None); push_mode_dependency(setting, prefix, "ngram", suffix); } - "ngram_proposer" | "ngram_max_proposal_tokens" => { + "ngram_proposer" | "ngram_fallback" | "ngram_max_proposal_tokens" => { push_mode_dependency(setting, prefix, "ngram", suffix); } "extension_initial_tokens" diff --git a/crates/mesh-llm-config/src/model_validation.rs b/crates/mesh-llm-config/src/model_validation.rs index adb22138a..5cd539f75 100644 --- a/crates/mesh-llm-config/src/model_validation.rs +++ b/crates/mesh-llm-config/src/model_validation.rs @@ -547,6 +547,19 @@ fn validate_speculative_proposer_controls( &["simple", "cache", "suffix"], &format!("{base_path}.ngram_proposer"), )?; + validate_optional_enum( + config.ngram_fallback.as_deref(), + &["simple"], + &format!("{base_path}.ngram_fallback"), + )?; + let simple_selected = config.ngram_proposer.as_deref() == Some("simple") + || config.strategy.as_deref() == Some("ngram-simple"); + if config.ngram_fallback.is_some() && simple_selected { + return Err(validation_diagnostic( + &format!("{base_path}.ngram_fallback"), + format!("{base_path}.ngram_fallback requires a cache or suffix primary proposer"), + )); + } let suffix_selected = config.ngram_proposer.as_deref() == Some("suffix") || config.strategy.as_deref() == Some("ngram-suffix"); if suffix_selected { diff --git a/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs b/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs index 4a5f5838c..946db7d32 100644 --- a/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs +++ b/crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs @@ -237,6 +237,14 @@ fn resolve_decode_config(input: DecodeResolutionInput<'_>) -> Result) -> Result true, + None => existing.is_some_and(|ngram| ngram.fallback_simple), + Some(_) => unreachable!("validated by mesh configuration"), + }; config.ngram = Some(NgramProposalConfig { kind, min_ngram: min_ngram as usize, max_ngram: max_ngram as usize, max_proposal_tokens, + fallback_simple, }); if config.effective_strategy == "disabled" { config.effective_strategy = ngram_effective_strategy(kind).to_string(); @@ -569,6 +583,7 @@ fn ngram_proposer_config( min_ngram: min_ngram as usize, max_ngram: max_ngram as usize, max_proposal_tokens: max_proposal_tokens as usize, + fallback_simple: false, }) } diff --git a/crates/mesh-llm-host-runtime/tests/fixtures/config_schema_defaults_ui_reference.json b/crates/mesh-llm-host-runtime/tests/fixtures/config_schema_defaults_ui_reference.json index af06512f8..bd3e6d0a3 100644 --- a/crates/mesh-llm-host-runtime/tests/fixtures/config_schema_defaults_ui_reference.json +++ b/crates/mesh-llm-host-runtime/tests/fixtures/config_schema_defaults_ui_reference.json @@ -868,6 +868,13 @@ "kind": "built_in" } }, + { + "canonical_path": "defaults.speculative.ngram_fallback", + "support": "supported", + "source": { + "kind": "built_in" + } + }, { "canonical_path": "defaults.speculative.ngram_max", "support": "supported", diff --git a/crates/skippy-server/src/binary_transport/binary_messaging.rs b/crates/skippy-server/src/binary_transport/binary_messaging.rs index 5e9090d2e..105b53ea7 100644 --- a/crates/skippy-server/src/binary_transport/binary_messaging.rs +++ b/crates/skippy-server/src/binary_transport/binary_messaging.rs @@ -304,6 +304,7 @@ mod tests { min_ngram: 3, max_ngram: 4, max_proposal_tokens: 64, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() } diff --git a/crates/skippy-server/src/binary_transport/options.rs b/crates/skippy-server/src/binary_transport/options.rs index 6783b757c..7707451e6 100644 --- a/crates/skippy-server/src/binary_transport/options.rs +++ b/crates/skippy-server/src/binary_transport/options.rs @@ -135,6 +135,7 @@ fn legacy_speculative_config(args: &ServeBinaryArgs) -> SpeculativeDecodeConfig min_ngram: args.openai_ngram_min, max_ngram: args.openai_ngram_max, max_proposal_tokens: args.openai_ngram_max, + fallback_simple: false, }); } config @@ -218,6 +219,7 @@ mod tests { min_ngram: 2, max_ngram: 4, max_proposal_tokens: 6, + fallback_simple: false, }), extension: Some(NgramExtensionConfig { initial_tokens: 2, diff --git a/crates/skippy-server/src/frontend/native_mtp/decode.rs b/crates/skippy-server/src/frontend/native_mtp/decode.rs index 86f9f3de1..7d0f8f367 100644 --- a/crates/skippy-server/src/frontend/native_mtp/decode.rs +++ b/crates/skippy-server/src/frontend/native_mtp/decode.rs @@ -716,6 +716,7 @@ mod tests { min_ngram: 2, max_ngram: 4, max_proposal_tokens: 7, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() }; diff --git a/crates/skippy-server/src/frontend/speculative.rs b/crates/skippy-server/src/frontend/speculative.rs index 2776a1af2..29a10fc53 100644 --- a/crates/skippy-server/src/frontend/speculative.rs +++ b/crates/skippy-server/src/frontend/speculative.rs @@ -73,6 +73,9 @@ pub struct NgramProposalConfig { pub min_ngram: usize, pub max_ngram: usize, pub max_proposal_tokens: usize, + /// Falls back to the simple proposer when a history proposer drafts nothing. + #[serde(default)] + pub fallback_simple: bool, } /// Bounds for extending an MTP prefix with an N-gram tail. @@ -153,6 +156,12 @@ impl SpeculativeDecodeConfig { "suffix N-gram proposer requires {SUFFIX_MIN_SEED_LEN} <= min_ngram <= max_ngram <= {SUFFIX_NGRAM_MAX_WINDOW}" ); } + if let Some(ngram) = &self.ngram + && ngram.fallback_simple + && ngram.kind == NgramProposerKind::Simple + { + bail!("simple N-gram fallback requires a cache or suffix primary proposer"); + } if self.extension.is_some() && (!self.native_mtp.enabled || self.ngram.is_none()) { bail!("N-gram extension requires both native MTP and an N-gram proposer"); } @@ -237,6 +246,7 @@ mod standalone_speculative_config_tests { min_ngram: 2, max_ngram: 4, max_proposal_tokens: 6, + fallback_simple: false, }), extension: Some(NgramExtensionConfig { initial_tokens: 2, @@ -261,6 +271,7 @@ mod standalone_speculative_config_tests { min_ngram: 2, max_ngram: skippy_runtime::NGRAM_CACHE_MAX_NGRAM + 1, max_proposal_tokens: 6, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() }; @@ -284,6 +295,7 @@ mod standalone_speculative_config_tests { min_ngram: 5, max_ngram: 32, max_proposal_tokens: 48, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() }; @@ -303,6 +315,7 @@ mod standalone_speculative_config_tests { min_ngram: 3, max_ngram: SUFFIX_NGRAM_MAX_WINDOW + 1, max_proposal_tokens: 6, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() }; @@ -312,6 +325,30 @@ mod standalone_speculative_config_tests { assert!(error.to_string().contains("min_ngram <= max_ngram <= 64")); } + #[test] + fn standalone_speculative_config_rejects_simple_fallback_on_simple_proposer() { + let config = SpeculativeDecodeConfig { + ngram: Some(NgramProposalConfig { + kind: NgramProposerKind::Simple, + min_ngram: 2, + max_ngram: 4, + max_proposal_tokens: 6, + fallback_simple: true, + }), + ..SpeculativeDecodeConfig::default() + }; + + let error = config + .validate() + .expect_err("fallback needs history proposer"); + + assert!( + error + .to_string() + .contains("requires a cache or suffix primary proposer") + ); + } + #[test] fn standalone_speculative_config_rejects_suffix_matches_below_seed_length() { let config = SpeculativeDecodeConfig { @@ -320,6 +357,7 @@ mod standalone_speculative_config_tests { min_ngram: 2, max_ngram: 16, max_proposal_tokens: 4, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() }; diff --git a/crates/skippy-server/src/frontend/speculative/standalone.rs b/crates/skippy-server/src/frontend/speculative/standalone.rs index 500aaf10f..10078f23b 100644 --- a/crates/skippy-server/src/frontend/speculative/standalone.rs +++ b/crates/skippy-server/src/frontend/speculative/standalone.rs @@ -20,8 +20,13 @@ pub(in crate::frontend) fn standalone_ngram_proposal_limit( .map_or(0, |ngram| ngram.max_proposal_tokens) } +/// Minimum match length the simple fallback proposer scans for. The history +/// proposers use longer configured bounds that the simple proposer cannot. +const SIMPLE_FALLBACK_MIN_NGRAM: usize = 2; + /// Runs the configured standalone N-gram proposer (simple, cache, or suffix) -/// over committed history and returns its draft. +/// over committed history and returns its draft. When enabled, a history +/// proposer miss falls back to the simple proposer. pub(in crate::frontend) fn propose_configured_ngram_tokens( config: &SpeculativeDecodeConfig, history_proposer: &mut Option, @@ -44,6 +49,16 @@ pub(in crate::frontend) fn propose_configured_ngram_tokens( .ok_or_else(|| OpenAiError::backend("configured history N-gram proposer is missing"))? .propose(committed_history, &[], proposal_limit)?, }; + if tokens.is_empty() && ngram.fallback_simple && ngram.kind != NgramProposerKind::Simple { + let fallback = + propose_ngram_tokens(committed_history, SIMPLE_FALLBACK_MIN_NGRAM, proposal_limit)?; + if !fallback.is_empty() { + return Ok(ConfiguredNgramProposal { + tokens: fallback, + source: "simple", + }); + } + } Ok(ConfiguredNgramProposal { tokens, source: ngram.kind.as_str(), @@ -67,11 +82,22 @@ mod tests { min_ngram, max_ngram, max_proposal_tokens: 3, + fallback_simple: false, }), ..SpeculativeDecodeConfig::default() } } + fn config_with_fallback( + kind: NgramProposerKind, + min_ngram: usize, + max_ngram: usize, + ) -> SpeculativeDecodeConfig { + let mut config = config(kind, min_ngram, max_ngram); + config.ngram.as_mut().unwrap().fallback_simple = true; + config + } + fn propose(config: &SpeculativeDecodeConfig, history: &[i32]) -> ConfiguredNgramProposal { let mut proposer = HistoryNgramProposer::from_config(config).unwrap(); propose_configured_ngram_tokens(config, &mut proposer, history, 8).unwrap() @@ -122,4 +148,44 @@ mod tests { assert_eq!(proposal.source, "suffix"); assert_eq!(proposal.tokens, vec![4, 5, 1]); } + + #[test] + fn simple_fallback_fires_on_a_history_proposer_miss() { + let proposal = propose( + &config_with_fallback(NgramProposerKind::Suffix, 3, 8), + &[5, 6, 1, 2, 9, 7, 1, 2], + ); + assert_eq!(proposal.source, "simple"); + assert_eq!(proposal.tokens, vec![9, 7, 1]); + } + + #[test] + fn simple_fallback_does_not_replace_a_history_proposer_hit() { + let proposal = propose( + &config_with_fallback(NgramProposerKind::Suffix, 3, 8), + &[1, 2, 3, 4, 5, 1, 2, 3], + ); + assert_eq!(proposal.source, "suffix"); + assert_eq!(proposal.tokens, vec![4, 5, 1]); + } + + #[test] + fn simple_fallback_stays_off_by_default() { + let proposal = propose( + &config(NgramProposerKind::Suffix, 3, 8), + &[5, 6, 1, 2, 9, 1, 2], + ); + assert_eq!(proposal.source, "suffix"); + assert!(proposal.tokens.is_empty()); + } + + #[test] + fn simple_fallback_miss_reports_the_primary_source() { + let proposal = propose( + &config_with_fallback(NgramProposerKind::Cache, 2, 4), + &[1, 2, 3, 4], + ); + assert_eq!(proposal.source, "cache"); + assert!(proposal.tokens.is_empty()); + } } diff --git a/docs/USAGE.md b/docs/USAGE.md index 0402babeb..942167fe9 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -349,6 +349,7 @@ spec_default = "auto" # bool or "auto" # `cache` is request-local and requires ngram_max <= 4; `suffix` is a pure-Rust # longest-suffix (prompt-lookup) matcher allowing ngram_max <= 64. # ngram_proposer = "cache" # simple | cache | suffix +# ngram_fallback = "simple" # retry cache/suffix misses with the simple proposer # ngram_min = 2 # ngram_max = 4 # ngram_max_proposal_tokens = 6 # output budget, separate from ngram_max @@ -784,7 +785,11 @@ verify_window_pipeline_depth = 2 Suffix can also run without MTP by setting `strategy = "ngram-suffix"` and omitting the extension controls. Layer packages may declare `ngram-suffix` as -a request-local proposer and standalone strategy. See +a request-local proposer and standalone strategy. Because `cache` and `suffix` +stay silent on a miss, `ngram_fallback = "simple"` retries the miss with the +llama.cpp simple proposer (2-token minimum match) so some draft is always in +flight. This targets high-latency links where a mediocre draft beats no draft; +on a local network it is roughly neutral. See [Suffix N-gram Proposer](skippy/SUFFIX_NGRAM_PROPOSER.md) for the lookup contract, telemetry, and benchmark requirements. diff --git a/docs/skippy/CONFIGURATION.md b/docs/skippy/CONFIGURATION.md index ec14c9fdf..ec4b64085 100644 --- a/docs/skippy/CONFIGURATION.md +++ b/docs/skippy/CONFIGURATION.md @@ -170,6 +170,7 @@ missing-key error. | 5.5 | Draft KV cache type | `speculative.draft_cache_type_k
speculative.draft_cache_type_v` | P1 | `plugin/config.rs` | Schema-reserved only; resolver fail-closed today | single-stage, staged | restart/reload only | runtime draft defaults | enum values are currently rejected by the resolver and are not executable today | `#speculative-decoding` | `.sisyphus/evidence/task-1-setting-matrix.txt`; negative: `.sisyphus/evidence/task-1-setting-matrix-error.txt` | draft-model KV cache dtype overrides; reserve until an executable draft-runtime path exists | | 5.5 | N-gram match range | `speculative.ngram_min
speculative.ngram_max` | P1 | `resolver/speculative.rs` | `NgramProposalConfig` | staged | restart/reload only | package proposer bounds, otherwise required for a direct N-gram plan | both values are required for a direct N-gram proposer and must satisfy `0 < min <= max`; request-local `cache` requires `max <= 4`; `suffix` requires `3 <= min <= max <= 64` | `#speculative-decoding` | resolver/config validation tests | Match range controls lookup context, not the number of proposed tokens. | | 5.5 | N-gram proposer kind | `speculative.ngram_proposer` | P1 | `resolver/speculative.rs` | `NgramProposalConfig.kind` | staged | restart/reload only | package strategy kind, otherwise `simple` | `simple`, `cache`, or `suffix` | `#speculative-decoding` | resolver/config validation tests | `simple` searches accepted history; `cache` owns request-local target-committed history and can extend an MTP prefix; `suffix` is a pure-Rust longest-suffix (prompt-lookup) matcher allowing `ngram_max <= 64` for long verbatim drafts on repetitive contexts. | +| 5.5 | N-gram simple fallback | `speculative.ngram_fallback` | P1 | `resolver/speculative.rs` | `NgramProposalConfig.fallback_simple` | staged | restart/reload only | disabled | `simple`; requires a `cache` or `suffix` primary proposer | `#speculative-decoding` | resolver/config validation tests | Retries a history-proposer miss with the llama.cpp simple proposer (2-token minimum match) so a draft is always in flight; aimed at high-latency links, roughly neutral on LAN. | | 5.5 | N-gram output budget | `speculative.ngram_max_proposal_tokens` | P1 | `resolver/speculative.rs` | `NgramProposalConfig.max_proposal_tokens` | staged | restart/reload only | package value, otherwise N-gram maximum | positive integer | `#speculative-decoding` | resolver/config validation tests | Independent from the N-gram match range. | | 5.5 | MTP N-gram extension policy | `speculative.extension_initial_tokens
speculative.extension_max_tokens
speculative.extension_tail_backoff_proposals` | P1 | `resolver/speculative.rs` | `NgramExtensionConfig` | staged | restart/reload only | package extension policy or bounded runtime defaults | requires both native MTP and an N-gram proposer; config must satisfy the frontend plan validation | `#speculative-decoding` | resolver/config validation tests | Controls the adaptive suffix only. MTP and N-gram tokens remain one target-verified candidate. | | 5.5 | Native MTP recovery | `speculative.native_mtp_reject_cooldown_tokens
speculative.native_mtp_suppress_cooldown_drafts
speculative.native_mtp_suppress_cooldown_draft_limit` | P2 | `resolver/speculative.rs` | `NativeMtpProposalConfig` | staged | restart/reload only | runtime defaults | non-negative integer bounds and boolean suppression flag | `#speculative-decoding` | resolver/config validation tests | Limits low-value MTP work after rejection without affecting target correctness. |