Add optional simple fallback for history ngram proposer misses

crates/mesh-llm-config/src/model.rs

@@ -560,6 +560,7 @@ pub struct SpeculativeConfig {
     pub ngram_min: Option<u32>,
     pub ngram_max: Option<u32>,
     pub ngram_proposer: Option<String>,
+    pub ngram_fallback: Option<String>,
     pub ngram_max_proposal_tokens: Option<u32>,
     pub extension_initial_tokens: Option<u32>,
     pub extension_max_tokens: Option<u32>,
@@ -611,6 +612,7 @@ impl SpeculativeConfig {
             ngram_min: pick!(ngram_min),
             ngram_max: pick!(ngram_max),
             ngram_proposer: pick!(ngram_proposer),
+            ngram_fallback: pick!(ngram_fallback),
             ngram_max_proposal_tokens: pick!(ngram_max_proposal_tokens),
             extension_initial_tokens: pick!(extension_initial_tokens),
             extension_max_tokens: pick!(extension_max_tokens),
@@ -680,6 +682,8 @@ struct SpeculativeConfigRaw {
     #[serde(default)]
     ngram_proposer: Option<String>,
     #[serde(default)]
+    ngram_fallback: Option<String>,
+    #[serde(default)]
     ngram_max_proposal_tokens: Option<u32>,
     #[serde(default)]
     extension_initial_tokens: Option<u32>,
@@ -736,6 +740,7 @@ impl<'de> Deserialize<'de> for SpeculativeConfig {
             ngram_min: raw.ngram_min,
             ngram_max: raw.ngram_max,
             ngram_proposer: raw.ngram_proposer,
+            ngram_fallback: raw.ngram_fallback,
             ngram_max_proposal_tokens: raw.ngram_max_proposal_tokens,
             extension_initial_tokens: raw.extension_initial_tokens,
             extension_max_tokens: raw.extension_max_tokens,
@@ -759,7 +764,7 @@ impl Serialize for SpeculativeConfig {
     {
         use serde::ser::SerializeMap;
 
-        let mut map = serializer.serialize_map(Some(32))?;
+        let mut map = serializer.serialize_map(Some(33))?;
         map.serialize_entry("strategy", &self.strategy)?;
         map.serialize_entry("mode", &self.mode)?;
         if self.legacy_draft_model_path_used {
@@ -788,6 +793,7 @@ impl Serialize for SpeculativeConfig {
         map.serialize_entry("ngram_min", &self.ngram_min)?;
         map.serialize_entry("ngram_max", &self.ngram_max)?;
         map.serialize_entry("ngram_proposer", &self.ngram_proposer)?;
+        map.serialize_entry("ngram_fallback", &self.ngram_fallback)?;
         map.serialize_entry("ngram_max_proposal_tokens", &self.ngram_max_proposal_tokens)?;
         map.serialize_entry("extension_initial_tokens", &self.extension_initial_tokens)?;
         map.serialize_entry("extension_max_tokens", &self.extension_max_tokens)?;

crates/mesh-llm-config/src/model/built_in_schema.rs

@@ -670,6 +670,7 @@ fn speculative_settings(prefix: &str) -> Vec<ConfigSettingSchema> {
             &format!("{prefix}.ngram_proposer"),
             string_enum(["simple", "cache", "suffix"]),
         ),
+        basic_setting(&format!("{prefix}.ngram_fallback"), string_enum(["simple"])),
         basic_setting(
             &format!("{prefix}.ngram_max_proposal_tokens"),
             ConfigValueSchema::Integer,

crates/mesh-llm-config/src/model/built_in_schema/control_behavior/speculative.rs

@@ -68,7 +68,7 @@ pub(super) fn apply_speculative_behavior(
             push_range_constraint(setting, Some(format!("{prefix}.ngram_min")), None);
             push_mode_dependency(setting, prefix, "ngram", suffix);
         }
-        "ngram_proposer" | "ngram_max_proposal_tokens" => {
+        "ngram_proposer" | "ngram_fallback" | "ngram_max_proposal_tokens" => {
             push_mode_dependency(setting, prefix, "ngram", suffix);
         }
         "extension_initial_tokens"

crates/mesh-llm-config/src/model_validation.rs

@@ -547,6 +547,19 @@ fn validate_speculative_proposer_controls(
         &["simple", "cache", "suffix"],
         &format!("{base_path}.ngram_proposer"),
     )?;
+    validate_optional_enum(
+        config.ngram_fallback.as_deref(),
+        &["simple"],
+        &format!("{base_path}.ngram_fallback"),
+    )?;
+    let simple_selected = config.ngram_proposer.as_deref() == Some("simple")
+        || config.strategy.as_deref() == Some("ngram-simple");
+    if config.ngram_fallback.is_some() && simple_selected {
+        return Err(validation_diagnostic(
+            &format!("{base_path}.ngram_fallback"),
+            format!("{base_path}.ngram_fallback requires a cache or suffix primary proposer"),
+        ));
+    }
     let suffix_selected = config.ngram_proposer.as_deref() == Some("suffix")
         || config.strategy.as_deref() == Some("ngram-suffix");
     if suffix_selected {

crates/mesh-llm-host-runtime/src/inference/skippy/resolver/speculative.rs

@@ -237,6 +237,14 @@ fn resolve_decode_config(input: DecodeResolutionInput<'_>) -> Result<Speculative
             .global_config
             .and_then(|config| config.ngram_proposer.clone()),
     );
+    let ngram_fallback = pick_owned(
+        input
+            .model_config
+            .and_then(|config| config.ngram_fallback.clone()),
+        input
+            .global_config
+            .and_then(|config| config.ngram_fallback.clone()),
+    );
     let ngram_max_proposal_tokens = pick_optional_u32(
         input
             .model_config
@@ -277,11 +285,17 @@ fn resolve_decode_config(input: DecodeResolutionInput<'_>) -> Result<Speculative
             .unwrap_or_else(|| {
                 existing.map_or(max_ngram as usize, |ngram| ngram.max_proposal_tokens)
             });
+        let fallback_simple = match ngram_fallback.as_deref() {
+            Some("simple") => true,
+            None => existing.is_some_and(|ngram| ngram.fallback_simple),
+            Some(_) => unreachable!("validated by mesh configuration"),
+        };
         config.ngram = Some(NgramProposalConfig {
             kind,
             min_ngram: min_ngram as usize,
             max_ngram: max_ngram as usize,
             max_proposal_tokens,
+            fallback_simple,
         });
         if config.effective_strategy == "disabled" {
             config.effective_strategy = ngram_effective_strategy(kind).to_string();
@@ -569,6 +583,7 @@ fn ngram_proposer_config(
         min_ngram: min_ngram as usize,
         max_ngram: max_ngram as usize,
         max_proposal_tokens: max_proposal_tokens as usize,
+        fallback_simple: false,
     })
 }
 

crates/mesh-llm-host-runtime/tests/fixtures/config_schema_defaults_ui_reference.json

@@ -868,6 +868,13 @@
         "kind": "built_in"
       }
     },
+    {
+      "canonical_path": "defaults.speculative.ngram_fallback",
+      "support": "supported",
+      "source": {
+        "kind": "built_in"
+      }
+    },
     {
       "canonical_path": "defaults.speculative.ngram_max",
       "support": "supported",

crates/skippy-server/src/binary_transport/binary_messaging.rs

@@ -304,6 +304,7 @@ mod tests {
                 min_ngram: 3,
                 max_ngram: 4,
                 max_proposal_tokens: 64,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         }

crates/skippy-server/src/binary_transport/options.rs

@@ -135,6 +135,7 @@ fn legacy_speculative_config(args: &ServeBinaryArgs) -> SpeculativeDecodeConfig
             min_ngram: args.openai_ngram_min,
             max_ngram: args.openai_ngram_max,
             max_proposal_tokens: args.openai_ngram_max,
+            fallback_simple: false,
         });
     }
     config
@@ -218,6 +219,7 @@ mod tests {
                 min_ngram: 2,
                 max_ngram: 4,
                 max_proposal_tokens: 6,
+                fallback_simple: false,
             }),
             extension: Some(NgramExtensionConfig {
                 initial_tokens: 2,

crates/skippy-server/src/frontend/native_mtp/decode.rs

@@ -716,6 +716,7 @@ mod tests {
                 min_ngram: 2,
                 max_ngram: 4,
                 max_proposal_tokens: 7,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         };

crates/skippy-server/src/frontend/speculative.rs

@@ -73,6 +73,9 @@ pub struct NgramProposalConfig {
     pub min_ngram: usize,
     pub max_ngram: usize,
     pub max_proposal_tokens: usize,
+    /// Falls back to the simple proposer when a history proposer drafts nothing.
+    #[serde(default)]
+    pub fallback_simple: bool,
 }
 
 /// Bounds for extending an MTP prefix with an N-gram tail.
@@ -153,6 +156,12 @@ impl SpeculativeDecodeConfig {
                 "suffix N-gram proposer requires {SUFFIX_MIN_SEED_LEN} <= min_ngram <= max_ngram <= {SUFFIX_NGRAM_MAX_WINDOW}"
             );
         }
+        if let Some(ngram) = &self.ngram
+            && ngram.fallback_simple
+            && ngram.kind == NgramProposerKind::Simple
+        {
+            bail!("simple N-gram fallback requires a cache or suffix primary proposer");
+        }
         if self.extension.is_some() && (!self.native_mtp.enabled || self.ngram.is_none()) {
             bail!("N-gram extension requires both native MTP and an N-gram proposer");
         }
@@ -237,6 +246,7 @@ mod standalone_speculative_config_tests {
                 min_ngram: 2,
                 max_ngram: 4,
                 max_proposal_tokens: 6,
+                fallback_simple: false,
             }),
             extension: Some(NgramExtensionConfig {
                 initial_tokens: 2,
@@ -261,6 +271,7 @@ mod standalone_speculative_config_tests {
                 min_ngram: 2,
                 max_ngram: skippy_runtime::NGRAM_CACHE_MAX_NGRAM + 1,
                 max_proposal_tokens: 6,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         };
@@ -284,6 +295,7 @@ mod standalone_speculative_config_tests {
                 min_ngram: 5,
                 max_ngram: 32,
                 max_proposal_tokens: 48,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         };
@@ -303,6 +315,7 @@ mod standalone_speculative_config_tests {
                 min_ngram: 3,
                 max_ngram: SUFFIX_NGRAM_MAX_WINDOW + 1,
                 max_proposal_tokens: 6,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         };
@@ -312,6 +325,30 @@ mod standalone_speculative_config_tests {
         assert!(error.to_string().contains("min_ngram <= max_ngram <= 64"));
     }
 
+    #[test]
+    fn standalone_speculative_config_rejects_simple_fallback_on_simple_proposer() {
+        let config = SpeculativeDecodeConfig {
+            ngram: Some(NgramProposalConfig {
+                kind: NgramProposerKind::Simple,
+                min_ngram: 2,
+                max_ngram: 4,
+                max_proposal_tokens: 6,
+                fallback_simple: true,
+            }),
+            ..SpeculativeDecodeConfig::default()
+        };
+
+        let error = config
+            .validate()
+            .expect_err("fallback needs history proposer");
+
+        assert!(
+            error
+                .to_string()
+                .contains("requires a cache or suffix primary proposer")
+        );
+    }
+
     #[test]
     fn standalone_speculative_config_rejects_suffix_matches_below_seed_length() {
         let config = SpeculativeDecodeConfig {
@@ -320,6 +357,7 @@ mod standalone_speculative_config_tests {
                 min_ngram: 2,
                 max_ngram: 16,
                 max_proposal_tokens: 4,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         };

crates/skippy-server/src/frontend/speculative/standalone.rs

@@ -20,8 +20,13 @@ pub(in crate::frontend) fn standalone_ngram_proposal_limit(
         .map_or(0, |ngram| ngram.max_proposal_tokens)
 }
 
+/// Minimum match length the simple fallback proposer scans for. The history
+/// proposers use longer configured bounds that the simple proposer cannot.
+const SIMPLE_FALLBACK_MIN_NGRAM: usize = 2;
+
 /// Runs the configured standalone N-gram proposer (simple, cache, or suffix)
-/// over committed history and returns its draft.
+/// over committed history and returns its draft. When enabled, a history
+/// proposer miss falls back to the simple proposer.
 pub(in crate::frontend) fn propose_configured_ngram_tokens(
     config: &SpeculativeDecodeConfig,
     history_proposer: &mut Option<HistoryNgramProposer>,
@@ -44,6 +49,16 @@ pub(in crate::frontend) fn propose_configured_ngram_tokens(
             .ok_or_else(|| OpenAiError::backend("configured history N-gram proposer is missing"))?
             .propose(committed_history, &[], proposal_limit)?,
     };
+    if tokens.is_empty() && ngram.fallback_simple && ngram.kind != NgramProposerKind::Simple {
+        let fallback =
+            propose_ngram_tokens(committed_history, SIMPLE_FALLBACK_MIN_NGRAM, proposal_limit)?;
+        if !fallback.is_empty() {
+            return Ok(ConfiguredNgramProposal {
+                tokens: fallback,
+                source: "simple",
+            });
+        }
+    }
     Ok(ConfiguredNgramProposal {
         tokens,
         source: ngram.kind.as_str(),
@@ -67,11 +82,22 @@ mod tests {
                 min_ngram,
                 max_ngram,
                 max_proposal_tokens: 3,
+                fallback_simple: false,
             }),
             ..SpeculativeDecodeConfig::default()
         }
     }
 
+    fn config_with_fallback(
+        kind: NgramProposerKind,
+        min_ngram: usize,
+        max_ngram: usize,
+    ) -> SpeculativeDecodeConfig {
+        let mut config = config(kind, min_ngram, max_ngram);
+        config.ngram.as_mut().unwrap().fallback_simple = true;
+        config
+    }
+
     fn propose(config: &SpeculativeDecodeConfig, history: &[i32]) -> ConfiguredNgramProposal {
         let mut proposer = HistoryNgramProposer::from_config(config).unwrap();
         propose_configured_ngram_tokens(config, &mut proposer, history, 8).unwrap()
@@ -122,4 +148,44 @@ mod tests {
         assert_eq!(proposal.source, "suffix");
         assert_eq!(proposal.tokens, vec![4, 5, 1]);
     }
+
+    #[test]
+    fn simple_fallback_fires_on_a_history_proposer_miss() {
+        let proposal = propose(
+            &config_with_fallback(NgramProposerKind::Suffix, 3, 8),
+            &[5, 6, 1, 2, 9, 7, 1, 2],
+        );
+        assert_eq!(proposal.source, "simple");
+        assert_eq!(proposal.tokens, vec![9, 7, 1]);
+    }
+
+    #[test]
+    fn simple_fallback_does_not_replace_a_history_proposer_hit() {
+        let proposal = propose(
+            &config_with_fallback(NgramProposerKind::Suffix, 3, 8),
+            &[1, 2, 3, 4, 5, 1, 2, 3],
+        );
+        assert_eq!(proposal.source, "suffix");
+        assert_eq!(proposal.tokens, vec![4, 5, 1]);
+    }
+
+    #[test]
+    fn simple_fallback_stays_off_by_default() {
+        let proposal = propose(
+            &config(NgramProposerKind::Suffix, 3, 8),
+            &[5, 6, 1, 2, 9, 1, 2],
+        );
+        assert_eq!(proposal.source, "suffix");
+        assert!(proposal.tokens.is_empty());
+    }
+
+    #[test]
+    fn simple_fallback_miss_reports_the_primary_source() {
+        let proposal = propose(
+            &config_with_fallback(NgramProposerKind::Cache, 2, 4),
+            &[1, 2, 3, 4],
+        );
+        assert_eq!(proposal.source, "cache");
+        assert!(proposal.tokens.is_empty());
+    }
 }

docs/USAGE.md

@@ -349,6 +349,7 @@ spec_default               = "auto"          # bool or "auto"
 # `cache` is request-local and requires ngram_max <= 4; `suffix` is a pure-Rust
 # longest-suffix (prompt-lookup) matcher allowing ngram_max <= 64.
 # ngram_proposer            = "cache"  # simple | cache | suffix
+# ngram_fallback            = "simple" # retry cache/suffix misses with the simple proposer
 # ngram_min                 = 2
 # ngram_max                 = 4
 # ngram_max_proposal_tokens = 6        # output budget, separate from ngram_max
@@ -784,7 +785,11 @@ verify_window_pipeline_depth = 2
 
 Suffix can also run without MTP by setting `strategy = "ngram-suffix"` and
 omitting the extension controls. Layer packages may declare `ngram-suffix` as
-a request-local proposer and standalone strategy. See
+a request-local proposer and standalone strategy. Because `cache` and `suffix`
+stay silent on a miss, `ngram_fallback = "simple"` retries the miss with the
+llama.cpp simple proposer (2-token minimum match) so some draft is always in
+flight. This targets high-latency links where a mediocre draft beats no draft;
+on a local network it is roughly neutral. See
 [Suffix N-gram Proposer](skippy/SUFFIX_NGRAM_PROPOSER.md) for the lookup
 contract, telemetry, and benchmark requirements.
 

docs/skippy/CONFIGURATION.md

@@ -170,6 +170,7 @@ missing-key error.
 | 5.5 | Draft KV cache type | `speculative.draft_cache_type_k<br>speculative.draft_cache_type_v` | P1 | `plugin/config.rs` | Schema-reserved only; resolver fail-closed today | single-stage, staged | restart/reload only | runtime draft defaults | enum values are currently rejected by the resolver and are not executable today | `#speculative-decoding` | `.sisyphus/evidence/task-1-setting-matrix.txt`; negative: `.sisyphus/evidence/task-1-setting-matrix-error.txt` | draft-model KV cache dtype overrides; reserve until an executable draft-runtime path exists |
 | 5.5 | N-gram match range | `speculative.ngram_min<br>speculative.ngram_max` | P1 | `resolver/speculative.rs` | `NgramProposalConfig` | staged | restart/reload only | package proposer bounds, otherwise required for a direct N-gram plan | both values are required for a direct N-gram proposer and must satisfy `0 < min <= max`; request-local `cache` requires `max <= 4`; `suffix` requires `3 <= min <= max <= 64` | `#speculative-decoding` | resolver/config validation tests | Match range controls lookup context, not the number of proposed tokens. |
 | 5.5 | N-gram proposer kind | `speculative.ngram_proposer` | P1 | `resolver/speculative.rs` | `NgramProposalConfig.kind` | staged | restart/reload only | package strategy kind, otherwise `simple` | `simple`, `cache`, or `suffix` | `#speculative-decoding` | resolver/config validation tests | `simple` searches accepted history; `cache` owns request-local target-committed history and can extend an MTP prefix; `suffix` is a pure-Rust longest-suffix (prompt-lookup) matcher allowing `ngram_max <= 64` for long verbatim drafts on repetitive contexts. |
+| 5.5 | N-gram simple fallback | `speculative.ngram_fallback` | P1 | `resolver/speculative.rs` | `NgramProposalConfig.fallback_simple` | staged | restart/reload only | disabled | `simple`; requires a `cache` or `suffix` primary proposer | `#speculative-decoding` | resolver/config validation tests | Retries a history-proposer miss with the llama.cpp simple proposer (2-token minimum match) so a draft is always in flight; aimed at high-latency links, roughly neutral on LAN. |
 | 5.5 | N-gram output budget | `speculative.ngram_max_proposal_tokens` | P1 | `resolver/speculative.rs` | `NgramProposalConfig.max_proposal_tokens` | staged | restart/reload only | package value, otherwise N-gram maximum | positive integer | `#speculative-decoding` | resolver/config validation tests | Independent from the N-gram match range. |
 | 5.5 | MTP N-gram extension policy | `speculative.extension_initial_tokens<br>speculative.extension_max_tokens<br>speculative.extension_tail_backoff_proposals` | P1 | `resolver/speculative.rs` | `NgramExtensionConfig` | staged | restart/reload only | package extension policy or bounded runtime defaults | requires both native MTP and an N-gram proposer; config must satisfy the frontend plan validation | `#speculative-decoding` | resolver/config validation tests | Controls the adaptive suffix only. MTP and N-gram tokens remain one target-verified candidate. |
 | 5.5 | Native MTP recovery | `speculative.native_mtp_reject_cooldown_tokens<br>speculative.native_mtp_suppress_cooldown_drafts<br>speculative.native_mtp_suppress_cooldown_draft_limit` | P2 | `resolver/speculative.rs` | `NativeMtpProposalConfig` | staged | restart/reload only | runtime defaults | non-negative integer bounds and boolean suppression flag | `#speculative-decoding` | resolver/config validation tests | Limits low-value MTP work after rejection without affecting target correctness. |