From 7f8ef6cf0a8251b4fb1f372faeb495b4a36152e8 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Wed, 22 Jul 2026 09:05:08 +1000 Subject: [PATCH 1/3] Lock split topology placement --- crates/mesh-llm-cli/src/parser.rs | 36 +++ crates/mesh-llm-cli/src/parser/commands.rs | 4 + .../src/runtime/local.rs | 1 + .../src/runtime/local_split.rs | 61 +++-- .../src/runtime/local_split/coordinator.rs | 193 +++------------- .../src/runtime/local_split/recovery.rs | 174 ++++++++++++++ .../src/runtime/local_split/test_support.rs | 1 + .../src/runtime/local_split/tests.rs | 17 ++ .../mesh-llm-host-runtime/src/runtime/mod.rs | 1 + .../src/runtime/model_lifecycle.rs | 1 + .../src/runtime/options.rs | 2 + .../src/runtime/run_auto.rs | 1 + .../src/runtime/serving_surface.rs | 1 + .../src/runtime/split_planning.rs | 69 +++++- .../src/runtime/split_topology_lock.rs | 216 ++++++++++++++++++ .../src/runtime/startup_handles.rs | 7 + .../src/runtime/startup_models.rs | 14 ++ crates/mesh-llm/src/lib.rs | 1 + crates/skippy-coordinator/src/topology.rs | 108 +++++++++ .../skippy-coordinator/src/topology/locked.rs | 164 +++++++++++++ docs/SKIPPY_SPLITS.md | 49 ++++ docs/design/TESTING.md | 4 +- 22 files changed, 934 insertions(+), 191 deletions(-) create mode 100644 crates/mesh-llm-host-runtime/src/runtime/local_split/recovery.rs create mode 100644 crates/mesh-llm-host-runtime/src/runtime/split_topology_lock.rs create mode 100644 crates/skippy-coordinator/src/topology/locked.rs diff --git a/crates/mesh-llm-cli/src/parser.rs b/crates/mesh-llm-cli/src/parser.rs index c0daff8f13..baab2bf467 100644 --- a/crates/mesh-llm-cli/src/parser.rs +++ b/crates/mesh-llm-cli/src/parser.rs @@ -98,8 +98,44 @@ pub fn assert_mesh_requirements_docs_examples_parse() { #[cfg(test)] mod tests { + use super::Cli; + use clap::Parser; + #[test] fn mesh_requirements_docs_examples_parse() { super::assert_mesh_requirements_docs_examples_parse(); } + + #[test] + fn split_topology_lock_requires_split_mode() { + let error = Cli::try_parse_from([ + "mesh-llm", + "--model", + "model.gguf", + "--split-topology-lock", + "topology.json", + ]) + .expect_err("topology lock without --split should fail"); + + assert!(error.to_string().contains("--split")); + } + + #[test] + fn split_topology_lock_parses_with_split_mode() { + let cli = Cli::try_parse_from([ + "mesh-llm", + "--model", + "model.gguf", + "--split", + "--split-topology-lock", + "topology.json", + ]) + .expect("locked split CLI should parse"); + + assert!(cli.split); + assert_eq!( + cli.split_topology_lock, + Some(std::path::PathBuf::from("topology.json")) + ); + } } diff --git a/crates/mesh-llm-cli/src/parser/commands.rs b/crates/mesh-llm-cli/src/parser/commands.rs index 9304a5807d..22333de6bc 100644 --- a/crates/mesh-llm-cli/src/parser/commands.rs +++ b/crates/mesh-llm-cli/src/parser/commands.rs @@ -620,6 +620,10 @@ pub struct Cli { #[arg(long, hide = true)] pub split: bool, + /// Pin split-serving node order and layer ranges from a JSON topology lock. + #[arg(long, value_name = "PATH", requires = "split", hide = true)] + pub split_topology_lock: Option, + /// Override context size (tokens). Default: auto-scaled to available VRAM. #[arg(long, hide = true)] pub ctx_size: Option, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local.rs b/crates/mesh-llm-host-runtime/src/runtime/local.rs index 44d5b65e22..e32e9cf9d5 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local.rs @@ -199,6 +199,7 @@ pub(super) struct LocalRuntimeModelStartSpec<'a> { pub(super) n_ubatch_override: Option, pub(super) flash_attention_override: FlashAttentionType, pub(super) parallel_override: Option, + pub(super) split_topology_lock: Option<&'a Path>, pub(super) planning_profile: RuntimeResourcePlanningProfile, pub(super) openai_guardrail_policy: OpenAiGuardrailPolicyHandle, pub(super) skippy_telemetry: skippy::SkippyTelemetryOptions, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split.rs index 0d1e01a6c0..b45e7a0869 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split.rs @@ -1,5 +1,6 @@ mod coordinator; mod loading; +mod recovery; #[cfg(test)] mod test_support; #[cfg(test)] @@ -18,9 +19,11 @@ use super::local_package::{ }; use super::split_planning::{ PlannedRuntimeSliceTopology, RuntimeSliceStagePlan, SplitTopologyResourceInputs, - plan_runtime_slice_topology_with_resources, split_participant_exclusion_labels, - split_participant_labels, split_participants_for_stages, split_stage_plan_labels, + plan_locked_runtime_slice_topology_with_resources, plan_runtime_slice_topology_with_resources, + split_participant_exclusion_labels, split_participant_labels, split_participants_for_stages, + split_stage_plan_labels, }; +use super::split_topology_lock::load_locked_split_assignments; use crate::inference::skippy; use crate::mesh; use crate::models; @@ -34,8 +37,9 @@ const SPLIT_COORDINATOR_LEASE_SECS: u64 = 4 * 60 * 60; use coordinator::{ SplitTopologyCoordinator, SplitTopologyGeneration, spawn_split_topology_coordinator, - split_stages_meet_minimum, stop_split_generation, + stop_split_generation, }; +use recovery::split_stages_meet_minimum; fn split_coordinator_lease_until_unix_ms() -> u64 { super::local::current_time_unix_ms() @@ -150,6 +154,7 @@ pub(super) async fn start_runtime_split_model( compact_meta, kv_bytes_per_token, planned_topology, + topology_locked, } = split_setup; let stages = planned_topology.stages; let planned_participants = @@ -255,6 +260,7 @@ pub(super) async fn start_runtime_split_model( survey_telemetry: spec.survey_telemetry.clone(), event_tx: coordinator_tx, stage_loss_first_seen: None, + topology_locked, })); Ok(SplitRuntimeStart::Started(Box::new(loaded))) @@ -266,6 +272,7 @@ struct SplitRuntimeStartPreparation { compact_meta: models::gguf::GgufCompactMeta, kv_bytes_per_token: u64, planned_topology: PlannedRuntimeSliceTopology, + topology_locked: bool, } async fn prepare_split_runtime_start( @@ -291,25 +298,47 @@ async fn prepare_split_runtime_start( spec.cache_type_k_override, spec.cache_type_v_override, )?; - let planned_topology = plan_runtime_slice_topology_with_resources( - topology_id, - model_ref, - &package, - &participant_snapshot.participants, - &participant_snapshot.excluded, - SplitTopologyResourceInputs { - native_context_length: compact_meta.context_length, - kv_bytes_per_token, - ctx_size_override: spec.ctx_size_override, - parallel_override: spec.parallel_override, - }, - )?; + let resources = SplitTopologyResourceInputs { + native_context_length: compact_meta.context_length, + kv_bytes_per_token, + ctx_size_override: spec.ctx_size_override, + parallel_override: spec.parallel_override, + }; + let planned_topology = if let Some(path) = spec.split_topology_lock { + let locked_stages = load_locked_split_assignments( + path, + spec.node, + model_ref, + &package, + &participant_snapshot.participants, + ) + .await?; + plan_locked_runtime_slice_topology_with_resources( + topology_id, + model_ref, + &package, + &participant_snapshot.participants, + &participant_snapshot.excluded, + resources, + &locked_stages, + )? + } else { + plan_runtime_slice_topology_with_resources( + topology_id, + model_ref, + &package, + &participant_snapshot.participants, + &participant_snapshot.excluded, + resources, + )? + }; Ok(SplitRuntimeStartPreparation { package, participant_snapshot, compact_meta, kv_bytes_per_token, planned_topology, + topology_locked: spec.split_topology_lock.is_some(), }) } diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs index 750826ece9..5ef364f096 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/coordinator.rs @@ -1,3 +1,10 @@ +use super::recovery::{ + SplitLossRecoveryDecision, SplitWithdrawGraceAction, + split_active_stage_nodes_pending_eligibility, split_connected_node_ids, + split_locked_loss_recovery_decision, split_loss_recovery_decision, + split_missing_active_stage_nodes, split_participants_meet_minimum, split_stages_meet_minimum, + split_unavailable_active_stage_nodes, split_withdraw_grace_action, +}; use super::{ OpenAiGuardrailPolicyHandle, RuntimeSliceStagePlan, SPLIT_PARTICIPANT_STABLE_FOR, SplitCoordinatorAck, SplitCoordinatorEvent, SplitCoordinatorLocalFallbackEvent, @@ -82,6 +89,9 @@ pub(super) struct SplitTopologyCoordinator { /// First observation of a withdraw-only stage loss. Cleared whenever the /// split is healthy or has a viable replacement or fallback path. pub(super) stage_loss_first_seen: Option, + /// A locked topology may be withdrawn after stage loss, but never replaced + /// or collapsed to a local fallback. + pub(super) topology_locked: bool, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -90,31 +100,6 @@ pub(super) enum SplitReplanDecision { Candidate, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum SplitLossRecoveryDecision { - NoActiveStageLoss, - ReplacementSplit, - LocalFallback, - Withdraw, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum SplitWithdrawGraceAction { - Defer, - Withdraw, -} - -pub(super) fn split_withdraw_grace_action( - first_seen: Option, - now: Instant, - grace: Duration, -) -> SplitWithdrawGraceAction { - match first_seen { - Some(seen) if now.duration_since(seen) >= grace => SplitWithdrawGraceAction::Withdraw, - _ => SplitWithdrawGraceAction::Defer, - } -} - pub(super) fn spawn_split_topology_coordinator( coordinator: SplitTopologyCoordinator, ) -> tokio::task::JoinHandle<()> { @@ -197,7 +182,9 @@ impl SplitTopologyCoordinator { &snapshot.participants, &unavailable_stage_nodes, ); - let candidate = if pending_eligibility_nodes.is_empty() { + let candidate = if self.topology_locked { + None + } else if pending_eligibility_nodes.is_empty() { self.replan_candidate(reason, &snapshot, &unavailable_stage_nodes) } else { tracing::debug!( @@ -222,6 +209,9 @@ impl SplitTopologyCoordinator { return should_continue; } + if self.topology_locked { + return true; + } self.apply_replan_candidate(reason, snapshot.participants.len(), candidate) .await } @@ -287,13 +277,21 @@ impl SplitTopologyCoordinator { unavailable_stage_nodes: &[iroh::EndpointId], candidate: Option<&SplitTopologyGeneration>, ) -> Option { - let decision = split_loss_recovery_decision( - &self.active, - connected_node_ids, - unavailable_stage_nodes, - candidate, - self.local_model_fits(), - ); + let decision = if self.topology_locked { + split_locked_loss_recovery_decision( + &self.active, + connected_node_ids, + unavailable_stage_nodes, + ) + } else { + split_loss_recovery_decision( + &self.active, + connected_node_ids, + unavailable_stage_nodes, + candidate, + self.local_model_fits(), + ) + }; if !matches!(decision, SplitLossRecoveryDecision::Withdraw) { self.stage_loss_first_seen = None; } @@ -773,54 +771,6 @@ pub(super) fn split_replan_decision_with_reason( (SplitReplanDecision::Keep, "candidate_not_materially_better") } -pub(super) fn split_loss_recovery_decision( - active: &SplitTopologyGeneration, - connected_node_ids: &[iroh::EndpointId], - unavailable_stage_nodes: &[iroh::EndpointId], - candidate: Option<&SplitTopologyGeneration>, - local_model_fits: bool, -) -> SplitLossRecoveryDecision { - if split_missing_active_stage_nodes(active, connected_node_ids).is_empty() - && unavailable_stage_nodes.is_empty() - { - return SplitLossRecoveryDecision::NoActiveStageLoss; - } - if candidate.is_some_and(|candidate| { - split_candidate_is_valid_replacement_split_after_loss(candidate, unavailable_stage_nodes) - }) { - return SplitLossRecoveryDecision::ReplacementSplit; - } - if local_model_fits { - return SplitLossRecoveryDecision::LocalFallback; - } - SplitLossRecoveryDecision::Withdraw -} - -pub(super) fn split_candidate_is_valid_replacement_split( - candidate: &SplitTopologyGeneration, -) -> bool { - split_participants_meet_minimum(&candidate.participants) - && split_stages_meet_minimum(&candidate.stages) -} - -pub(super) fn split_candidate_is_valid_replacement_split_after_loss( - candidate: &SplitTopologyGeneration, - unavailable_stage_nodes: &[iroh::EndpointId], -) -> bool { - split_candidate_is_valid_replacement_split(candidate) - && !split_candidate_uses_unavailable_stage_node(candidate, unavailable_stage_nodes) -} - -pub(super) fn split_candidate_uses_unavailable_stage_node( - candidate: &SplitTopologyGeneration, - unavailable_stage_nodes: &[iroh::EndpointId], -) -> bool { - candidate - .stages - .iter() - .any(|stage| unavailable_stage_nodes.contains(&stage.node_id)) -} - pub(super) fn split_recovery_candidate_participants( participants: &[SplitParticipant], unavailable_stage_nodes: &[iroh::EndpointId], @@ -860,14 +810,6 @@ pub(super) fn log_split_replan_quorum_not_met( ); } -pub(super) fn split_participants_meet_minimum(participants: &[SplitParticipant]) -> bool { - participants.len() >= SPLIT_DEFAULT_MIN_PARTICIPANTS -} - -pub(super) fn split_stages_meet_minimum(stages: &[RuntimeSliceStagePlan]) -> bool { - stages.len() >= SPLIT_DEFAULT_MIN_PARTICIPANTS -} - pub(super) fn split_active_stage_node_missing_from_participants( active: &SplitTopologyGeneration, participants: &[SplitParticipant], @@ -879,79 +821,6 @@ pub(super) fn split_active_stage_node_missing_from_participants( }) } -pub(super) fn split_missing_active_stage_nodes( - active: &SplitTopologyGeneration, - connected_node_ids: &[iroh::EndpointId], -) -> Vec { - let mut missing = Vec::new(); - for stage in &active.stages { - if connected_node_ids.contains(&stage.node_id) || missing.contains(&stage.node_id) { - continue; - } - missing.push(stage.node_id); - } - missing -} - -pub(super) fn split_unavailable_active_stage_nodes( - active: &SplitTopologyGeneration, - connected_node_ids: &[iroh::EndpointId], - runtime_statuses: &[mesh::StageRuntimeStatus], -) -> Vec { - let mut unavailable = split_missing_active_stage_nodes(active, connected_node_ids); - for status in runtime_statuses { - if !matches!( - status.state, - skippy::StageRuntimeState::Failed - | skippy::StageRuntimeState::Stopping - | skippy::StageRuntimeState::Stopped - ) || status.topology_id != active.topology_id - || status.run_id != active.run_id - || active - .stages - .iter() - .all(|stage| stage.stage_id != status.stage_id) - { - continue; - } - let Some(node_id) = status.node_id else { - continue; - }; - if !unavailable.contains(&node_id) { - unavailable.push(node_id); - } - } - unavailable -} - -pub(super) fn split_active_stage_nodes_pending_eligibility( - active: &SplitTopologyGeneration, - connected_node_ids: &[iroh::EndpointId], - eligible_participants: &[SplitParticipant], - unavailable_stage_nodes: &[iroh::EndpointId], -) -> Vec { - active - .stages - .iter() - .filter_map(|stage| { - let connected = connected_node_ids.contains(&stage.node_id); - let eligible = eligible_participants - .iter() - .any(|participant| participant.node_id == stage.node_id); - let unavailable = unavailable_stage_nodes.contains(&stage.node_id); - (connected && !eligible && !unavailable).then_some(stage.node_id) - }) - .collect() -} - -pub(super) async fn split_connected_node_ids(node: &mesh::Node) -> Vec { - let mut node_ids = vec![node.id()]; - node_ids.extend(node.peers().await.into_iter().map(|peer| peer.id)); - node_ids.sort_by_key(ToString::to_string); - node_ids.dedup(); - node_ids -} - pub(super) async fn stop_split_generation( node: &mesh::Node, generation: &SplitTopologyGeneration, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/recovery.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/recovery.rs new file mode 100644 index 0000000000..393d281816 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/recovery.rs @@ -0,0 +1,174 @@ +use super::coordinator::SplitTopologyGeneration; +use super::{RuntimeSliceStagePlan, SplitParticipant}; +use crate::inference::skippy; +use crate::mesh; +use crate::runtime::local_package::SPLIT_DEFAULT_MIN_PARTICIPANTS; +use std::time::{Duration, Instant}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SplitLossRecoveryDecision { + NoActiveStageLoss, + ReplacementSplit, + LocalFallback, + Withdraw, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SplitWithdrawGraceAction { + Defer, + Withdraw, +} + +pub(super) fn split_withdraw_grace_action( + first_seen: Option, + now: Instant, + grace: Duration, +) -> SplitWithdrawGraceAction { + match first_seen { + Some(seen) if now.duration_since(seen) >= grace => SplitWithdrawGraceAction::Withdraw, + _ => SplitWithdrawGraceAction::Defer, + } +} + +pub(super) fn split_loss_recovery_decision( + active: &SplitTopologyGeneration, + connected_node_ids: &[iroh::EndpointId], + unavailable_stage_nodes: &[iroh::EndpointId], + candidate: Option<&SplitTopologyGeneration>, + local_model_fits: bool, +) -> SplitLossRecoveryDecision { + if split_missing_active_stage_nodes(active, connected_node_ids).is_empty() + && unavailable_stage_nodes.is_empty() + { + return SplitLossRecoveryDecision::NoActiveStageLoss; + } + if candidate.is_some_and(|candidate| { + split_candidate_is_valid_replacement_split_after_loss(candidate, unavailable_stage_nodes) + }) { + return SplitLossRecoveryDecision::ReplacementSplit; + } + if local_model_fits { + return SplitLossRecoveryDecision::LocalFallback; + } + SplitLossRecoveryDecision::Withdraw +} + +pub(super) fn split_locked_loss_recovery_decision( + active: &SplitTopologyGeneration, + connected_node_ids: &[iroh::EndpointId], + unavailable_stage_nodes: &[iroh::EndpointId], +) -> SplitLossRecoveryDecision { + split_loss_recovery_decision( + active, + connected_node_ids, + unavailable_stage_nodes, + None, + false, + ) +} + +pub(super) fn split_candidate_is_valid_replacement_split( + candidate: &SplitTopologyGeneration, +) -> bool { + split_participants_meet_minimum(&candidate.participants) + && split_stages_meet_minimum(&candidate.stages) +} + +pub(super) fn split_candidate_is_valid_replacement_split_after_loss( + candidate: &SplitTopologyGeneration, + unavailable_stage_nodes: &[iroh::EndpointId], +) -> bool { + split_candidate_is_valid_replacement_split(candidate) + && !split_candidate_uses_unavailable_stage_node(candidate, unavailable_stage_nodes) +} + +pub(super) fn split_candidate_uses_unavailable_stage_node( + candidate: &SplitTopologyGeneration, + unavailable_stage_nodes: &[iroh::EndpointId], +) -> bool { + candidate + .stages + .iter() + .any(|stage| unavailable_stage_nodes.contains(&stage.node_id)) +} + +pub(super) fn split_participants_meet_minimum(participants: &[SplitParticipant]) -> bool { + participants.len() >= SPLIT_DEFAULT_MIN_PARTICIPANTS +} + +pub(super) fn split_stages_meet_minimum(stages: &[RuntimeSliceStagePlan]) -> bool { + stages.len() >= SPLIT_DEFAULT_MIN_PARTICIPANTS +} + +pub(super) fn split_missing_active_stage_nodes( + active: &SplitTopologyGeneration, + connected_node_ids: &[iroh::EndpointId], +) -> Vec { + let mut missing = Vec::new(); + for stage in &active.stages { + if connected_node_ids.contains(&stage.node_id) || missing.contains(&stage.node_id) { + continue; + } + missing.push(stage.node_id); + } + missing +} + +pub(super) fn split_unavailable_active_stage_nodes( + active: &SplitTopologyGeneration, + connected_node_ids: &[iroh::EndpointId], + runtime_statuses: &[mesh::StageRuntimeStatus], +) -> Vec { + let mut unavailable = split_missing_active_stage_nodes(active, connected_node_ids); + for status in runtime_statuses { + if !matches!( + status.state, + skippy::StageRuntimeState::Failed + | skippy::StageRuntimeState::Stopping + | skippy::StageRuntimeState::Stopped + ) || status.topology_id != active.topology_id + || status.run_id != active.run_id + || active + .stages + .iter() + .all(|stage| stage.stage_id != status.stage_id) + { + continue; + } + let Some(node_id) = status.node_id else { + continue; + }; + if !unavailable.contains(&node_id) { + unavailable.push(node_id); + } + } + unavailable +} + +pub(super) fn split_active_stage_nodes_pending_eligibility( + active: &SplitTopologyGeneration, + connected_node_ids: &[iroh::EndpointId], + eligible_participants: &[SplitParticipant], + unavailable_stage_nodes: &[iroh::EndpointId], +) -> Vec { + active + .stages + .iter() + .filter_map(|stage| { + let connected = connected_node_ids.contains(&stage.node_id); + let eligible = eligible_participants + .iter() + .any(|participant| participant.node_id == stage.node_id); + let unavailable = unavailable_stage_nodes.contains(&stage.node_id); + (connected && !eligible && !unavailable).then_some(stage.node_id) + }) + .collect() +} + +pub(super) async fn split_connected_node_ids(node: &mesh::Node) -> Vec { + let mut node_ids = vec![node.id()]; + node_ids.extend(node.peers().await.into_iter().map(|peer| peer.id)); + node_ids.sort_by_key(ToString::to_string); + node_ids.dedup(); + node_ids +} diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs index ec29edd6e4..33aff978e3 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/test_support.rs @@ -509,6 +509,7 @@ max_tokens = 222 n_ubatch_override: None, flash_attention_override: FlashAttentionType::Auto, parallel_override: None, + split_topology_lock: None, planning_profile: RuntimeResourcePlanningProfile::DedicatedLocal, openai_guardrail_policy: openai_guardrail_policy_handle( openai_frontend::GuardrailMode::Disabled, diff --git a/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs b/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs index 3e197ce3f0..e132980c13 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/local_split/tests.rs @@ -1,5 +1,6 @@ use super::coordinator::*; use super::loading::*; +use super::recovery::*; use super::test_support::*; use super::*; use crate::inference::election; @@ -1277,6 +1278,22 @@ fn split_loss_recovery_withdraws_when_split_and_local_paths_are_unavailable() { ); } +#[test] +fn locked_split_withdraws_instead_of_replanning_or_falling_back() { + let active = SplitTopologyGeneration::new( + "topology-a".into(), + "run-a".into(), + 1, + vec![participant(1), participant(2)], + vec![stage(1, 0, 0, 20), stage(2, 1, 20, 40)], + ); + + assert_eq!( + split_locked_loss_recovery_decision(&active, &[make_id(1)], &[]), + SplitLossRecoveryDecision::Withdraw + ); +} + #[test] fn split_withdraw_grace_defers_a_fresh_loss() { let first_seen = Instant::now(); diff --git a/crates/mesh-llm-host-runtime/src/runtime/mod.rs b/crates/mesh-llm-host-runtime/src/runtime/mod.rs index ed515e1bc5..3e6fcea889 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/mod.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/mod.rs @@ -20,6 +20,7 @@ mod run_auto; mod runtime_registry; mod serving_surface; mod split_planning; +mod split_topology_lock; mod startup_handles; mod startup_identity; mod startup_models; diff --git a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs index a0d85b4a23..7daf58639a 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/model_lifecycle.rs @@ -404,6 +404,7 @@ pub(super) async fn run_auto_load_runtime_model( .and_then(|m| m.flash_attention) .unwrap_or(FlashAttentionType::Auto), parallel_override, + split_topology_lock: None, planning_profile: runtime_resource_planning_profile(ctx.options), openai_guardrail_policy: ctx.openai_guardrail_policy.clone(), skippy_telemetry: skippy_telemetry_options(ctx.options), diff --git a/crates/mesh-llm-host-runtime/src/runtime/options.rs b/crates/mesh-llm-host-runtime/src/runtime/options.rs index ef53ecfa68..4b213be130 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/options.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/options.rs @@ -59,6 +59,7 @@ pub struct RuntimeOptions { pub no_draft: bool, pub speculative_overrides: Option, pub split: bool, + pub split_topology_lock: Option, pub ctx_size: Option, pub max_vram: Option, pub no_enumerate_host: bool, @@ -125,6 +126,7 @@ impl Default for RuntimeOptions { no_draft: false, speculative_overrides: None, split: false, + split_topology_lock: None, ctx_size: None, max_vram: None, no_enumerate_host: false, diff --git a/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs b/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs index 8db1940370..bfb812852f 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/run_auto.rs @@ -995,6 +995,7 @@ pub(super) async fn spawn_run_auto_startup_model_tasks( n_ubatch: primary_n_ubatch, flash_attention: primary_flash_attention, parallel_override: primary_parallel_override, + split_topology_lock: options.split_topology_lock.clone(), resource_planning_profile, openai_guardrail_policy: runtime_state.openai_guardrail_policy.clone(), split: options.split, diff --git a/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs b/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs index 764ef33d20..78c0ca26f2 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/serving_surface.rs @@ -1125,6 +1125,7 @@ pub(super) async fn spawn_run_auto_additional_model_tasks(ctx: RunAutoAdditional extra_model.parallel, &ctx.config.gpu, ), + split_topology_lock: ctx.options.split_topology_lock.clone(), resource_planning_profile: runtime_resource_planning_profile(ctx.options), openai_guardrail_policy: ctx.openai_guardrail_policy.clone(), split: ctx.options.split, diff --git a/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs b/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs index 92e1ad3429..eb1194f739 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs @@ -1,11 +1,13 @@ use crate::inference::skippy; use anyhow::{Context, Result}; use skippy_coordinator::topology::{ - TopologyNode, TopologyPlanningInput, TopologyStagePlan, minimum_valid_context, plan_topology, + LockedTopologyStage, TopologyNode, TopologyPlanningInput, TopologyStagePlan, + minimum_valid_context, plan_locked_topology, plan_topology, }; use std::collections::HashMap; use super::local::{SplitParticipant, SplitParticipantExclusion}; +use super::split_topology_lock::LockedSplitStageAssignment; // Fixed per-node reserve the split planner will not fill with weights or KV. // @@ -96,7 +98,20 @@ pub(super) struct PlannedRuntimeSliceTopology { } pub(super) fn plan_split_topology(input: SplitTopologyPlanInput) -> Result { - let plan = plan_topology(&TopologyPlanningInput { + let plan = + plan_topology(&topology_planning_input(input)).context("plan skippy split topology")?; + + Ok(SplitTopologyPlan { + context_length: plan.context_length, + parallel_lanes: plan.parallel_lanes, + estimated_decode_network_ms_per_token: plan.estimated_decode_network_ms_per_token, + decode_tpot_target_met: plan.decode_tpot_target_met, + stages: plan.stages, + }) +} + +fn topology_planning_input(input: SplitTopologyPlanInput) -> TopologyPlanningInput { + TopologyPlanningInput { native_context_length: input.native_context_length, layer_count: input.layer_count, model_weight_bytes: input.model_weight_bytes, @@ -117,16 +132,7 @@ pub(super) fn plan_split_topology(input: SplitTopologyPlanInput) -> Result u64 { @@ -199,6 +205,45 @@ pub(super) fn plan_runtime_slice_topology_with_resources( }) } +pub(super) fn plan_locked_runtime_slice_topology_with_resources( + topology_id: &str, + model_ref: &str, + package: &skippy::SkippyPackageIdentity, + participants: &[SplitParticipant], + excluded: &[SplitParticipantExclusion], + resources: SplitTopologyResourceInputs, + locked_stages: &[LockedSplitStageAssignment], +) -> Result { + let participant_by_id = participant_index_by_id(participants); + let locked_stages = locked_stages + .iter() + .map(|stage| LockedTopologyStage { + node_id: stage.node_id.to_string(), + layer_start: stage.layer_start, + layer_end: stage.layer_end, + }) + .collect::>(); + let input = runtime_slice_plan_input(package, participants, resources); + let plan = plan_locked_topology(&topology_planning_input(input), &locked_stages) + .context("validate locked skippy split topology")?; + let mut stages = map_runtime_slice_stages(plan.stages, &participant_by_id)?; + stages.sort_by_key(|stage| stage.stage_index); + validate_split_capacity(model_ref, package, participants, &stages, excluded)?; + tracing::info!( + topology_id, + model_ref, + context_length = plan.context_length, + slots = plan.parallel_lanes, + stages = ?split_stage_plan_labels(&stages), + "validated locked split runtime topology" + ); + Ok(PlannedRuntimeSliceTopology { + stages, + context_length: plan.context_length, + slots: plan.parallel_lanes, + }) +} + fn plan_runtime_slice_topology_result( topology_id: &str, model_ref: &str, diff --git a/crates/mesh-llm-host-runtime/src/runtime/split_topology_lock.rs b/crates/mesh-llm-host-runtime/src/runtime/split_topology_lock.rs new file mode 100644 index 0000000000..d4d9bcea79 --- /dev/null +++ b/crates/mesh-llm-host-runtime/src/runtime/split_topology_lock.rs @@ -0,0 +1,216 @@ +use super::local_package::SplitParticipant; +use crate::inference::skippy; +use crate::mesh; +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs::File; +use std::path::Path; + +const SPLIT_TOPOLOGY_LOCK_VERSION: u32 = 1; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct LockedSplitStageAssignment { + pub(super) node_id: iroh::EndpointId, + pub(super) layer_start: u32, + pub(super) layer_end: u32, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SplitTopologyLockFile { + version: u32, + model: String, + manifest_sha256: String, + stages: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SplitTopologyLockStage { + node: String, + layer_start: u32, + layer_end: u32, +} + +#[derive(Clone, Debug)] +struct ParticipantIdentity { + node_id: iroh::EndpointId, + hostname: Option, +} + +pub(super) async fn load_locked_split_assignments( + path: &Path, + node: &mesh::Node, + model_ref: &str, + package: &skippy::SkippyPackageIdentity, + participants: &[SplitParticipant], +) -> Result> { + let file = + File::open(path).with_context(|| format!("open split topology lock {}", path.display()))?; + let topology: SplitTopologyLockFile = serde_json::from_reader(file) + .with_context(|| format!("parse split topology lock {}", path.display()))?; + validate_lock_identity(&topology, model_ref, package)?; + let identities = participant_identities(node, participants).await; + topology + .stages + .iter() + .enumerate() + .map(|(index, stage)| resolve_stage(index, stage, &identities)) + .collect() +} + +fn validate_lock_identity( + topology: &SplitTopologyLockFile, + model_ref: &str, + package: &skippy::SkippyPackageIdentity, +) -> Result<()> { + anyhow::ensure!( + topology.version == SPLIT_TOPOLOGY_LOCK_VERSION, + "unsupported split topology lock version {}; expected {}", + topology.version, + SPLIT_TOPOLOGY_LOCK_VERSION + ); + anyhow::ensure!( + topology.model == model_ref || topology.model == package.package_ref, + "split topology lock model {} does not match requested model {} or package {}", + topology.model, + model_ref, + package.package_ref + ); + anyhow::ensure!( + topology.manifest_sha256 == package.manifest_sha256, + "split topology lock manifest {} does not match resolved package manifest {}", + topology.manifest_sha256, + package.manifest_sha256 + ); + anyhow::ensure!( + topology.stages.len() >= super::local::SPLIT_DEFAULT_MIN_PARTICIPANTS, + "split topology lock requires at least two stages" + ); + Ok(()) +} + +async fn participant_identities( + node: &mesh::Node, + participants: &[SplitParticipant], +) -> Vec { + let hostnames = node + .peers() + .await + .into_iter() + .map(|peer| (peer.id, peer.hostname)) + .collect::>(); + participants + .iter() + .map(|participant| ParticipantIdentity { + node_id: participant.node_id, + hostname: if participant.node_id == node.id() { + node.hostname.clone() + } else { + hostnames.get(&participant.node_id).cloned().flatten() + }, + }) + .collect() +} + +fn resolve_stage( + index: usize, + stage: &SplitTopologyLockStage, + identities: &[ParticipantIdentity], +) -> Result { + let selector = stage.node.trim(); + anyhow::ensure!( + !selector.is_empty(), + "split topology lock stage {index} has an empty node selector" + ); + let matches = identities + .iter() + .filter(|identity| participant_matches(identity, selector)) + .collect::>(); + anyhow::ensure!( + matches.len() == 1, + "split topology lock stage {index} selector {selector:?} matched {} eligible nodes; available: {}", + matches.len(), + participant_identity_labels(identities).join(", ") + ); + Ok(LockedSplitStageAssignment { + node_id: matches[0].node_id, + layer_start: stage.layer_start, + layer_end: stage.layer_end, + }) +} + +fn participant_matches(identity: &ParticipantIdentity, selector: &str) -> bool { + identity.node_id.to_string() == selector + || identity + .hostname + .as_deref() + .is_some_and(|hostname| normalize_hostname(hostname) == normalize_hostname(selector)) +} + +fn normalize_hostname(hostname: &str) -> String { + hostname.trim().trim_end_matches('.').to_ascii_lowercase() +} + +fn participant_identity_labels(identities: &[ParticipantIdentity]) -> Vec { + identities + .iter() + .map(|identity| { + identity.hostname.as_ref().map_or_else( + || identity.node_id.to_string(), + |hostname| format!("{hostname}={}", identity.node_id), + ) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn endpoint(seed: u8) -> iroh::EndpointId { + iroh::SecretKey::from_bytes(&[seed; 32]).public() + } + + #[test] + fn hostname_selector_is_case_insensitive_and_ignores_trailing_dot() { + let identities = vec![ParticipantIdentity { + node_id: endpoint(1), + hostname: Some("micstudio.local".to_string()), + }]; + let stage = SplitTopologyLockStage { + node: "MICSTUDIO.LOCAL.".to_string(), + layer_start: 0, + layer_end: 12, + }; + + let resolved = resolve_stage(0, &stage, &identities).unwrap(); + + assert_eq!(resolved.node_id, endpoint(1)); + assert_eq!((resolved.layer_start, resolved.layer_end), (0, 12)); + } + + #[test] + fn ambiguous_hostname_selector_is_rejected() { + let identities = vec![ + ParticipantIdentity { + node_id: endpoint(1), + hostname: Some("worker.local".to_string()), + }, + ParticipantIdentity { + node_id: endpoint(2), + hostname: Some("worker.local".to_string()), + }, + ]; + let stage = SplitTopologyLockStage { + node: "worker.local".to_string(), + layer_start: 0, + layer_end: 12, + }; + + let error = resolve_stage(0, &stage, &identities).unwrap_err(); + + assert!(error.to_string().contains("matched 2 eligible nodes")); + } +} diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs index f0fad88bfd..cbcaf7b5e7 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_handles.rs @@ -68,6 +68,7 @@ pub(super) struct StartupLocalModelTask { pub(super) n_ubatch: Option, pub(super) flash_attention: FlashAttentionType, pub(super) parallel_override: Option, + pub(super) split_topology_lock: Option, pub(super) resource_planning_profile: RuntimeResourcePlanningProfile, pub(super) openai_guardrail_policy: OpenAiGuardrailPolicyHandle, pub(super) split: bool, @@ -222,6 +223,7 @@ pub(super) struct StartupLaunchRuntimeContext<'a> { pub(super) n_ubatch: Option, pub(super) flash_attention: FlashAttentionType, pub(super) parallel_override: Option, + pub(super) split_topology_lock: Option<&'a Path>, pub(super) resource_planning_profile: RuntimeResourcePlanningProfile, pub(super) openai_guardrail_policy: OpenAiGuardrailPolicyHandle, pub(super) skippy_telemetry: &'a skippy::SkippyTelemetryOptions, @@ -697,6 +699,7 @@ pub(super) async fn startup_handle_local_fallback_event( n_ubatch_override: ctx.n_ubatch, flash_attention_override: ctx.flash_attention, parallel_override: ctx.parallel_override, + split_topology_lock: None, planning_profile: ctx.resource_planning_profile, openai_guardrail_policy: ctx.openai_guardrail_policy.clone(), skippy_telemetry: ctx.skippy_telemetry.clone(), @@ -1032,6 +1035,7 @@ pub(super) async fn startup_launch_runtime( n_ubatch, flash_attention, parallel_override, + split_topology_lock, resource_planning_profile, openai_guardrail_policy, skippy_telemetry, @@ -1060,6 +1064,7 @@ pub(super) async fn startup_launch_runtime( n_ubatch_override: n_ubatch, flash_attention_override: flash_attention, parallel_override, + split_topology_lock, planning_profile: resource_planning_profile, openai_guardrail_policy: openai_guardrail_policy.clone(), skippy_telemetry: skippy_telemetry.clone(), @@ -1174,6 +1179,7 @@ pub(super) async fn startup_local_model_loop(params: StartupLocalModelTask) { n_ubatch, flash_attention, parallel_override, + split_topology_lock, resource_planning_profile, openai_guardrail_policy, split, @@ -1234,6 +1240,7 @@ pub(super) async fn startup_local_model_loop(params: StartupLocalModelTask) { n_ubatch, flash_attention, parallel_override, + split_topology_lock: split_topology_lock.as_deref(), resource_planning_profile, openai_guardrail_policy: openai_guardrail_policy.clone(), skippy_telemetry: &skippy_telemetry, diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs index 5259f86520..48a1558c99 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs @@ -396,6 +396,14 @@ pub(super) fn validate_runtime_cli_model_options(options: &RuntimeOptions) -> Re mmproj.display() ); } + if let Some(path) = &options.split_topology_lock { + anyhow::ensure!(options.split, "--split-topology-lock requires --split"); + anyhow::ensure!( + path.is_file(), + "split topology lock path is not a file: {}", + path.display() + ); + } Ok(()) } @@ -406,6 +414,12 @@ pub(super) async fn prepare_runtime_startup( ) -> Result> { validate_runtime_cli_model_options(options)?; let startup_specs = build_startup_model_specs(options, config)?; + if options.split_topology_lock.is_some() { + anyhow::ensure!( + startup_specs.len() == 1, + "--split-topology-lock requires exactly one startup model" + ); + } if should_show_serve_config_help(explicit_surface, options, &startup_specs) { let config_path = plugin::config_path(options.config.as_deref()).unwrap_or_else(|_| { dirs::home_dir() diff --git a/crates/mesh-llm/src/lib.rs b/crates/mesh-llm/src/lib.rs index d967980699..3afd74be00 100644 --- a/crates/mesh-llm/src/lib.rs +++ b/crates/mesh-llm/src/lib.rs @@ -169,6 +169,7 @@ fn runtime_options_from_cli(cli: mesh_llm_cli::Cli) -> mesh_llm_host_runtime::Ru no_draft: cli.no_draft, speculative_overrides, split: cli.split, + split_topology_lock: cli.split_topology_lock, ctx_size: cli.ctx_size, max_vram: cli.max_vram, no_enumerate_host: cli.no_enumerate_host, diff --git a/crates/skippy-coordinator/src/topology.rs b/crates/skippy-coordinator/src/topology.rs index f2a0423979..aef5d55301 100644 --- a/crates/skippy-coordinator/src/topology.rs +++ b/crates/skippy-coordinator/src/topology.rs @@ -1,5 +1,9 @@ use std::cmp::Ordering; +mod locked; + +pub use locked::{LockedTopologyStage, plan_locked_topology}; + /// Default auto lane cap. Matches llama-server's default of `--parallel 4`. /// Users can override via `gpu.parallel` in config.toml or the per-model /// `parallel` setting. @@ -82,6 +86,30 @@ pub enum TopologyPlanError { ZeroParallelLanes, #[error("no topology can distribute all layers and keep context >= {minimum_context}")] NoValidTopology { minimum_context: u32 }, + #[error("locked topology must contain at least {minimum} stages; found {actual}")] + LockedStageCount { minimum: usize, actual: usize }, + #[error("locked topology references unknown node {node_id}")] + LockedUnknownNode { node_id: String }, + #[error("locked topology assigns node {node_id} more than once")] + LockedDuplicateNode { node_id: String }, + #[error( + "locked topology stage {stage_index} must start at layer {expected_start}; found {actual_start}" + )] + LockedNonContiguousRange { + stage_index: usize, + expected_start: u32, + actual_start: u32, + }, + #[error("locked topology stage {stage_index} has empty or reversed range {start}..{end}")] + LockedInvalidRange { + stage_index: usize, + start: u32, + end: u32, + }, + #[error("locked topology ends at layer {actual_end}; model has {layer_count} layers")] + LockedIncompleteCoverage { actual_end: u32, layer_count: u32 }, + #[error("locked topology cannot fit context >= {minimum_context}")] + LockedTopologyDoesNotFit { minimum_context: u32 }, } pub fn plan_topology(input: &TopologyPlanningInput) -> Result { @@ -693,6 +721,86 @@ mod tests { ); } + #[test] + fn locked_topology_preserves_node_order_and_layer_ranges() { + let mut request = input(vec![node("large", 48), node("small", 24)]); + request.minimum_nodes = 2; + let locked = vec![ + LockedTopologyStage { + node_id: "small".to_string(), + layer_start: 0, + layer_end: 12, + }, + LockedTopologyStage { + node_id: "large".to_string(), + layer_start: 12, + layer_end: 40, + }, + ]; + + let plan = plan_locked_topology(&request, &locked).unwrap(); + + assert_eq!( + plan.stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage.layer_start, stage.layer_end)) + .collect::>(), + vec![("small", 0, 12), ("large", 12, 40)] + ); + } + + #[test] + fn locked_topology_rejects_non_contiguous_ranges() { + let mut request = input(vec![node("a", 48), node("b", 48)]); + request.minimum_nodes = 2; + let locked = vec![ + LockedTopologyStage { + node_id: "a".to_string(), + layer_start: 0, + layer_end: 19, + }, + LockedTopologyStage { + node_id: "b".to_string(), + layer_start: 20, + layer_end: 40, + }, + ]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedNonContiguousRange { + stage_index: 1, + expected_start: 19, + actual_start: 20, + }) + ); + } + + #[test] + fn locked_topology_fails_when_pinned_stage_exceeds_node_capacity() { + let mut request = input(vec![node("small", 12), node("large", 48)]); + request.minimum_nodes = 2; + let locked = vec![ + LockedTopologyStage { + node_id: "small".to_string(), + layer_start: 0, + layer_end: 30, + }, + LockedTopologyStage { + node_id: "large".to_string(), + layer_start: 30, + layer_end: 40, + }, + ]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedTopologyDoesNotFit { + minimum_context: 65_536, + }) + ); + } + #[test] fn applies_max_vram_and_runtime_headroom_per_node() { let mut capped = node("capped", 80); diff --git a/crates/skippy-coordinator/src/topology/locked.rs b/crates/skippy-coordinator/src/topology/locked.rs new file mode 100644 index 0000000000..af50d27657 --- /dev/null +++ b/crates/skippy-coordinator/src/topology/locked.rs @@ -0,0 +1,164 @@ +use super::{ + CandidatePlan, TopologyPlan, TopologyPlanError, TopologyPlanningInput, TopologyStagePlan, + UsableNode, context_candidates, decode_tpot_target_met, estimate_decode_network_ms_per_token, + layer_required_bytes, layer_weight_bytes, minimum_valid_context, parallel_lane_candidates, + sum_u64, usable_nodes, validate_input, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LockedTopologyStage { + pub node_id: String, + pub layer_start: u32, + pub layer_end: u32, +} + +pub fn plan_locked_topology( + input: &TopologyPlanningInput, + locked_stages: &[LockedTopologyStage], +) -> Result { + validate_input(input)?; + validate_locked_stages(input, locked_stages)?; + + let minimum_context = minimum_valid_context(input.native_context_length); + let context_candidates = context_candidates( + input.native_context_length, + minimum_context, + input.context_length_override, + )?; + let lane_candidates = parallel_lane_candidates(input.parallel_lanes_override)?; + let nodes = usable_nodes(&input.nodes); + let locked_nodes = locked_stage_nodes(&nodes, locked_stages)?; + + for context_length in context_candidates { + for parallel_lanes in lane_candidates.iter().copied() { + if let Some(candidate) = fit_locked_candidate( + input, + locked_stages, + &locked_nodes, + context_length, + parallel_lanes, + ) { + return Ok(candidate.plan); + } + } + } + + Err(TopologyPlanError::LockedTopologyDoesNotFit { minimum_context }) +} + +fn validate_locked_stages( + input: &TopologyPlanningInput, + locked_stages: &[LockedTopologyStage], +) -> Result<(), TopologyPlanError> { + let minimum = input.minimum_nodes.max(1); + if locked_stages.len() < minimum { + return Err(TopologyPlanError::LockedStageCount { + minimum, + actual: locked_stages.len(), + }); + } + + let mut expected_start = 0; + let mut seen_nodes = std::collections::HashSet::new(); + for (stage_index, stage) in locked_stages.iter().enumerate() { + if stage.layer_end <= stage.layer_start { + return Err(TopologyPlanError::LockedInvalidRange { + stage_index, + start: stage.layer_start, + end: stage.layer_end, + }); + } + if stage.layer_start != expected_start { + return Err(TopologyPlanError::LockedNonContiguousRange { + stage_index, + expected_start, + actual_start: stage.layer_start, + }); + } + if !seen_nodes.insert(stage.node_id.as_str()) { + return Err(TopologyPlanError::LockedDuplicateNode { + node_id: stage.node_id.clone(), + }); + } + expected_start = stage.layer_end; + } + if expected_start != input.layer_count { + return Err(TopologyPlanError::LockedIncompleteCoverage { + actual_end: expected_start, + layer_count: input.layer_count, + }); + } + Ok(()) +} + +fn locked_stage_nodes( + nodes: &[UsableNode], + locked_stages: &[LockedTopologyStage], +) -> Result, TopologyPlanError> { + locked_stages + .iter() + .map(|stage| { + nodes + .iter() + .find(|node| node.node_id == stage.node_id) + .cloned() + .ok_or_else(|| TopologyPlanError::LockedUnknownNode { + node_id: stage.node_id.clone(), + }) + }) + .collect() +} + +fn fit_locked_candidate( + input: &TopologyPlanningInput, + locked_stages: &[LockedTopologyStage], + nodes: &[UsableNode], + context_length: u32, + parallel_lanes: usize, +) -> Option { + let layer_weights = layer_weight_bytes(input); + let kv_per_layer = input + .kv_bytes_per_token + .div_ceil(u64::from(input.layer_count)); + let layer_required_bytes = + layer_required_bytes(&layer_weights, kv_per_layer, context_length, parallel_lanes)?; + let mut stages = Vec::with_capacity(locked_stages.len()); + let mut minimum_remaining_vram = u64::MAX; + let mut total_remaining_vram = 0u128; + + for (stage_index, (locked, node)) in locked_stages.iter().zip(nodes).enumerate() { + let range = locked.layer_start as usize..locked.layer_end as usize; + let parameter_bytes = sum_u64(&layer_weights[range.clone()]); + let required_bytes = sum_u64(&layer_required_bytes[range]); + if required_bytes > node.usable_vram_bytes { + return None; + } + let remaining = node.usable_vram_bytes - required_bytes; + minimum_remaining_vram = minimum_remaining_vram.min(remaining); + total_remaining_vram += u128::from(remaining); + stages.push(TopologyStagePlan { + stage_id: format!("stage-{stage_index}"), + stage_index: stage_index as u32, + node_id: locked.node_id.clone(), + layer_start: locked.layer_start, + layer_end: locked.layer_end, + parameter_bytes, + }); + } + + let estimated_decode_network_ms_per_token = estimate_decode_network_ms_per_token(nodes); + Some(CandidatePlan { + plan: TopologyPlan { + context_length, + parallel_lanes, + stages, + estimated_decode_network_ms_per_token, + decode_tpot_target_met: decode_tpot_target_met( + estimated_decode_network_ms_per_token, + input.target_decode_tpot_ms, + ), + }, + minimum_remaining_vram, + total_remaining_vram, + }) +} diff --git a/docs/SKIPPY_SPLITS.md b/docs/SKIPPY_SPLITS.md index 5eb260fe9a..ceb8e465a7 100644 --- a/docs/SKIPPY_SPLITS.md +++ b/docs/SKIPPY_SPLITS.md @@ -80,6 +80,55 @@ curl -sS http://127.0.0.1:9447/v1/chat/completions \ -d '{"model":"meshllm/Qwen3-8B-Q4_K_M-layers","messages":[{"role":"user","content":"Reply with OK"}],"max_tokens":16}' ``` +## Lock node order and layer ranges + +Maintainer and benchmark runs can replace automatic placement with an exact, +fail-closed topology. Create the same JSON file on every serving node: + +```json +{ + "version": 1, + "model": "hf://meshllm/example-layers@immutable-revision", + "manifest_sha256": "", + "stages": [ + { + "node": "micstudio.local", + "layer_start": 0, + "layer_end": 31 + }, + { + "node": "studio54-3.local", + "layer_start": 31, + "layer_end": 47 + } + ] +} +``` + +`node` accepts either a full iroh endpoint id or an advertised hostname. Node +selectors must resolve uniquely among eligible split participants. Ranges must +be non-empty, contiguous, cover `0..layer_count`, and assign each node once. +The model and manifest digest must match the resolved package. + +Pass the lock together with `--split` on every node: + +```bash +mesh-llm serve \ + --model hf://meshllm/example-layers@immutable-revision \ + --split \ + --split-topology-lock /path/to/topology-lock.json +``` + +The normal context, KV-cache, headroom, and VRAM checks still apply. Startup +fails if the locked ranges do not fit. Once running, membership changes do not +replace the stages or collapse the model to a local fallback. If a locked stage +is lost, the topology becomes unavailable and is withdrawn after the normal +stage-loss grace period. + +Use `skippy-model-package preflight --verify-sha256` to obtain the +manifest digest for a local package. Confirm the realized assignments through +`GET /api/runtime/stages`. + ## Use a local GGUF Direct GGUFs still work: diff --git a/docs/design/TESTING.md b/docs/design/TESTING.md index 9270dc307f..f8344b1370 100644 --- a/docs/design/TESTING.md +++ b/docs/design/TESTING.md @@ -357,7 +357,9 @@ mesh-llm serve \ > > Use real flags only: `--split`, `--max-vram`, `--join`, `--bind-port`, > `--port`, `--console`. There is no `--layer-range` flag — layer placement -> is decided by the runtime from advertised VRAM, not pinned by CLI. +> is normally decided by the runtime from advertised VRAM. Maintainer runs that +> require exact placement can supply a complete topology manifest with +> `--split-topology-lock `; see `docs/SKIPPY_SPLITS.md`. #### Multi-interface Docker/Linux bind-IP validation From 9160b31420cd6f698a6ca74b1c4d81e113b8f7ee Mon Sep 17 00:00:00 2001 From: James Dumay Date: Wed, 22 Jul 2026 09:34:06 +1000 Subject: [PATCH 2/3] Document locked split topology --- website/src/docs/pages/CLI.md | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/website/src/docs/pages/CLI.md b/website/src/docs/pages/CLI.md index 663b0aaad4..234661f009 100644 --- a/website/src/docs/pages/CLI.md +++ b/website/src/docs/pages/CLI.md @@ -208,6 +208,56 @@ Switches: - `--trust-policy `: override peer ownership trust policy. - `--trust-owner `: add trusted owner IDs on top of the local trust store. +### Locked split topology + +Automatic split planning chooses nodes and layer boundaries from the capacity +currently advertised by the mesh. For controlled lab benchmarks, use a +topology lock so every host runs the same node order and layer ranges across +repeated runs, branches, and binaries. + +Copy the same versioned JSON file to every serving host: + +```json +{ + "version": 1, + "model": "hf://meshllm/example-layers@immutable-revision", + "manifest_sha256": "", + "stages": [ + { + "node": "micstudio.local", + "layer_start": 0, + "layer_end": 31 + }, + { + "node": "studio54-3.local", + "layer_start": 31, + "layer_end": 47 + } + ] +} +``` + +Then pass the lock with `--split` on every host: + +```bash +mesh-llm serve \ + --model hf://meshllm/example-layers@immutable-revision \ + --split \ + --split-topology-lock /path/to/topology-lock.json +``` + +The runtime verifies the resolved package and manifest digest, resolves each +node selector uniquely, requires contiguous ranges covering the full model, +and applies the normal context, KV-cache, headroom, and VRAM checks to those +exact assignments. A node selector may be a full iroh endpoint ID or an +advertised hostname. + +The lock is fail-closed, not a placement hint. If the requested topology cannot +be reproduced, startup fails. If an assigned stage is later lost, mesh-llm +withdraws the route after the normal grace period instead of replanning or +falling back to a local model. This prevents benchmark results from silently +mixing different execution topologies. + ### Speculative decoding overrides Advanced `serve` invocations can temporarily override a package or config-file From 1d02631465a0635dd6e2947ac0a682998cfe8c81 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Wed, 22 Jul 2026 09:38:08 +1000 Subject: [PATCH 3/3] Address locked topology review feedback --- .../src/runtime/split_planning.rs | 11 ++ crates/skippy-coordinator/src/topology.rs | 80 --------- .../skippy-coordinator/src/topology/locked.rs | 153 ++++++++++++++++++ docs/SKIPPY_SPLITS.md | 7 +- website/src/docs/pages/CLI.md | 3 +- 5 files changed, 170 insertions(+), 84 deletions(-) diff --git a/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs b/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs index eb1194f739..771e4df4c0 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/split_planning.rs @@ -214,6 +214,15 @@ pub(super) fn plan_locked_runtime_slice_topology_with_resources( resources: SplitTopologyResourceInputs, locked_stages: &[LockedSplitStageAssignment], ) -> Result { + tracing::info!( + topology_id, + model_ref, + participants = ?split_participant_labels(participants), + layer_count = package.layer_count, + native_context_length = resources.native_context_length, + "planning locked resource-aware split runtime topology" + ); + let participant_by_id = participant_index_by_id(participants); let locked_stages = locked_stages .iter() @@ -234,6 +243,8 @@ pub(super) fn plan_locked_runtime_slice_topology_with_resources( model_ref, context_length = plan.context_length, slots = plan.parallel_lanes, + estimated_decode_network_ms_per_token = plan.estimated_decode_network_ms_per_token, + decode_tpot_target_met = plan.decode_tpot_target_met, stages = ?split_stage_plan_labels(&stages), "validated locked split runtime topology" ); diff --git a/crates/skippy-coordinator/src/topology.rs b/crates/skippy-coordinator/src/topology.rs index aef5d55301..8f311861d9 100644 --- a/crates/skippy-coordinator/src/topology.rs +++ b/crates/skippy-coordinator/src/topology.rs @@ -721,86 +721,6 @@ mod tests { ); } - #[test] - fn locked_topology_preserves_node_order_and_layer_ranges() { - let mut request = input(vec![node("large", 48), node("small", 24)]); - request.minimum_nodes = 2; - let locked = vec![ - LockedTopologyStage { - node_id: "small".to_string(), - layer_start: 0, - layer_end: 12, - }, - LockedTopologyStage { - node_id: "large".to_string(), - layer_start: 12, - layer_end: 40, - }, - ]; - - let plan = plan_locked_topology(&request, &locked).unwrap(); - - assert_eq!( - plan.stages - .iter() - .map(|stage| (stage.node_id.as_str(), stage.layer_start, stage.layer_end)) - .collect::>(), - vec![("small", 0, 12), ("large", 12, 40)] - ); - } - - #[test] - fn locked_topology_rejects_non_contiguous_ranges() { - let mut request = input(vec![node("a", 48), node("b", 48)]); - request.minimum_nodes = 2; - let locked = vec![ - LockedTopologyStage { - node_id: "a".to_string(), - layer_start: 0, - layer_end: 19, - }, - LockedTopologyStage { - node_id: "b".to_string(), - layer_start: 20, - layer_end: 40, - }, - ]; - - assert_eq!( - plan_locked_topology(&request, &locked), - Err(TopologyPlanError::LockedNonContiguousRange { - stage_index: 1, - expected_start: 19, - actual_start: 20, - }) - ); - } - - #[test] - fn locked_topology_fails_when_pinned_stage_exceeds_node_capacity() { - let mut request = input(vec![node("small", 12), node("large", 48)]); - request.minimum_nodes = 2; - let locked = vec![ - LockedTopologyStage { - node_id: "small".to_string(), - layer_start: 0, - layer_end: 30, - }, - LockedTopologyStage { - node_id: "large".to_string(), - layer_start: 30, - layer_end: 40, - }, - ]; - - assert_eq!( - plan_locked_topology(&request, &locked), - Err(TopologyPlanError::LockedTopologyDoesNotFit { - minimum_context: 65_536, - }) - ); - } - #[test] fn applies_max_vram_and_runtime_headroom_per_node() { let mut capped = node("capped", 80); diff --git a/crates/skippy-coordinator/src/topology/locked.rs b/crates/skippy-coordinator/src/topology/locked.rs index af50d27657..4726d80489 100644 --- a/crates/skippy-coordinator/src/topology/locked.rs +++ b/crates/skippy-coordinator/src/topology/locked.rs @@ -162,3 +162,156 @@ fn fit_locked_candidate( total_remaining_vram, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::topology::TopologyNode; + + const GIB: u64 = 1024 * 1024 * 1024; + + fn node(id: &str, gib: u64) -> TopologyNode { + TopologyNode { + node_id: id.to_string(), + detected_vram_bytes: gib * GIB, + max_vram_bytes: None, + runtime_headroom_bytes: 0, + stage_transfer_latency_ms: None, + } + } + + fn input(nodes: Vec) -> TopologyPlanningInput { + TopologyPlanningInput { + native_context_length: 65_536, + layer_count: 40, + model_weight_bytes: 40 * GIB, + layer_weight_bytes: Vec::new(), + kv_bytes_per_token: 64 * 1024, + minimum_nodes: 2, + nodes, + context_length_override: None, + parallel_lanes_override: None, + target_decode_tpot_ms: None, + } + } + + fn stage(node_id: &str, layer_start: u32, layer_end: u32) -> LockedTopologyStage { + LockedTopologyStage { + node_id: node_id.to_string(), + layer_start, + layer_end, + } + } + + #[test] + fn preserves_node_order_and_layer_ranges() { + let request = input(vec![node("large", 48), node("small", 24)]); + let locked = vec![stage("small", 0, 12), stage("large", 12, 40)]; + + let plan = plan_locked_topology(&request, &locked).unwrap(); + + assert_eq!( + plan.stages + .iter() + .map(|stage| (stage.node_id.as_str(), stage.layer_start, stage.layer_end)) + .collect::>(), + vec![("small", 0, 12), ("large", 12, 40)] + ); + } + + #[test] + fn rejects_too_few_stages() { + let request = input(vec![node("a", 80), node("b", 80)]); + + assert_eq!( + plan_locked_topology(&request, &[stage("a", 0, 40)]), + Err(TopologyPlanError::LockedStageCount { + minimum: 2, + actual: 1, + }) + ); + } + + #[test] + fn rejects_unknown_node() { + let request = input(vec![node("a", 48), node("b", 48)]); + let locked = vec![stage("a", 0, 20), stage("missing", 20, 40)]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedUnknownNode { + node_id: "missing".to_string(), + }) + ); + } + + #[test] + fn rejects_duplicate_node() { + let request = input(vec![node("a", 48), node("b", 48)]); + let locked = vec![stage("a", 0, 20), stage("a", 20, 40)]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedDuplicateNode { + node_id: "a".to_string(), + }) + ); + } + + #[test] + fn rejects_invalid_range() { + let request = input(vec![node("a", 48), node("b", 48)]); + let locked = vec![stage("a", 0, 20), stage("b", 20, 20)]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedInvalidRange { + stage_index: 1, + start: 20, + end: 20, + }) + ); + } + + #[test] + fn rejects_non_contiguous_ranges() { + let request = input(vec![node("a", 48), node("b", 48)]); + let locked = vec![stage("a", 0, 19), stage("b", 20, 40)]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedNonContiguousRange { + stage_index: 1, + expected_start: 19, + actual_start: 20, + }) + ); + } + + #[test] + fn rejects_incomplete_coverage() { + let request = input(vec![node("a", 48), node("b", 48)]); + let locked = vec![stage("a", 0, 20), stage("b", 20, 39)]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedIncompleteCoverage { + actual_end: 39, + layer_count: 40, + }) + ); + } + + #[test] + fn fails_when_pinned_stage_exceeds_node_capacity() { + let request = input(vec![node("small", 12), node("large", 48)]); + let locked = vec![stage("small", 0, 30), stage("large", 30, 40)]; + + assert_eq!( + plan_locked_topology(&request, &locked), + Err(TopologyPlanError::LockedTopologyDoesNotFit { + minimum_context: 65_536, + }) + ); + } +} diff --git a/docs/SKIPPY_SPLITS.md b/docs/SKIPPY_SPLITS.md index ceb8e465a7..13bb69c177 100644 --- a/docs/SKIPPY_SPLITS.md +++ b/docs/SKIPPY_SPLITS.md @@ -106,9 +106,10 @@ fail-closed topology. Create the same JSON file on every serving node: ``` `node` accepts either a full iroh endpoint id or an advertised hostname. Node -selectors must resolve uniquely among eligible split participants. Ranges must -be non-empty, contiguous, cover `0..layer_count`, and assign each node once. -The model and manifest digest must match the resolved package. +selectors must resolve uniquely among eligible split participants. Ranges are +half-open: `layer_start` is inclusive and `layer_end` is exclusive. They must be +non-empty, contiguous, cover `0..layer_count`, and assign each node once. The +model and manifest digest must match the resolved package. Pass the lock together with `--split` on every node: diff --git a/website/src/docs/pages/CLI.md b/website/src/docs/pages/CLI.md index 234661f009..0b407a496a 100644 --- a/website/src/docs/pages/CLI.md +++ b/website/src/docs/pages/CLI.md @@ -250,7 +250,8 @@ The runtime verifies the resolved package and manifest digest, resolves each node selector uniquely, requires contiguous ranges covering the full model, and applies the normal context, KV-cache, headroom, and VRAM checks to those exact assignments. A node selector may be a full iroh endpoint ID or an -advertised hostname. +advertised hostname. Ranges are half-open: `layer_start` is inclusive and +`layer_end` is exclusive. The lock is fail-closed, not a placement hint. If the requested topology cannot be reproduced, startup fails. If an assigned stage is later lost, mesh-llm