Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions crates/mesh-llm-cli/src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,44 @@ pub fn assert_mesh_requirements_docs_examples_parse() {

#[cfg(test)]
mod tests {
use super::Cli;
use clap::Parser;

#[test]
fn mesh_requirements_docs_examples_parse() {
super::assert_mesh_requirements_docs_examples_parse();
}

#[test]
fn split_topology_lock_requires_split_mode() {
let error = Cli::try_parse_from([
"mesh-llm",
"--model",
"model.gguf",
"--split-topology-lock",
"topology.json",
])
.expect_err("topology lock without --split should fail");

assert!(error.to_string().contains("--split"));
}

#[test]
fn split_topology_lock_parses_with_split_mode() {
let cli = Cli::try_parse_from([
"mesh-llm",
"--model",
"model.gguf",
"--split",
"--split-topology-lock",
"topology.json",
])
.expect("locked split CLI should parse");

assert!(cli.split);
assert_eq!(
cli.split_topology_lock,
Some(std::path::PathBuf::from("topology.json"))
);
}
}
4 changes: 4 additions & 0 deletions crates/mesh-llm-cli/src/parser/commands.rs
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,10 @@ pub struct Cli {
#[arg(long, hide = true)]
pub split: bool,

/// Pin split-serving node order and layer ranges from a JSON topology lock.
#[arg(long, value_name = "PATH", requires = "split", hide = true)]
pub split_topology_lock: Option<PathBuf>,

/// Override context size (tokens). Default: auto-scaled to available VRAM.
#[arg(long, hide = true)]
pub ctx_size: Option<u32>,
Expand Down
1 change: 1 addition & 0 deletions crates/mesh-llm-host-runtime/src/runtime/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ pub(super) struct LocalRuntimeModelStartSpec<'a> {
pub(super) n_ubatch_override: Option<u32>,
pub(super) flash_attention_override: FlashAttentionType,
pub(super) parallel_override: Option<usize>,
pub(super) split_topology_lock: Option<&'a Path>,
pub(super) planning_profile: RuntimeResourcePlanningProfile,
pub(super) openai_guardrail_policy: OpenAiGuardrailPolicyHandle,
pub(super) skippy_telemetry: skippy::SkippyTelemetryOptions,
Expand Down
61 changes: 45 additions & 16 deletions crates/mesh-llm-host-runtime/src/runtime/local_split.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
mod coordinator;
mod loading;
mod recovery;
#[cfg(test)]
mod test_support;
#[cfg(test)]
Expand All @@ -18,9 +19,11 @@ use super::local_package::{
};
use super::split_planning::{
PlannedRuntimeSliceTopology, RuntimeSliceStagePlan, SplitTopologyResourceInputs,
plan_runtime_slice_topology_with_resources, split_participant_exclusion_labels,
split_participant_labels, split_participants_for_stages, split_stage_plan_labels,
plan_locked_runtime_slice_topology_with_resources, plan_runtime_slice_topology_with_resources,
split_participant_exclusion_labels, split_participant_labels, split_participants_for_stages,
split_stage_plan_labels,
};
use super::split_topology_lock::load_locked_split_assignments;
use crate::inference::skippy;
use crate::mesh;
use crate::models;
Expand All @@ -34,8 +37,9 @@ const SPLIT_COORDINATOR_LEASE_SECS: u64 = 4 * 60 * 60;

use coordinator::{
SplitTopologyCoordinator, SplitTopologyGeneration, spawn_split_topology_coordinator,
split_stages_meet_minimum, stop_split_generation,
stop_split_generation,
};
use recovery::split_stages_meet_minimum;

fn split_coordinator_lease_until_unix_ms() -> u64 {
super::local::current_time_unix_ms()
Expand Down Expand Up @@ -150,6 +154,7 @@ pub(super) async fn start_runtime_split_model(
compact_meta,
kv_bytes_per_token,
planned_topology,
topology_locked,
} = split_setup;
let stages = planned_topology.stages;
let planned_participants =
Expand Down Expand Up @@ -255,6 +260,7 @@ pub(super) async fn start_runtime_split_model(
survey_telemetry: spec.survey_telemetry.clone(),
event_tx: coordinator_tx,
stage_loss_first_seen: None,
topology_locked,
}));

Ok(SplitRuntimeStart::Started(Box::new(loaded)))
Expand All @@ -266,6 +272,7 @@ struct SplitRuntimeStartPreparation {
compact_meta: models::gguf::GgufCompactMeta,
kv_bytes_per_token: u64,
planned_topology: PlannedRuntimeSliceTopology,
topology_locked: bool,
}

async fn prepare_split_runtime_start(
Expand All @@ -291,25 +298,47 @@ async fn prepare_split_runtime_start(
spec.cache_type_k_override,
spec.cache_type_v_override,
)?;
let planned_topology = plan_runtime_slice_topology_with_resources(
topology_id,
model_ref,
&package,
&participant_snapshot.participants,
&participant_snapshot.excluded,
SplitTopologyResourceInputs {
native_context_length: compact_meta.context_length,
kv_bytes_per_token,
ctx_size_override: spec.ctx_size_override,
parallel_override: spec.parallel_override,
},
)?;
let resources = SplitTopologyResourceInputs {
native_context_length: compact_meta.context_length,
kv_bytes_per_token,
ctx_size_override: spec.ctx_size_override,
parallel_override: spec.parallel_override,
};
let planned_topology = if let Some(path) = spec.split_topology_lock {
let locked_stages = load_locked_split_assignments(
path,
spec.node,
model_ref,
&package,
&participant_snapshot.participants,
)
.await?;
plan_locked_runtime_slice_topology_with_resources(
topology_id,
model_ref,
&package,
&participant_snapshot.participants,
&participant_snapshot.excluded,
resources,
&locked_stages,
)?
} else {
plan_runtime_slice_topology_with_resources(
topology_id,
model_ref,
&package,
&participant_snapshot.participants,
&participant_snapshot.excluded,
resources,
)?
};
Ok(SplitRuntimeStartPreparation {
package,
participant_snapshot,
compact_meta,
kv_bytes_per_token,
planned_topology,
topology_locked: spec.split_topology_lock.is_some(),
})
}

Expand Down
Loading
Loading