Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 115 additions & 78 deletions mesh-llm/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,131 +21,122 @@ use std::path::PathBuf;
pub const VERSION: &str = "0.42.0";

#[derive(Parser, Debug)]
#[command(name = "mesh-llm", version = VERSION, about = "P2P mesh for distributed llama.cpp inference over QUIC")]
#[command(name = "mesh-llm", version = VERSION,
about = "Pool GPUs over the internet for LLM inference",
after_help = "Run with --help-advanced for all options.")]
struct Cli {
#[command(subcommand)]
command: Option<Command>,

/// Join an existing mesh via an invite token.
/// Can be specified multiple times — only one needs to be reachable.
/// Show all options (including advanced/niche ones).
#[arg(long, hide = true)]
help_advanced: bool,

/// Join a mesh via invite token (can repeat).
#[arg(long, short)]
join: Vec<String>,

/// Discover a mesh from Nostr and join it automatically.
/// Optionally specify a model name to filter by.
/// Discover a mesh via Nostr and join it.
#[arg(long, default_missing_value = "", num_args = 0..=1)]
discover: Option<String>,

/// Auto-join: discover a mesh via Nostr and join it.
/// Equivalent to: mesh-llm --join $(mesh-llm discover --auto)
/// Auto-join the best mesh found via Nostr.
#[arg(long)]
auto: bool,

/// GGUF model(s) for this mesh. Can be a path, catalog name, or HuggingFace URL.
/// First model is served by this node; additional models are wanted by the mesh
/// (other nodes joining will pick them up if they have them on disk).
/// When joining without --model, the mesh assigns one automatically.
/// Model to serve (path, catalog name, or HuggingFace URL).
#[arg(long)]
model: Vec<PathBuf>,

/// Local HTTP port for the API (default: 9337).
/// The elected host runs llama-server here; workers proxy to the host.
/// API port (default: 9337).
#[arg(long, default_value = "9337")]
port: u16,

/// Path to directory containing rpc-server and llama-server binaries.
/// Defaults to the same directory as the mesh-llm binary itself.
/// Run as a client — no GPU, no model needed.
#[arg(long)]
bin_dir: Option<PathBuf>,
client: bool,

/// Device for rpc-server (e.g. MTL0, CPU). Default: auto-detect.
/// Web console port (default: 3131).
#[arg(long, default_value = "3131")]
console: u16,

/// Publish this mesh for discovery by others.
#[arg(long)]
device: Option<String>,
publish: bool,

/// Tensor split ratios for llama-server (e.g. "0.8,0.2").
/// Without this, split is auto-calculated from VRAM.
/// Name for this mesh (shown in discovery).
#[arg(long)]
tensor_split: Option<String>,
mesh_name: Option<String>,

/// Run as a lite client — no GPU, no rpc-server, no model needed.
/// Region tag, e.g. "US", "EU", "AU" (shown in discovery).
#[arg(long)]
client: bool,
region: Option<String>,

/// Enable the blackboard for sharing context across agents.
#[arg(long)]
blackboard: bool,

/// Path to a draft model for speculative decoding (e.g. a small quant of the same model).
/// Only used on the host — the draft model runs locally, not distributed.
/// If omitted, auto-detected from catalog when the main model has a known draft pairing.
/// Your display name on the blackboard.
#[arg(long)]
name: Option<String>,

// ── Advanced options (hidden from default --help) ─────────────

/// Draft model for speculative decoding.
#[arg(long, hide = true)]
draft: Option<PathBuf>,

/// Max draft tokens for speculative decoding (default: 8).
#[arg(long, default_value = "8")]
/// Max draft tokens (default: 8).
#[arg(long, default_value = "8", hide = true)]
draft_max: u16,

/// Disable automatic draft model detection from catalog.
#[arg(long)]
/// Disable automatic draft model detection.
#[arg(long, hide = true)]
no_draft: bool,

/// Force tensor split across all GPU nodes even if the model fits on the host.
/// Without this, the host loads solo when it has enough VRAM.
#[arg(long)]
/// Force tensor split even if the model fits on one node.
#[arg(long, hide = true)]
split: bool,

/// Limit VRAM advertised to the mesh (in GB). Other nodes will see this
/// instead of your actual VRAM, capping how much work gets split to you.
#[arg(long)]
/// Limit VRAM advertised to the mesh (GB).
#[arg(long, hide = true)]
max_vram: Option<f64>,

/// Override iroh relay URLs (e.g. --relay https://staging-use1-1.relay.iroh.network./).
/// Can be specified multiple times. Without this, iroh uses its built-in defaults.
#[arg(long)]
/// Path to rpc-server and llama-server binaries.
#[arg(long, hide = true)]
bin_dir: Option<PathBuf>,

/// Device for rpc-server (e.g. MTL0, CPU).
#[arg(long, hide = true)]
device: Option<String>,

/// Tensor split ratios for llama-server (e.g. "0.8,0.2").
#[arg(long, hide = true)]
tensor_split: Option<String>,

/// Override iroh relay URLs.
#[arg(long, hide = true)]
relay: Vec<String>,

/// Bind QUIC to a fixed UDP port (for NAT port forwarding).
#[arg(long)]
#[arg(long, hide = true)]
bind_port: Option<u16>,

/// Web console port (default: 3131).
#[arg(long, default_value = "3131")]
console: u16,

/// Ignored (kept for backward compatibility).
/// Bind to 0.0.0.0 (for containers/Fly.io).
#[arg(long, hide = true)]
no_console: bool,

/// Bind API and console to 0.0.0.0 instead of 127.0.0.1.
/// Use for containers (Docker, Fly.io) where external access is needed.
#[arg(long)]
listen_all: bool,

/// Publish this mesh to Nostr for discovery by others.
/// Republishes every 60s so the listing stays fresh.
#[arg(long)]
publish: bool,

/// Human-readable name for this mesh (shown in discovery).
#[arg(long)]
mesh_name: Option<String>,

/// Geographic region tag (e.g. "US", "EU", "AU"). Shown in discovery.
#[arg(long)]
region: Option<String>,

/// Stop advertising on Nostr when this many clients are connected.
/// Re-publishes when clients drop below the cap. No cap by default.
#[arg(long)]
/// Stop advertising when N clients connected.
#[arg(long, hide = true)]
max_clients: Option<usize>,

/// Nostr relay URLs for publishing/discovery (default: damus, nos.lol, nostr.band).
#[arg(long)]
/// Custom Nostr relay URLs.
#[arg(long, hide = true)]
nostr_relay: Vec<String>,

/// Enable the blackboard — agents share status, findings, and questions across the mesh. Works on any node (with or without a model).
#[arg(long)]
blackboard: bool,

/// Your display name on the blackboard (default: short node ID).
#[arg(long)]
name: Option<String>,
/// Ignored (backward compat).
#[arg(long, hide = true)]
no_console: bool,

/// Internal: set when this node joined via Nostr discovery (not --join).
#[arg(skip)]
Expand All @@ -162,15 +153,16 @@ enum Command {
#[arg(long)]
draft: bool,
},
/// Drop a model from the mesh — stops all nodes serving it
/// Drop a model from the mesh.
#[command(hide = true)]
Drop {
/// Model name to drop
name: String,
/// API port of the running mesh-llm instance (default: 9337)
#[arg(long, default_value = "9337")]
port: u16,
},
/// Discover meshes published to Nostr and optionally auto-join one
/// Discover meshes on Nostr and optionally auto-join one.
Discover {
/// Filter by model name (substring match)
#[arg(long)]
Expand All @@ -188,7 +180,8 @@ enum Command {
#[arg(long)]
relay: Vec<String>,
},
/// Rotate the Nostr identity key (forces new keypair on next --publish)
/// Rotate the Nostr identity key.
#[command(hide = true)]
RotateKey,
/// Launch Goose with mesh-llm as the inference provider.
///
Expand All @@ -214,6 +207,8 @@ enum Command {
#[arg(long, default_value = "9337")]
port: u16,
},
/// Stop all running mesh-llm, llama-server, and rpc-server processes.
Stop,
/// Blackboard — post, search, and read messages shared across the mesh.
///
/// Post a message: mesh-llm blackboard "your message here"
Expand Down Expand Up @@ -262,6 +257,25 @@ async fn main() -> Result<()> {
.with_writer(std::io::stderr)
.init();

// --help-advanced: print full help with all hidden options and commands visible
if std::env::args().any(|a| a == "--help-advanced") {
use clap::CommandFactory;
let mut cmd = Cli::command();
// Unhide all arguments
let args: Vec<clap::Id> = cmd.get_arguments().map(|a| a.get_id().clone()).collect();
for id in args {
cmd = cmd.mut_arg(id, |a| a.hide(false));
}
// Unhide all subcommands
let sub_names: Vec<String> = cmd.get_subcommands().map(|s| s.get_name().to_string()).collect();
for name in sub_names {
cmd = cmd.mut_subcommand(name, |s| s.hide(false));
}
cmd.print_help().ok();
eprintln!();
std::process::exit(0);
}

let mut cli = Cli::parse();

// Clean up orphan processes from previous runs (skip for client — never runs llama-server)
Expand Down Expand Up @@ -301,6 +315,9 @@ async fn main() -> Result<()> {
Command::Drop { name, port } => {
return run_drop(name, *port).await;
}
Command::Stop => {
return run_stop();
}
Command::Discover { model, min_vram, region, auto, relay } => {
return run_discover(model.clone(), *min_vram, region.clone(), *auto, relay.clone()).await;
}
Expand Down Expand Up @@ -1960,6 +1977,26 @@ async fn run_discover(
}

/// Drop a model from the mesh by sending a control request to the running instance.
fn run_stop() -> Result<()> {
use std::process::Command as Cmd;
let mut killed = 0u32;
for name in &["mesh-llm", "llama-server", "rpc-server"] {
// pkill sends SIGTERM; ignore errors (process might not exist)
let status = Cmd::new("pkill").arg("-f").arg(name).status();
match status {
Ok(s) if s.success() => {
eprintln!("🧹 Stopped {name}");
killed += 1;
}
_ => {}
}
}
if killed == 0 {
eprintln!("Nothing running.");
}
Ok(())
}

async fn run_drop(model_name: &str, port: u16) -> Result<()> {
use tokio::io::{AsyncReadExt, AsyncWriteExt};

Expand Down
Loading