Rust SDK: in-process mesh node from cargo (design + trial)

.github/workflows/release.yml

@@ -123,6 +123,11 @@ jobs:
           just --shell bash --shell-arg -c ${{ matrix.build_recipe }}
           just --shell bash --shell-arg -c ${{ matrix.bundle_recipe }} "$RELEASE_TAG" dist
 
+      - name: Package llama-stage tarball
+        env:
+          LLAMA_STAGE_BACKEND: ${{ matrix.backend }}
+        run: scripts/package-llama-stage.sh --out dist
+
       - name: Upload release bundle
         uses: actions/upload-artifact@v6
         with:
@@ -290,6 +295,10 @@ jobs:
         run: |
           just --shell bash --shell-arg -c release-build-arm64
           just --shell bash --shell-arg -c release-bundle-arm64 "$RELEASE_TAG" dist
+      - name: Package llama-stage tarball
+        env:
+          LLAMA_STAGE_BACKEND: cpu
+        run: scripts/package-llama-stage.sh --out dist
       - uses: actions/upload-artifact@v6
         with:
           name: release-linux-arm64
@@ -337,6 +346,12 @@ jobs:
         run: |
           just --shell bash --shell-arg -c release-build-cuda
           just --shell bash --shell-arg -c release-bundle-cuda "$RELEASE_TAG" dist
+      - name: Package llama-stage tarball
+        env:
+          LLAMA_STAGE_BACKEND: cuda
+        run: |
+          BUILD_DIR="$(LLAMA_STAGE_BACKEND=cuda scripts/build-llama.sh --print-build-dir)"
+          scripts/package-llama-stage.sh --build-dir "$BUILD_DIR" --out dist
       - uses: actions/upload-artifact@v6
         with:
           name: release-linux-cuda
@@ -384,6 +399,14 @@ jobs:
         run: |
           just --shell bash --shell-arg -c release-build-cuda-blackwell
           just --shell bash --shell-arg -c release-bundle-cuda-blackwell "$RELEASE_TAG" dist
+      - name: Package llama-stage tarball
+        env:
+          LLAMA_STAGE_BACKEND: cuda
+          CUDA_ARCH: '75;80;86;87;89;90;100;120'
+          MESH_LLAMA_STAGE_TARGET: x86_64-unknown-linux-gnu
+        run: |
+          BUILD_DIR="$(LLAMA_STAGE_BACKEND=cuda CUDA_ARCH="$CUDA_ARCH" scripts/build-llama.sh --print-build-dir)"
+          scripts/package-llama-stage.sh --build-dir "$BUILD_DIR" --backend cuda-blackwell --out dist
       - uses: actions/upload-artifact@v6
         with:
           name: release-linux-cuda-blackwell
@@ -431,6 +454,12 @@ jobs:
         run: |
           just --shell bash --shell-arg -c release-build-rocm
           just --shell bash --shell-arg -c release-bundle-rocm "$RELEASE_TAG" dist
+      - name: Package llama-stage tarball
+        env:
+          LLAMA_STAGE_BACKEND: rocm
+        run: |
+          BUILD_DIR="$(LLAMA_STAGE_BACKEND=rocm scripts/build-llama.sh --print-build-dir)"
+          scripts/package-llama-stage.sh --build-dir "$BUILD_DIR" --out dist
       - uses: actions/upload-artifact@v6
         with:
           name: release-linux-rocm
@@ -466,6 +495,10 @@ jobs:
         run: |
           just --shell bash --shell-arg -c release-build-vulkan
           just --shell bash --shell-arg -c release-bundle-vulkan "$RELEASE_TAG" dist
+      - name: Package llama-stage tarball
+        env:
+          LLAMA_STAGE_BACKEND: vulkan
+        run: scripts/package-llama-stage.sh --out dist
       - uses: actions/upload-artifact@v6
         with:
           name: release-linux-vulkan

.gitignore

@@ -20,6 +20,9 @@ __pycache__/
 .venv/
 .pytest_cache/
 dist/MeshLLMFFI.xcframework.zip
+dist/native-sdk/
+dist/native-sdk-static/
+dist/llama-stage-static/
 sdk/kotlin/src/main/kotlin/uniffi/
 sdk/kotlin/example/example-jvm/src/main/kotlin/uniffi/
 sdk/node/native/
@@ -28,3 +31,5 @@ sdk/swift/Sources/MeshLLM/Generated/*
 sdk/swift/Generated/FFI/
 sdk/swift/Generated/MeshLLMFFI.xcframework/
 .impeccable.md
+examples/rust-sdk-trial/Cargo.lock
+examples/rust-sdk-trial/target/

crates/llama-spec-bench/Cargo.toml

@@ -7,6 +7,6 @@ version.workspace = true
 [dependencies]
 anyhow.workspace = true
 clap.workspace = true
-skippy-runtime = { path = "../skippy-runtime" }
+skippy-runtime = { version = "0.66.0", path = "../skippy-runtime" }
 serde.workspace = true
 serde_json.workspace = true

crates/mesh-llm-api-server/Cargo.toml

@@ -13,11 +13,25 @@ categories = ["api-bindings", "network-programming"]
 [features]
 host-io = ["mesh-llm-api-client/host-io"]
 
+# Run a real iroh-backed mesh node in-process (gossip, relay registration,
+# invite tokens, QUIC peer connections) instead of the default HTTP-shim
+# client behaviour. Drags `mesh-llm-host-runtime` and its transitive deps
+# (skippy, llama.cpp link path, etc.) into the build, so it is off by
+# default. Consumers who want a Rust app to act as a real mesh peer
+# should enable it.
+#
+# Note: gated iroh-relay (--relay-auth URL=TOKEN) support is a separate,
+# in-flight piece of work; once it lands, this feature will pick it up
+# transparently with no consumer-side change.
+host-runtime = ["dep:mesh-llm-host-runtime"]
+
 [dependencies]
 anyhow.workspace = true
 mesh-llm-api-client = { path = "../mesh-llm-api-client", version = "0.66.0" }
 mesh-llm-node = { path = "../mesh-llm-node", version = "0.66.0" }
+# Optional, gated by the `host-runtime` feature.
+mesh-llm-host-runtime = { path = "../mesh-llm-host-runtime", optional = true, default-features = false }
 tokio = { version = "1", features = ["sync"] }
 
 [dev-dependencies]
-tokio = { version = "1", features = ["macros", "rt"] }
+tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "time"] }

crates/mesh-llm-api-server/README.md

@@ -33,3 +33,48 @@ high-level serving errors.
 If an API is meant for client-only app integration, it belongs in
 `mesh-llm-api-client`. If it requires model management or local serving, it
 belongs in `mesh-llm-api-server`.
+
+## Running the full mesh-llm runtime in-process (`host-runtime` feature)
+
+For applications that want to run **exactly what `mesh-llm serve` /
+`mesh-llm client` does** — not just consume mesh inference, but be the
+running node — enable the `host-runtime` feature:
+
+```toml
+mesh-llm-api-server = { version = "0.66.0", features = ["host-runtime"] }
+```
+
+Then call `run_serve(MeshServeSpec { ... })`:
+
+```rust
+use mesh_llm_api_server::{run_serve, MeshServeSpec};
+
+run_serve(MeshServeSpec {
+    client: true,
+    auto: true,
+    relays: vec!["https://public.example/".into()],
+    port: Some(9337),
+    console_port: Some(3131),
+    headless: true,
+    max_vram_gb: Some(0.0),
+    ..Default::default()
+})
+.await?;
+```
+
+(Gated iroh-relay support — per-relay bearer tokens via `--relay-auth
+URL=TOKEN` and a `relay_auths` field on `MeshServeSpec` — lives on the
+separate gated-relay PR. Once that lands, this snippet will gain the
+`HashMap` shape used by `mesh-llm`'s CLI today.)
+
+This drives the same `runtime::run_with_args` entry point the binary
+uses. You get auto-discovery, election, tunnel manager, OpenAI HTTP
+proxy on `--port`, management console on `--console`, local model
+serving (when configured), plugin host — the entire mesh-llm runtime
+inside your process.
+
+`MeshNode::builder()` (`host-runtime` feature also required for the
+fine-grained options like `.relay(...)`) is the
+composable alternative for apps that want to wire pieces themselves
+rather than running the whole orchestration. See `docs/SDK.md` for the
+full comparison.

crates/mesh-llm-api-server/src/lib.rs

@@ -13,11 +13,55 @@ pub use mesh_llm_api_client::{
     ResponsesRequest, Status,
 };
 pub use mesh_llm_node::serving::ServingController;
+
+/// Run the full mesh-llm runtime in-process — the same code path the
+/// `mesh-llm` binary runs. Only available with the `host-runtime` feature.
+///
+/// This is the SDK entry point for embedders who want their Rust app to
+/// act exactly like running `mesh-llm serve` or `mesh-llm client` —
+/// with auto-discovery, election, tunnel manager, OpenAI HTTP proxy,
+/// management console, and local model serving (when configured) —
+/// without spawning the binary as a subprocess.
+///
+/// # Example
+///
+/// ```no_run
+/// use mesh_llm_api_server::{run_serve, MeshServeSpec};
+///
+/// # async fn run() -> anyhow::Result<()> {
+/// run_serve(MeshServeSpec {
+///     // Same flags `mesh-llm serve` / `mesh-llm client` accept.
+///     client: true,                       // false (default) = serve role
+///     auto: true,                         // == --auto
+///     relays: vec!["https://public.example/".into()],
+///     port: Some(9337),                   // OpenAI HTTP proxy port
+///     console_port: Some(3131),           // management API / web console
+///     headless: true,                     // skip embedded web UI
+///     max_vram_gb: Some(0.0),             // client-only, no VRAM advert
+///     ..MeshServeSpec::default()
+/// })
+/// .await?;
+/// # Ok(())
+/// # }
+/// ```
+///
+/// (Gated iroh-relay support — per-relay bearer tokens via
+/// `--relay-auth URL=TOKEN` and a `relay_auths` field on
+/// `MeshServeSpec` — lives on the separate gated-relay PR.)
+///
+/// The future blocks until the runtime exits. The runtime is not
+/// currently `Send`-clean; if you need concurrent work, run on a
+/// `tokio::task::LocalSet` rather than `tokio::spawn`.
+///
+/// For finer-grained control — composing pieces without running the
+/// whole orchestration — see [`MeshNodeBuilder`] instead.
+#[cfg(feature = "host-runtime")]
+pub use mesh_llm_host_runtime::host_node::{MeshServeSpec, run_serve};
 pub use node::{
     CapabilityLevel, CleanupPolicy, CleanupResult, DeleteModelOptions, DeleteModelResult,
     DevicePolicy, DownloadId, DownloadOptions, DownloadedModel, InstalledModel, LoadModelOptions,
-    MeshEvents, MeshInference, MeshModels, MeshNode, MeshNodeBuilder, MeshNodeConfig, MeshServing,
-    MeshStatusApi, ModelCacheStatus, ModelCapabilities, ModelDetails, ModelKind, ModelSearchQuery,
-    ModelSource, ModelSummary, PrunePolicy, PruneResult, ServedModel, ServingModelState,
-    ServingStatus, UnloadModelOptions, UnloadTarget,
+    MeshEvents, MeshInference, MeshModels, MeshNode, MeshNodeBuilder, MeshNodeConfig, MeshQuicBind,
+    MeshRole, MeshServing, MeshStatusApi, ModelCacheStatus, ModelCapabilities, ModelDetails,
+    ModelKind, ModelSearchQuery, ModelSource, ModelSummary, PrunePolicy, PruneResult, ServedModel,
+    ServingModelState, ServingStatus, UnloadModelOptions, UnloadTarget,
 };