From 1da30eda77a9d0bb3c148ac446313bfb6ba8e4cc Mon Sep 17 00:00:00 2001 From: James Dumay Date: Mon, 13 Jul 2026 12:45:17 +1000 Subject: [PATCH 1/2] Document architecture and model packages --- website/.eleventy.js | 3 + website/src/_data/docs.js | 12 + website/src/_includes/docs-base.njk | 1 + website/src/assets/mermaid.js | 49 +++ website/src/assets/site.css | 16 + website/src/docs/pages/architecture.md | 169 +++++++++++ .../docs/pages/contributing-layer-packages.md | 2 +- website/src/docs/pages/model-package-spec.md | 284 ++++++++++++++++++ .../src/docs/pages/running-large-models.md | 17 ++ 9 files changed, 552 insertions(+), 1 deletion(-) create mode 100644 website/src/assets/mermaid.js create mode 100644 website/src/docs/pages/architecture.md create mode 100644 website/src/docs/pages/model-package-spec.md diff --git a/website/.eleventy.js b/website/.eleventy.js index ea0ac243e0..ca1ea9848c 100644 --- a/website/.eleventy.js +++ b/website/.eleventy.js @@ -51,6 +51,9 @@ export default function(eleventyConfig) { console.debug("highlight.js error for lang=%s: %s", hl, e); } } + if (lang === "mermaid") { + return `
${md.utils.escapeHtml(str)}
`; + } return `
${md.utils.escapeHtml(str)}
`; }, }); diff --git a/website/src/_data/docs.js b/website/src/_data/docs.js index 3ef5a3574c..5e2e5fdf3a 100644 --- a/website/src/_data/docs.js +++ b/website/src/_data/docs.js @@ -69,6 +69,18 @@ export default [ ["Publish mesh", "/docs/pages/publish-mesh/"] ] }, + { + title: "Architecture", + description: "Understand node roles, mesh routing, Skippy stages, model artifacts, and subsystem ownership.", + links: [ + ["Architecture hub", "/docs/pages/architecture/"], + ["Mesh workflows", "/docs/pages/private-meshes/"], + ["Large-model splits", "/docs/pages/running-large-models/"], + ["Model package spec", "/docs/pages/model-package-spec/"], + ["Plugin architecture", "/docs/pages/plugin-architecture/"], + ["SDK embedding", "/docs/pages/sdk/"] + ] + }, { title: "Integrations", description: "Connect agent tools and OpenAI-compatible applications.", diff --git a/website/src/_includes/docs-base.njk b/website/src/_includes/docs-base.njk index 5b8bf0d23c..68dba65572 100644 --- a/website/src/_includes/docs-base.njk +++ b/website/src/_includes/docs-base.njk @@ -11,6 +11,7 @@ + diff --git a/website/src/assets/mermaid.js b/website/src/assets/mermaid.js new file mode 100644 index 0000000000..02b0fa1193 --- /dev/null +++ b/website/src/assets/mermaid.js @@ -0,0 +1,49 @@ +async function renderMermaid() { + const blocks = document.querySelectorAll("pre > code.language-mermaid"); + + if (!blocks.length) { + return; + } + + const { default: mermaid } = await import( + "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs" + ); + + mermaid.initialize({ + startOnLoad: false, + securityLevel: "strict", + theme: "dark", + flowchart: { + curve: "basis", + htmlLabels: true, + }, + sequence: { + useMaxWidth: true, + }, + }); + + const nodes = []; + + blocks.forEach((code) => { + const container = document.createElement("div"); + const pre = code.parentElement; + const frame = pre?.parentElement?.classList.contains("code-copy-frame") + ? pre.parentElement + : pre; + + container.className = "mermaid"; + container.setAttribute("role", "img"); + container.setAttribute("aria-label", "Architecture diagram"); + container.textContent = code.textContent; + frame?.replaceWith(container); + nodes.push(container); + }); + + mermaid.run({ nodes }); +} + +if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", renderMermaid, { once: true }); +} else { + renderMermaid(); +} diff --git a/website/src/assets/site.css b/website/src/assets/site.css index d93f1628d9..c846a9e83a 100644 --- a/website/src/assets/site.css +++ b/website/src/assets/site.css @@ -4503,6 +4503,22 @@ html:has(> body.docs-body)::-webkit-scrollbar-corner, color: var(--fg); } +.docs-body .doc .mermaid { + margin: 24px 0; + overflow-x: auto; + padding: 16px; + border: 1px solid var(--line-2); + border-radius: 8px; + background: #07090d; + text-align: center; +} + +.docs-body .doc .mermaid svg { + display: inline-block; + max-width: 100%; + height: auto; +} + .docs-body .doc .code-copy-frame { --docs-copy-button-space: 42px; } diff --git a/website/src/docs/pages/architecture.md b/website/src/docs/pages/architecture.md new file mode 100644 index 0000000000..c13c77fa8b --- /dev/null +++ b/website/src/docs/pages/architecture.md @@ -0,0 +1,169 @@ +--- +title: Architecture +--- + +# Mesh LLM architecture + +Mesh LLM turns several machines into one OpenAI-compatible inference surface. Each node can expose an API, participate in discovery and gossip, serve local models, route requests to peers, or provide compute for a model split. + +This page is the map. It explains the boundaries between the mesh product, the Skippy execution runtime, and the model artifacts they consume. Use the linked deep dives when you need protocol fields, operational commands, or implementation details. + +## The one-minute mental model + +```mermaid +flowchart TD + App["Application
OpenAI client · SDK · console · plugin"] + APIs["Node APIs
9337 /v1
3131 /api"] + Host["Mesh host runtime
identity · discovery · gossip
routing · models · lifecycle"] + Local["Local execution
one Skippy stage"] + Split["Split execution
Skippy stage pipeline"] + Artifacts["Artifacts
GGUF · layer packages · native runtimes"] + + App --> APIs --> Host + Host --> Local + Host --> Split + Artifacts --> Local + Artifacts --> Split +``` + +The important boundary is that mesh-llm owns the product and network behavior, while Skippy owns model execution. The OpenAI API stays stable whether the request is handled locally, by a peer, or by a multi-stage pipeline. + +## What happens to a request + +1. An application sends an OpenAI-compatible request to the node's `/v1` endpoint. SDK clients may use direct mesh transport instead of a local HTTP listener. +2. The node classifies the request and reads the requested model, capability, and routing signals. +3. The router selects a local target, a peer host, or a stage-0 target for a split model. Model availability, hardware fit, demand, request affinity, and health all contribute to the decision. +4. If the target is remote, the node uses the mesh QUIC transport and a tunnel or route request to reach it. The caller still sees one local OpenAI endpoint. +5. If the target is split, stage 0 runs the input and sends activations downstream through Skippy stage transport. Later stages send results back toward the driver, which streams the OpenAI response. +6. Runtime status, model state, routing observations, and events are published to the management API and telemetry surfaces without changing the inference contract. + +```mermaid +sequenceDiagram + participant App as Application + participant Node as Local node + participant Router as Router / election + participant Peer as Peer or stage 0 + participant Stage as Downstream stages + + App->>Node: POST /v1/chat/completions + Node->>Router: Classify model and request + Router-->>Node: Local target or remote target + alt Local model fits + Node->>Peer: Execute in local Skippy stage + else Remote or split target + Node->>Peer: Route over QUIC + Peer->>Stage: Send activation frames + Stage-->>Peer: Generated tokens / activations + end + Peer-->>Node: OpenAI-compatible stream + Node-->>App: SSE or JSON response +``` + +## How the mesh works + +### Nodes and roles + +Nodes are peers with a stable owner identity and mesh membership. Their runtime role depends on what they are doing: + +| Role | Responsibility | +| --- | --- | +| Client | Consumes inference without contributing local model execution. | +| Host | Owns a routable model target and exposes the local inference API. | +| Worker | Provides compute or a stage for a model execution plan. | +| Standby | Has useful capacity or model state and can be promoted when demand or topology changes. | + +Roles and live serving state are different concepts. A node can be connected while its model is loading, an endpoint is unhealthy, or its capacity is waiting for election. + +### Discovery, admission, and gossip + +Public meshes are discovered through published listings. Private meshes are joined with invite tokens; LAN deployments can use the configured local discovery mode. After transport negotiation, nodes exchange additive gossip containing peer identity, capabilities, model visibility, serving state, demand signals, and mesh metadata. + +Discovery finds a candidate mesh. Admission decides whether the node is allowed to participate. Gossip tells admitted peers what the topology currently looks like. These are separate stages and should not be treated as one trust decision. + +### Transport and compatibility + +Mesh transport uses QUIC through iroh. A connection is multiplexed into control, gossip, route, tunnel, and lifecycle streams. The current protobuf lane uses ALPN `mesh-llm/1`; legacy JSON peers can still negotiate `mesh-llm/0` for mixed-version operation. + +Protocol changes should be additive: older nodes must be able to ignore new optional fields, and newer nodes must continue to understand the legacy lane where compatibility is required. See the [protocol deep dive](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/design/message_protocol.md) for the wire-level contract. + +### Routing and election + +Every node exposes the same OpenAI-facing shape. A request is routed by model identity rather than by a user selecting a machine. Per-model election decides which node or stage-0 target is authoritative, while passive clients receive a smaller route view instead of full mesh gossip. + +Routing also considers request affinity. Reusing a target for a shared prefix can preserve cache locality, while health and topology changes can drain or replace a target. The router is advisory only until a target is healthy and ready; a process that has merely spawned is not routable. + +## How Skippy works + +Skippy is the embedded staged runtime used for local execution and large-model splits. It provides the model/session runtime, layer topology primitives, stage protocol, activation wire encoding, and package materialization. + +### Single-node execution + +If one node can fit the model, the host runtime starts one Skippy stage in-process. The stage owns model loading, token generation, sessions, KV state, and backend execution. Mesh still owns the public API, model identity, lifecycle, status, and routing decisions around it. + +### Multi-node stage execution + +If the model is too large for one node, a layer package supplies the shared and per-layer artifacts needed to construct contiguous stages: + +```mermaid +flowchart LR + Input["Request
tokens + session state"] --> S0["Stage 0
embeddings + layers 0..N
OpenAI driver"] + S0 -->|"activation frames"| S1["Stage 1
layers N+1..M"] + S1 -->|"activation frames"| S2["Stage 2
layers M+1..end"] + S2 -->|"results upstream"| S1 + S1 -->|"results upstream"| S0 + S0 --> Output["OpenAI-compatible stream"] + Package["Layer package
manifest + GGUF fragments"] -.-> S0 + Package -.-> S1 + Package -.-> S2 +``` + +The coordinator selects stage boundaries from model metadata, available memory, backend capability, and topology policy. Downstream stages become ready before upstream stages send work. Activations travel over the Skippy stage transport; the caller does not need to know how many stages are involved. + +### Packages and materialization + +The durable artifact is a package repository with `model-package.json`, shared GGUF fragments, layer GGUFs, optional projectors, and checksums. A node materializes only the stage files it needs into its local derived cache. Materialized stage files are replaceable cache output; the package manifest and immutable model reference are the source of identity. + +### Runtime artifacts + +The Skippy ABI is carried by a verified native runtime artifact selected for the Mesh release, exact ABI, operating system, architecture, and backend lane. SDKs and packaged deployments resolve these artifacts at startup or bundle them with the application. A model package and a native runtime solve different problems: the package supplies model data, while the runtime supplies execution code. + +## Main operating shapes + +| Shape | Use it when | Main path | +| --- | --- | --- | +| Single node | The model fits locally and you want the shortest path. | API → local Skippy stage | +| Private mesh | You control the machines and want invite-token membership. | API → QUIC peer routing → host or stage | +| Public mesh | You want discovery and shared public capacity. | API → public discovery → selected mesh target | +| Client-only | The app should consume inference without serving a model. | SDK client → direct mesh transport or local proxy | +| Split serving | No single node can fit the model. | API → stage 0 → stage pipeline → response | +| SDK-embedded node | Another application owns the process lifecycle and UI. | App → language SDK → embedded node/runtime | + +Start with [Mesh workflows](/docs/pages/private-meshes/) for operators, [Running large models](/docs/pages/running-large-models/) for split serving, or [SDK embedding](/docs/pages/sdk/) for application developers. + +## Where to look in the repository + +| Concern | Primary source | +| --- | --- | +| Shipped binary and CLI wiring | `crates/mesh-llm/`, `crates/mesh-llm-cli/`, `crates/mesh-llm-commands/` | +| Host orchestration | `crates/mesh-llm-host-runtime/src/runtime/` | +| Mesh, gossip, and peers | `crates/mesh-llm-host-runtime/src/mesh/` | +| Routing, proxying, tunnels, and affinity | `crates/mesh-llm-host-runtime/src/network/` | +| Model resolution and inventory | `crates/mesh-llm-host-runtime/src/models/` and `model-*` crates | +| SDK facade and embedded node | `crates/mesh-llm-sdk/`, `crates/mesh-llm-api-server/`, `crates/mesh-llm-node/` | +| FFI and language packages | `crates/mesh-llm-ffi/`, `crates/mesh-llm-nodejs/`, `sdk/` | +| Skippy runtime and stage serving | `crates/skippy-*` and `crates/mesh-llm-embedded-runtime/` | +| Protocol definitions | `crates/mesh-llm-protocol/`, `crates/skippy-protocol/`, `proto/` | + +## Deep dives + +- [Mesh design](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/design/DESIGN.md) — host architecture, node roles, transport streams, routing, and management APIs. +- [Mesh workflows](/docs/pages/private-meshes/) — public, private, published, and client-only deployment shapes. +- [Skippy split serving](/docs/pages/running-large-models/) — package refs, stage planning, readiness, caches, and diagnostics. +- [Model package specification](/docs/pages/model-package-spec/) — `model-package.json` schema, artifact integrity, stage selection, and compatibility rules. +- [Skippy integration notes](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/SKIPPY.md) — execution/runtime ownership and migration boundaries. +- [Layer package repositories](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/LAYER_PACKAGE_REPOS.md) — durable package layout and validation. +- [Native runtime artifacts](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/design/NATIVE_RUNTIMES.md) — platform/backend packaging and ABI compatibility. +- [Protocol compatibility](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/design/message_protocol.md) — ALPN lanes, framed protobuf messages, and mixed-version rules. +- [Plugin architecture](/docs/pages/plugin-architecture/) — host projections, plugin processes, capabilities, and side streams. +- [SDK embedding](/docs/pages/sdk/) — embed a client or serving node in Rust, Node.js, JVM/Android, or Swift. +- [Testing playbook](/docs/pages/testing/) — local, multi-node, split-serving, and agent-harness validation. diff --git a/website/src/docs/pages/contributing-layer-packages.md b/website/src/docs/pages/contributing-layer-packages.md index c5726325c8..330c24bab3 100644 --- a/website/src/docs/pages/contributing-layer-packages.md +++ b/website/src/docs/pages/contributing-layer-packages.md @@ -1,6 +1,6 @@ # Contributing Layer Packages -Layer packages let Mesh place a model across multiple machines without every node downloading the full model. A package records the source model, quantization, layer artifacts, and validation metadata. +Layer packages let Mesh place a model across multiple machines without every node downloading the full model. A package records the source model, quantization, layer artifacts, and validation metadata. See the [model package specification](/docs/pages/model-package-spec/) for the `model-package.json` contract. ## Local contribution flow diff --git a/website/src/docs/pages/model-package-spec.md b/website/src/docs/pages/model-package-spec.md new file mode 100644 index 0000000000..cdbec152b8 --- /dev/null +++ b/website/src/docs/pages/model-package-spec.md @@ -0,0 +1,284 @@ +--- +title: Model Package Specification +--- + +# `model-package.json` specification + +`model-package.json` is the manifest for a Skippy layer-package repository. It +binds a source model identity to the GGUF artifacts needed to run contiguous +layer ranges across one or more nodes. + +The current manifest schema is version `1`. The manifest is the source of +truth for package identity, artifact paths, layer ownership, checksums, and +runtime compatibility. Repository names and README files are descriptive; a +consumer must validate the manifest before loading a stage. + +## Package repository + +The manifest must be at the repository root. A typical package has this shape: + +```text +model-package.json +shared/ + metadata.gguf + embeddings.gguf + output.gguf +layers/ + layer-00000.gguf + layer-00001.gguf + ... +projectors/ + mmproj-model-f16.gguf +README.md +``` + +Required artifacts are `shared/metadata.gguf`, `shared/embeddings.gguf`, +`shared/output.gguf`, and one `layers/layer-*.gguf` artifact for every +transformer layer. `projectors/*.gguf` is optional and is currently used for +multimodal `mmproj` artifacts. + +Every artifact path in the manifest must be relative to the package root. An +absolute path or a path containing `..` is invalid. Each owned tensor from the +source model must occur in exactly one package artifact. Shared metadata may be +repeated when required to keep a GGUF fragment loadable. + +## Manifest shape + +This example shows the complete schema shape. Values such as checksums and +sizes are illustrative: + +```json +{ + "schema_version": 1, + "model_id": "Qwen/Qwen3-235B-A22B-GGUF:UD-Q4_K_XL", + "source_model": { + "path": "/cache/Qwen3-235B-A22B-UD-Q4_K_XL.gguf", + "sha256": "<64 hex characters>", + "repo": "Qwen/Qwen3-235B-A22B-GGUF", + "revision": "", + "primary_file": "Qwen3-235B-A22B-UD-Q4_K_XL.gguf", + "canonical_ref": "Qwen/Qwen3-235B-A22B-GGUF:UD-Q4_K_XL", + "distribution_id": "UD-Q4_K_XL", + "files": [ + { + "path": "Qwen3-235B-A22B-UD-Q4_K_XL.gguf", + "size_bytes": 123, + "sha256": "<64 hex characters>" + } + ] + }, + "format": "layer-package", + "layer_count": 94, + "activation_width": 8192, + "shared": { + "metadata": { + "path": "shared/metadata.gguf", + "tensor_count": 0, + "tensor_bytes": 0, + "artifact_bytes": 123, + "sha256": "<64 hex characters>" + }, + "embeddings": { + "path": "shared/embeddings.gguf", + "tensor_count": 4, + "tensor_bytes": 123, + "artifact_bytes": 123, + "sha256": "<64 hex characters>" + }, + "output": { + "path": "shared/output.gguf", + "tensor_count": 4, + "tensor_bytes": 123, + "artifact_bytes": 123, + "sha256": "<64 hex characters>" + } + }, + "layers": [ + { + "layer_index": 0, + "path": "layers/layer-00000.gguf", + "tensor_count": 32, + "tensor_bytes": 123, + "artifact_bytes": 123, + "sha256": "<64 hex characters>" + } + ], + "projectors": [ + { + "kind": "mmproj", + "path": "projectors/mmproj-model-f16.gguf", + "tensor_count": 128, + "tensor_bytes": 123, + "artifact_bytes": 123, + "sha256": "<64 hex characters>" + } + ], + "skippy_abi_version": "1.2.3", + "created_at_unix_secs": 1790000000 +} +``` + +## Top-level fields + +| Field | Required | Description | +| --- | --- | --- | +| `schema_version` | Yes | Must be `1` for this specification. | +| `model_id` | Yes | Non-empty model coordinate, including its distribution or quantization identity. | +| `source_model` | Yes | Provenance for the source GGUF model. | +| `format` | Yes | Must be `layer-package`. | +| `layer_count` | Yes | Number of transformer layers. Valid layer indices are `0` through `layer_count - 1`. | +| `activation_width` | Recommended | Hidden-state width used by topology and activation planning. | +| `shared` | Yes | `metadata`, `embeddings`, and `output` artifact entries. | +| `layers` | Yes | Exactly one artifact entry for every layer index. | +| `projectors` | No | Package-level projector artifacts; currently `kind: "mmproj"`. | +| `generation` | No | Package-owned generation or speculative-decoding defaults. | +| `skippy_abi_version` | Yes | Skippy/llama ABI used to write the fragments. | +| `created_at_unix_secs` | Recommended | Unix timestamp for package provenance. | + +### Source model identity + +`source_model.path` and `source_model.sha256` identify the source artifact used +to create the package. When the source came from a model repository, include +`repo`, `revision`, `primary_file`, `canonical_ref`, and `distribution_id`. +The optional `files` list records the source files, sizes, and checksums used +by the package job. + +The source identity is distinct from the package repository name. Consumers +must not infer model compatibility from a repository name alone. + +### Artifact entries + +Every `shared`, `layers`, and `projectors` artifact entry contains: + +| Field | Description | +| --- | --- | +| `path` | Safe, repository-relative path. | +| `tensor_count` | Number of tensors in the fragment. | +| `tensor_bytes` | Total bytes occupied by tensor payloads. | +| `artifact_bytes` | Exact file size in bytes; must be greater than zero. | +| `sha256` | 64-character SHA-256 digest of the complete artifact. | + +`tensor_bytes` must be zero when `tensor_count` is zero, and greater than zero +when tensors are present. A projector also has a non-empty `kind`; the current +schema defines only `mmproj`. Consumers must reject an unknown projector kind +unless they explicitly support it. + +## Stage selection + +For a stage with the half-open layer range `layer_start..layer_end`, select: + +1. `shared.metadata`; +2. `shared.embeddings` when the stage owns the input boundary; +3. every `layers[]` entry whose `layer_index` is in `layer_start..layer_end`; +4. `shared.output` when the stage owns the final output boundary. + +The requested range must be non-empty and must not exceed `layer_count`. A +materialized per-stage GGUF is derived cache output; it is not the published +package format. + +Projectors are selected independently: + +1. an explicit `projector_path` wins; +2. otherwise, stage 0 or a single-stage runtime uses the first `mmproj` entry; +3. downstream stages do not load projector artifacts. + +Consumers must use the manifest to identify projectors rather than guessing +from sibling filenames. + +## Generation defaults + +`generation` is optional. When present, it may declare a recommended +speculative-decoding strategy: + +```json +{ + "generation": { + "speculative_decoding": { + "default": "mtp", + "strategies": { + "mtp": { + "type": "native-mtp", + "prediction_depth": 1, + "layer_indices": [47], + "window_policy": { + "default": "fixed", + "initial_window": 1, + "min_window": 1, + "max_window": 1 + } + } + } + } + } +} +``` + +For the current native MTP path, `type` is `native-mtp`, +`prediction_depth` is `1`, and `layer_indices` must identify package layers +containing the native MTP tensors. `default` must name an entry in +`strategies`. An unrecognized strategy type must be ignored unless it is the +declared default for a request the runtime is trying to serve. + +## Validation and integrity + +Before starting a stage, a consumer must verify that: + +- the file is UTF-8 JSON with `schema_version: 1` and `format: "layer-package"`; +- the model identity, source identity, and Skippy ABI version are present; +- `layer_count` and `activation_width` are valid for the requested topology; +- layer entries cover every index in `0..layer_count` exactly once; +- selected paths are safe relative paths and exist in the package; +- selected file sizes match `artifact_bytes`; +- selected artifact SHA-256 digests match the manifest; +- the runtime ABI is compatible with `skippy_abi_version`; +- any selected projector is declared, supported, and valid. + +Checksum verification applies to selected artifacts, including cache-hit +resolutions of `hf://` packages. A peer-transfer implementation must also +verify the manifest size and digest before installing an artifact, and must +install downloaded files atomically from a fresh partial file. + +## Package references and publishing + +Package references use the `hf://` scheme: + +```text +hf://meshllm/Qwen3-235B-A22B-UD-Q4_K_XL-layers +hf://meshllm/Qwen3-235B-A22B-UD-Q4_K_XL-layers:8f4c2d1 +hf://meshllm/Qwen3-235B-A22B-UD-Q4_K_XL-layers@main +``` + +Production configurations should use an immutable commit or tag rather than +a moving branch. + +Create and validate a package with the package tool: + +```sh +skippy-model-package write-package org/repo:distribution --out-dir model-package/ +skippy-model-package validate-package /path/to/source.gguf model-package/ +``` + +For multimodal packages, declare the projector when writing the package: + +```sh +skippy-model-package write-package org/repo:distribution \ + --projector mmproj-model-f16.gguf \ + --out-dir model-package/ +``` + +A package README should record the source coordinate and revision, source and +manifest checksums, layer count, activation width, Skippy ABI, validation +result, projector checksums, and any declared generation defaults. + +## Compatibility rules + +Schema version `1` changes should be additive when older runtimes can safely +ignore new optional fields. Packages without `projectors` remain valid. + +Changes to tensor ownership, layer indexing, path semantics, ABI requirements, +or required fields require a new schema version or `format` value. Runtimes +must reject unknown schema versions, formats, and incompatible ABI versions; +they must not attempt best-effort loading. + +For the implementation-level rules and peer artifact-transfer behavior, see +the [layer package repository specification](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/specs/layer-package-repos.md). diff --git a/website/src/docs/pages/running-large-models.md b/website/src/docs/pages/running-large-models.md index 547858220f..b2e6bea107 100644 --- a/website/src/docs/pages/running-large-models.md +++ b/website/src/docs/pages/running-large-models.md @@ -40,4 +40,21 @@ Use layer packages when: - the catalog marks a package as available - the machines are on a low-latency network +## What a split does + +Mesh keeps the public request path in one place while Skippy runs contiguous layer ranges on the machines that have the required package artifacts and capacity. + +```mermaid +flowchart LR + Request["OpenAI request
http://localhost:9337/v1"] --> Driver["Stage 0
driver + first layers"] + Driver -->|"activation transport"| Middle["Stage 1..N
contiguous layer ranges"] + Middle --> Result["Tokens / response
back through stage 0"] + Package["model-package.json
GGUF fragments + checksums"] -.-> Driver + Package -.-> Middle + Ready["Readiness + topology
published before routing"] -.-> Driver + Ready -.-> Middle +``` + +The [architecture hub](/docs/pages/architecture/) explains how Mesh routes requests into Skippy. See the [model package specification](/docs/pages/model-package-spec/) for the manifest schema, artifact checksums, and stage-selection rules. For package publishing and validation, see [Layer package repositories](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/LAYER_PACKAGE_REPOS.md). + If you are just trying Mesh for the first time, do not start here. Run the [Quickstart](/docs/pages/quickstart/) first. From 2f394d81f8c903f306a3faf6b774000b27c6f4f5 Mon Sep 17 00:00:00 2001 From: James Dumay Date: Tue, 14 Jul 2026 07:55:02 +1000 Subject: [PATCH 2/2] docs: address architecture docs review feedback --- website/src/assets/mermaid.js | 74 +++++++++--------- website/src/assets/site.css | 2 +- website/src/docs/pages/architecture.md | 7 +- website/src/docs/pages/model-package-spec.md | 77 +++++-------------- .../src/docs/pages/running-large-models.md | 4 +- 5 files changed, 67 insertions(+), 97 deletions(-) diff --git a/website/src/assets/mermaid.js b/website/src/assets/mermaid.js index 02b0fa1193..020d607ef4 100644 --- a/website/src/assets/mermaid.js +++ b/website/src/assets/mermaid.js @@ -5,41 +5,45 @@ async function renderMermaid() { return; } - const { default: mermaid } = await import( - "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs" - ); - - mermaid.initialize({ - startOnLoad: false, - securityLevel: "strict", - theme: "dark", - flowchart: { - curve: "basis", - htmlLabels: true, - }, - sequence: { - useMaxWidth: true, - }, - }); - - const nodes = []; - - blocks.forEach((code) => { - const container = document.createElement("div"); - const pre = code.parentElement; - const frame = pre?.parentElement?.classList.contains("code-copy-frame") - ? pre.parentElement - : pre; - - container.className = "mermaid"; - container.setAttribute("role", "img"); - container.setAttribute("aria-label", "Architecture diagram"); - container.textContent = code.textContent; - frame?.replaceWith(container); - nodes.push(container); - }); - - mermaid.run({ nodes }); + try { + const { default: mermaid } = await import( + "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs" + ); + + mermaid.initialize({ + startOnLoad: false, + securityLevel: "loose", + theme: "dark", + flowchart: { + curve: "basis", + htmlLabels: true, + }, + sequence: { + useMaxWidth: true, + }, + }); + + const nodes = []; + + blocks.forEach((code) => { + const container = document.createElement("div"); + const pre = code.parentElement; + const frame = pre?.parentElement?.classList.contains("code-copy-frame") + ? pre.parentElement + : pre; + + container.className = "mermaid"; + container.setAttribute("role", "img"); + container.setAttribute("aria-label", "Diagram"); + container.textContent = code.textContent; + frame?.replaceWith(container); + nodes.push(container); + }); + + await mermaid.run({ nodes }); + } catch (err) { + console.error("Mermaid rendering failed:", err); + } } if (document.readyState === "loading") { diff --git a/website/src/assets/site.css b/website/src/assets/site.css index c846a9e83a..ba5d319e0a 100644 --- a/website/src/assets/site.css +++ b/website/src/assets/site.css @@ -4509,7 +4509,7 @@ html:has(> body.docs-body)::-webkit-scrollbar-corner, padding: 16px; border: 1px solid var(--line-2); border-radius: 8px; - background: #07090d; + background: var(--bg); text-align: center; } diff --git a/website/src/docs/pages/architecture.md b/website/src/docs/pages/architecture.md index c13c77fa8b..5cf0f48c48 100644 --- a/website/src/docs/pages/architecture.md +++ b/website/src/docs/pages/architecture.md @@ -28,6 +28,10 @@ flowchart TD The important boundary is that mesh-llm owns the product and network behavior, while Skippy owns model execution. The OpenAI API stays stable whether the request is handled locally, by a peer, or by a multi-stage pipeline. +## Product and control surfaces + +The inference API at `:9337/v1` is the application-facing path. The local management API at `:3131` supplies status, discovery, lifecycle, and runtime views to the React Mesh LLM console and to operators using scripts or the CLI. An owned node-control API provides configuration, inventory, and runtime commands for attested hosts through the owner-control lane; it uses explicit endpoint authorization and remains separate from the mesh plane used for join, gossip, routing, and inference. + ## What happens to a request 1. An application sends an OpenAI-compatible request to the node's `/v1` endpoint. SDK clients may use direct mesh transport instead of a local HTTP listener. @@ -88,7 +92,7 @@ Protocol changes should be additive: older nodes must be able to ignore new opti ### Routing and election -Every node exposes the same OpenAI-facing shape. A request is routed by model identity rather than by a user selecting a machine. Per-model election decides which node or stage-0 target is authoritative, while passive clients receive a smaller route view instead of full mesh gossip. +Every node exposes the same OpenAI-facing shape and can route a request to an eligible host or worker node. Routing is based on model identity rather than a user selecting a machine. Per-model election decides which node or stage-0 target is authoritative, while passive clients receive a smaller route view instead of full mesh gossip. Routing also considers request affinity. Reusing a target for a shared prefix can preserve cache locality, while health and topology changes can drain or replace a target. The router is advisory only until a target is healthy and ready; a process that has merely spawned is not routable. @@ -149,6 +153,7 @@ Start with [Mesh workflows](/docs/pages/private-meshes/) for operators, [Running | Mesh, gossip, and peers | `crates/mesh-llm-host-runtime/src/mesh/` | | Routing, proxying, tunnels, and affinity | `crates/mesh-llm-host-runtime/src/network/` | | Model resolution and inventory | `crates/mesh-llm-host-runtime/src/models/` and `model-*` crates | +| React Mesh LLM console and management server | `crates/mesh-llm-ui/`, `crates/mesh-llm-console-server/` | | SDK facade and embedded node | `crates/mesh-llm-sdk/`, `crates/mesh-llm-api-server/`, `crates/mesh-llm-node/` | | FFI and language packages | `crates/mesh-llm-ffi/`, `crates/mesh-llm-nodejs/`, `sdk/` | | Skippy runtime and stage serving | `crates/skippy-*` and `crates/mesh-llm-embedded-runtime/` | diff --git a/website/src/docs/pages/model-package-spec.md b/website/src/docs/pages/model-package-spec.md index cdbec152b8..990cb888a1 100644 --- a/website/src/docs/pages/model-package-spec.md +++ b/website/src/docs/pages/model-package-spec.md @@ -4,14 +4,9 @@ title: Model Package Specification # `model-package.json` specification -`model-package.json` is the manifest for a Skippy layer-package repository. It -binds a source model identity to the GGUF artifacts needed to run contiguous -layer ranges across one or more nodes. +`model-package.json` is the manifest for a Skippy layer-package repository. It binds a source model identity to the GGUF artifacts needed to run contiguous layer ranges across one or more nodes. -The current manifest schema is version `1`. The manifest is the source of -truth for package identity, artifact paths, layer ownership, checksums, and -runtime compatibility. Repository names and README files are descriptive; a -consumer must validate the manifest before loading a stage. +The current manifest schema is version `1`. The manifest is the source of truth for package identity, artifact paths, layer ownership, checksums, and runtime compatibility. Repository names and README files are descriptive; a consumer must validate the manifest before loading a stage. ## Package repository @@ -32,20 +27,13 @@ projectors/ README.md ``` -Required artifacts are `shared/metadata.gguf`, `shared/embeddings.gguf`, -`shared/output.gguf`, and one `layers/layer-*.gguf` artifact for every -transformer layer. `projectors/*.gguf` is optional and is currently used for -multimodal `mmproj` artifacts. +Required artifacts are `shared/metadata.gguf`, `shared/embeddings.gguf`, `shared/output.gguf`, and one `layers/layer-*.gguf` artifact for every transformer layer. `projectors/*.gguf` is optional and is currently used for multimodal `mmproj` artifacts. -Every artifact path in the manifest must be relative to the package root. An -absolute path or a path containing `..` is invalid. Each owned tensor from the -source model must occur in exactly one package artifact. Shared metadata may be -repeated when required to keep a GGUF fragment loadable. +Every artifact path in the manifest must be relative to the package root. An absolute path or a path containing `..` is invalid. Each owned tensor from the source model must occur in exactly one package artifact. Shared metadata may be repeated when required to keep a GGUF fragment loadable. ## Manifest shape -This example shows the complete schema shape. Values such as checksums and -sizes are illustrative: +This example shows the complete schema shape. Values such as checksums and sizes are illustrative: ```json { @@ -127,7 +115,7 @@ sizes are illustrative: | `source_model` | Yes | Provenance for the source GGUF model. | | `format` | Yes | Must be `layer-package`. | | `layer_count` | Yes | Number of transformer layers. Valid layer indices are `0` through `layer_count - 1`. | -| `activation_width` | Recommended | Hidden-state width used by topology and activation planning. | +| `activation_width` | Yes | Hidden-state width used by topology and activation planning. | | `shared` | Yes | `metadata`, `embeddings`, and `output` artifact entries. | | `layers` | Yes | Exactly one artifact entry for every layer index. | | `projectors` | No | Package-level projector artifacts; currently `kind: "mmproj"`. | @@ -137,14 +125,9 @@ sizes are illustrative: ### Source model identity -`source_model.path` and `source_model.sha256` identify the source artifact used -to create the package. When the source came from a model repository, include -`repo`, `revision`, `primary_file`, `canonical_ref`, and `distribution_id`. -The optional `files` list records the source files, sizes, and checksums used -by the package job. +`source_model.path` and `source_model.sha256` identify the source artifact used to create the package. When the source came from a model repository, include `repo`, `revision`, `primary_file`, `canonical_ref`, and `distribution_id`. The optional `files` list records the source files, sizes, and checksums used by the package job. -The source identity is distinct from the package repository name. Consumers -must not infer model compatibility from a repository name alone. +The source identity is distinct from the package repository name. Consumers must not infer model compatibility from a repository name alone. ### Artifact entries @@ -158,10 +141,7 @@ Every `shared`, `layers`, and `projectors` artifact entry contains: | `artifact_bytes` | Exact file size in bytes; must be greater than zero. | | `sha256` | 64-character SHA-256 digest of the complete artifact. | -`tensor_bytes` must be zero when `tensor_count` is zero, and greater than zero -when tensors are present. A projector also has a non-empty `kind`; the current -schema defines only `mmproj`. Consumers must reject an unknown projector kind -unless they explicitly support it. +`tensor_bytes` must be zero when `tensor_count` is zero, and greater than zero when tensors are present. A projector also has a non-empty `kind`; the current schema defines only `mmproj`. Consumers must reject an unknown projector kind unless they explicitly support it. ## Stage selection @@ -172,9 +152,7 @@ For a stage with the half-open layer range `layer_start..layer_end`, select: 3. every `layers[]` entry whose `layer_index` is in `layer_start..layer_end`; 4. `shared.output` when the stage owns the final output boundary. -The requested range must be non-empty and must not exceed `layer_count`. A -materialized per-stage GGUF is derived cache output; it is not the published -package format. +The requested range must be non-empty and must not exceed `layer_count`. A materialized per-stage GGUF is derived cache output; it is not the published package format. Projectors are selected independently: @@ -182,13 +160,11 @@ Projectors are selected independently: 2. otherwise, stage 0 or a single-stage runtime uses the first `mmproj` entry; 3. downstream stages do not load projector artifacts. -Consumers must use the manifest to identify projectors rather than guessing -from sibling filenames. +Consumers must use the manifest to identify projectors rather than guessing from sibling filenames. ## Generation defaults -`generation` is optional. When present, it may declare a recommended -speculative-decoding strategy: +`generation` is optional. When present, it may declare a recommended speculative-decoding strategy: ```json { @@ -213,11 +189,7 @@ speculative-decoding strategy: } ``` -For the current native MTP path, `type` is `native-mtp`, -`prediction_depth` is `1`, and `layer_indices` must identify package layers -containing the native MTP tensors. `default` must name an entry in -`strategies`. An unrecognized strategy type must be ignored unless it is the -declared default for a request the runtime is trying to serve. +For the current native MTP path, `type` is `native-mtp`, `prediction_depth` is `1`, and `layer_indices` must identify package layers containing the native MTP tensors. `default` must name an entry in `strategies`. An unrecognized strategy type must be ignored unless it is the declared default for a request the runtime is trying to serve. ## Validation and integrity @@ -233,10 +205,7 @@ Before starting a stage, a consumer must verify that: - the runtime ABI is compatible with `skippy_abi_version`; - any selected projector is declared, supported, and valid. -Checksum verification applies to selected artifacts, including cache-hit -resolutions of `hf://` packages. A peer-transfer implementation must also -verify the manifest size and digest before installing an artifact, and must -install downloaded files atomically from a fresh partial file. +Checksum verification applies to selected artifacts, including cache-hit resolutions of `hf://` packages. A peer-transfer implementation must verify each downloaded artifact's size and SHA-256 against its manifest entry before installing it, and must install downloaded files atomically from a fresh partial file. ## Package references and publishing @@ -248,8 +217,7 @@ hf://meshllm/Qwen3-235B-A22B-UD-Q4_K_XL-layers:8f4c2d1 hf://meshllm/Qwen3-235B-A22B-UD-Q4_K_XL-layers@main ``` -Production configurations should use an immutable commit or tag rather than -a moving branch. +Production configurations should use an immutable commit or tag rather than a moving branch. Create and validate a package with the package tool: @@ -266,19 +234,12 @@ skippy-model-package write-package org/repo:distribution \ --out-dir model-package/ ``` -A package README should record the source coordinate and revision, source and -manifest checksums, layer count, activation width, Skippy ABI, validation -result, projector checksums, and any declared generation defaults. +A package README should record the source coordinate and revision, source and manifest checksums, layer count, activation width, Skippy ABI, validation result, projector checksums, and any declared generation defaults. ## Compatibility rules -Schema version `1` changes should be additive when older runtimes can safely -ignore new optional fields. Packages without `projectors` remain valid. +Schema version `1` changes should be additive when older runtimes can safely ignore new optional fields. Packages without `projectors` remain valid. -Changes to tensor ownership, layer indexing, path semantics, ABI requirements, -or required fields require a new schema version or `format` value. Runtimes -must reject unknown schema versions, formats, and incompatible ABI versions; -they must not attempt best-effort loading. +Changes to tensor ownership, layer indexing, path semantics, ABI requirements, or required fields require a new schema version or `format` value. Runtimes must reject unknown schema versions, formats, and incompatible ABI versions; they must not attempt best-effort loading. -For the implementation-level rules and peer artifact-transfer behavior, see -the [layer package repository specification](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/specs/layer-package-repos.md). +For the implementation-level rules and peer artifact-transfer behavior, see the [layer package repository specification](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/specs/layer-package-repos.md). diff --git a/website/src/docs/pages/running-large-models.md b/website/src/docs/pages/running-large-models.md index b2e6bea107..75d96232b9 100644 --- a/website/src/docs/pages/running-large-models.md +++ b/website/src/docs/pages/running-large-models.md @@ -2,6 +2,8 @@ Start with one working node first. After console chat works, use additional machines or catalog layer packages for larger models. +If you are just trying Mesh for the first time, do not start here. Run the [Quickstart](/docs/pages/quickstart/) first. + ## Add serving machines Run the same private mesh name on each machine: @@ -56,5 +58,3 @@ flowchart LR ``` The [architecture hub](/docs/pages/architecture/) explains how Mesh routes requests into Skippy. See the [model package specification](/docs/pages/model-package-spec/) for the manifest schema, artifact checksums, and stage-selection rules. For package publishing and validation, see [Layer package repositories](https://github.com/Mesh-LLM/mesh-llm/blob/main/docs/LAYER_PACKAGE_REPOS.md). - -If you are just trying Mesh for the first time, do not start here. Run the [Quickstart](/docs/pages/quickstart/) first.