Skip to content
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/compute-changes/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ runs:
fi
if [[ -n "$DIRECT_SDK_INPUTS" ]]; then
SDK_SMOKE_REQUIRED="true"
elif echo "$AFFECTED_CRATES" | jq -e 'index("mesh-llm-client") or index("mesh-llm-api-client") or index("mesh-llm-api-server") or index("mesh-llm-config") or index("mesh-llm-console-server") or index("mesh-llm-ffi") or index("mesh-llm-native-runtime") or index("mesh-llm-protocol") or index("mesh-llm-routing") or index("mesh-llm-types")' >/dev/null; then
elif echo "$AFFECTED_CRATES" | jq -e 'index("mesh-llm-client") or index("mesh-llm-api-client") or index("mesh-llm-api-server") or index("mesh-llm-config") or index("mesh-llm-console-server") or index("mesh-llm-ffi") or index("mesh-llm-native-runtime") or index("mesh-llm-provider-runtime") or index("mesh-llm-protocol") or index("mesh-llm-routing") or index("mesh-llm-types")' >/dev/null; then
SDK_SMOKE_REQUIRED="true"
fi
fi
Expand Down
11 changes: 11 additions & 0 deletions .github/workflows/ci-platform-checks-slice.yml
Original file line number Diff line number Diff line change
Expand Up @@ -188,3 +188,14 @@ jobs:
cargo test --locked -p model-artifact --lib
cargo test --locked -p mesh-llm-host-runtime --lib
cargo test --locked -p mesh-llm --lib
- name: Install just
if: ${{ matrix.check.kind == 'unit' }}
uses: taiki-e/install-action@3d23c1bbdafe696dfccad2664945a04f47d03dc3 # just
with:
tool: just
- name: Build and test the Apple provider
if: ${{ matrix.check.kind == 'unit' }}
run: |
set -euo pipefail
just apple::build
just apple::test
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
1 change: 1 addition & 0 deletions .github/workflows/docker-precheck.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ jobs:
crates/mesh-llm-hardware-profile/ \
crates/mesh-llm-identity/ \
crates/mesh-llm-native-runtime/ \
crates/mesh-llm-provider-runtime/ \
crates/mesh-llm-release-footer/ \
crates/mesh-llm-protocol/ \
crates/mesh-llm-routing/ \
Expand Down
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ Shared foundations:
- `mesh-llm-identity/` — owner identity and envelope crypto primitives.
- `mesh-llm-guardrails/` — guardrail and compaction primitives for OpenAI-compatible paths.
- `mesh-llm-hardware-profile/`, `mesh-llm-native-runtime/`, `mesh-llm-runtime-install/` — hardware profile detection, native runtime manifest/selection, runtime download/install/cache.
- `mesh-llm-provider-runtime/` — signed executable-provider manifests, selection, download verification, and immutable cache installation; separate from Skippy-native libraries.
- `mesh-llm-plugin/` — plugin runtime/DSL primitives.
- `mesh-llm-plugin-manager/` — plugin package management (catalog, install, store).
- `mesh-llm-skills/` — agent skill data model and installer primitives.
Expand Down
19 changes: 19 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ members = [
"crates/mesh-llm-hardware-profile",
"crates/mesh-llm-identity",
"crates/mesh-llm-native-runtime",
"crates/mesh-llm-provider-runtime",
"crates/mesh-llm-protocol",
"crates/mesh-llm-release-footer",
"crates/mesh-llm-routing",
Expand Down
2 changes: 2 additions & 0 deletions Justfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Distributed LLM Inference — build & run tasks

mod apple 'providers/apple/Justfile'

llama_dir := env("MESH_LLM_LLAMA_DIR", ".deps/llama.cpp")
llama_build_root := env("MESH_LLM_LLAMA_BUILD_ROOT", ".deps/llama-build")
mesh_dir := "crates/mesh-llm"
Expand Down
13 changes: 12 additions & 1 deletion Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@ let hasRemoteFFIXCFramework =
!remoteFFIXCFrameworkURL.contains("__MESH_SWIFT_RELEASE_TAG__")
&& !remoteFFIXCFrameworkChecksum.contains("__MESH_SWIFT_RELEASE_CHECKSUM__")

var meshLLMDependencies: [Target.Dependency] = []
var meshLLMDependencies: [Target.Dependency] = [
.target(
name: "MeshLLMAppleProviderResources",
condition: .when(platforms: [.macOS])
),
]
var packageTargets: [Target] = []

if hasLocalFFIXCFramework {
Expand Down Expand Up @@ -62,6 +67,7 @@ let package = Package(
.linkedFramework("Accelerate"),
.linkedFramework("AppKit", .when(platforms: [.macOS])),
.linkedFramework("CoreGraphics"),
.linkedFramework("CoreWLAN", .when(platforms: [.macOS])),
.linkedFramework("Foundation"),
.linkedFramework("Metal"),
.linkedFramework("MetalKit"),
Expand All @@ -74,5 +80,10 @@ let package = Package(
dependencies: ["MeshLLM"],
path: "sdk/swift/Tests/MeshLLMTests"
),
.target(
name: "MeshLLMAppleProviderResources",
path: "sdk/swift/Sources/MeshLLMAppleProviderResources",
resources: [.copy("Resources/apple")]
),
] + packageTargets
)
1 change: 1 addition & 0 deletions crates/mesh-client/src/network/affinity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ mod tests {
target: target.clone(),
learn_prefix_hash: None,
cached_target: None,
affinity_selected: false,
};
let prepared: PreparedTargets = PreparedTargets {
ordered: vec![target],
Expand Down
10 changes: 10 additions & 0 deletions crates/mesh-client/tests/mesh_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@ fn model_runtime_descriptor_not_ready_by_default() {
identity_hash: None,
context_length: None,
ready: false,
provider_kind: None,
model_version: None,
max_concurrent_requests: None,
active_requests: None,
queued_requests: None,
};
assert!(d.advertised_context_length().is_none());
}
Expand All @@ -45,6 +50,11 @@ fn model_runtime_descriptor_ready_returns_context_length() {
identity_hash: None,
context_length: Some(4096),
ready: true,
provider_kind: None,
model_version: None,
max_concurrent_requests: None,
active_requests: None,
queued_requests: None,
};
assert_eq!(d.advertised_context_length(), Some(4096));
}
Expand Down
5 changes: 5 additions & 0 deletions crates/mesh-llm-embedded-runtime/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,8 @@ async fn main() -> anyhow::Result<()> {
}
```

Provider-only hosts can omit `.model(...)` and supply one or more packaged
provider roots with `.provider_runtime_root(...)`. This path does not require a
Skippy native runtime. Provider roots are SDK-owned resource locations; bundle
validation, selection, launch, health, restart, routing, and shutdown remain
owned by the shared host runtime.
7 changes: 4 additions & 3 deletions crates/mesh-llm-embedded-runtime/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ pub use mesh_llm_host_runtime::sdk::{
EmbeddedMeshHttpConfig, EmbeddedMeshLogFormat, EmbeddedMeshNetworkConfig,
EmbeddedMeshNodeBuilder, EmbeddedMeshNodeConfig, EmbeddedMeshNodeHandle, EmbeddedMeshNodeMode,
EmbeddedMeshNodeStatus, EmbeddedMeshRequirementsConfig, EmbeddedMeshServingConfig,
EmbeddedMeshStorageConfig, EmbeddedServeConfig, EmbeddedServeHandle, EmbeddedServeMode,
EmbeddedServeStatus, EmbeddedServingController, EmbeddedTrustPolicy,
SIGNED_JOIN_TOKEN_MIN_PROTOCOL_VERSION, start_embedded_node, start_embedded_serve,
EmbeddedMeshStorageConfig, EmbeddedProviderRuntimeConfig, EmbeddedServeConfig,
EmbeddedServeHandle, EmbeddedServeMode, EmbeddedServeStatus, EmbeddedServingController,
EmbeddedTrustPolicy, SIGNED_JOIN_TOKEN_MIN_PROTOCOL_VERSION, start_embedded_node,
start_embedded_serve,
};

pub mod config {
Expand Down
9 changes: 9 additions & 0 deletions crates/mesh-llm-ffi/src/handles.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ use mesh_llm_sdk::node::MeshNode;

#[cfg(feature = "embedded-runtime")]
use mesh_llm_sdk::embedded_runtime::EmbeddedServingController;
#[cfg(feature = "embedded-runtime")]
use mesh_llm_sdk::provider_host::ProviderHost;

#[derive(uniffi::Object)]
pub struct MeshClientHandle {
Expand All @@ -26,3 +28,10 @@ pub struct ConsoleHandle {
pub(crate) inner: Mutex<Option<mesh_llm_sdk::console::ConsoleServerHandle>>,
pub(crate) url: String,
}

#[derive(uniffi::Object)]
pub struct ProviderHostHandle {
#[cfg(feature = "embedded-runtime")]
pub(crate) host: tokio::sync::Mutex<Option<ProviderHost>>,
pub(crate) api_base_url: String,
}
6 changes: 3 additions & 3 deletions crates/mesh-llm-ffi/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ mod runtime_blocking;
pub use client::{create_auto_client, create_client};
pub use errors::FfiError;
pub use events::ClientEvent;
pub use handles::{ConsoleHandle, MeshClientHandle, MeshNodeHandle};
pub use handles::{ConsoleHandle, MeshClientHandle, MeshNodeHandle, ProviderHostHandle};
pub use identity::generate_owner_keypair_hex;
pub use model_types::{
CapabilityLevel, CleanupPolicy, CleanupResult, DeleteModelOptions, DeleteModelResult,
Expand All @@ -47,9 +47,9 @@ pub use native_runtime_types::{
NativeRuntimeProgressListener, NativeRuntimePruneModeNative, NativeRuntimePruneResultNative,
NativeRuntimeVerificationPolicyNative,
};
pub use node::{create_auto_node, create_node};
pub use node::{create_auto_node, create_node, start_provider_host};
pub use public_mesh::discover_public_meshes;
pub use request_types::{
ChatMessageNative, ChatRequestNative, ClientStatus, ConsoleOptionsNative, ModelNative,
PublicMesh, PublicMeshQuery, ResponsesRequestNative,
ProviderRuntimeOptionsNative, PublicMesh, PublicMeshQuery, ResponsesRequestNative,
};
21 changes: 21 additions & 0 deletions crates/mesh-llm-ffi/src/mesh_ffi.udl
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ namespace mesh_ffi {
[Throws=FfiError]
MeshNodeHandle create_auto_node(string owner_keypair_bytes_hex, PublicMeshQuery query);

[Throws=FfiError]
ProviderHostHandle start_provider_host(ProviderRuntimeOptionsNative options);

[Throws=FfiError]
sequence<PublicMesh> discover_public_meshes(PublicMeshQuery query);

Expand Down Expand Up @@ -53,6 +56,24 @@ namespace mesh_ffi {
);
};

dictionary ProviderRuntimeOptionsNative {
sequence<string> bundle_roots;
string? release_manifest;
string? cache_dir;
boolean allow_download;
u64 startup_timeout_ms;
};

interface ProviderHostHandle {
string api_base_url();

[Throws=FfiError]
string status_json();

[Throws=FfiError]
void stop();
};

[Error]
enum FfiError {
"InvalidInviteToken",
Expand Down
91 changes: 88 additions & 3 deletions crates/mesh-llm-ffi/src/node.rs
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
use std::sync::{Arc, Mutex};
#[cfg(feature = "embedded-runtime")]
use std::{path::PathBuf, time::Duration};

use mesh_llm_sdk::node as sdk_node;
use mesh_llm_sdk::node::{MeshNode, create_auto_node as sdk_create_auto_node};
use mesh_llm_sdk::{InviteToken, RequestId};

#[cfg(feature = "embedded-runtime")]
use mesh_llm_sdk::embedded_runtime::{EmbeddedChatMessage, EmbeddedServingController};
#[cfg(feature = "embedded-runtime")]
use mesh_llm_sdk::provider_host::{ProviderHost, ProviderHostConfig};

use crate::errors::{
FfiError, map_mesh_api_error, map_model_error, map_serving_error, map_stream_error,
};
#[cfg(feature = "embedded-runtime")]
use crate::events::ClientEvent;
use crate::events::EventListenerBridge;
use crate::handles::{ConsoleHandle, MeshNodeHandle};
use crate::handles::{ConsoleHandle, MeshNodeHandle, ProviderHostHandle};
use crate::identity::parse_owner_keypair;
use crate::model_types::{
CleanupPolicy, CleanupResult, DeleteModelOptions, DeleteModelResult, DevicePolicy,
Expand All @@ -23,8 +27,8 @@ use crate::model_types::{
};
use crate::native_runtime_types::EventListener;
use crate::request_types::{
ChatRequestNative, ClientStatus, ConsoleOptionsNative, ModelNative, PublicMeshQuery,
ResponsesRequestNative,
ChatRequestNative, ClientStatus, ConsoleOptionsNative, ModelNative,
ProviderRuntimeOptionsNative, PublicMeshQuery, ResponsesRequestNative,
};
use crate::runtime_blocking::block_on;

Expand Down Expand Up @@ -398,3 +402,84 @@ fn new_request_id() -> String {
static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1);
format!("local-{}", NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed))
}

#[uniffi::export]
pub fn start_provider_host(
options: ProviderRuntimeOptionsNative,
) -> Result<Arc<ProviderHostHandle>, FfiError> {
#[cfg(not(feature = "embedded-runtime"))]
{
let _ = options;
Err(FfiError::ServingUnsupported(
"this native library was built without embedded-runtime support".to_string(),
))
}
#[cfg(feature = "embedded-runtime")]
{
let host = block_on(ProviderHost::start(provider_host_config(options)))
.map_err(|error| FfiError::HostUnavailable(error.to_string()))?;
let api_base_url = host.api_base_url().to_string();
Ok(Arc::new(ProviderHostHandle {
host: tokio::sync::Mutex::new(Some(host)),
api_base_url,
}))
}
}

#[uniffi::export]
impl ProviderHostHandle {
pub fn api_base_url(&self) -> String {
self.api_base_url.clone()
}

pub fn status_json(&self) -> Result<String, FfiError> {
#[cfg(not(feature = "embedded-runtime"))]
return Err(FfiError::ServingUnsupported(
"this native library was built without embedded-runtime support".to_string(),
));
#[cfg(feature = "embedded-runtime")]
block_on(async {
let host = self.host.lock().await;
let host = host
.as_ref()
.ok_or_else(|| FfiError::HostUnavailable("provider host is stopped".to_string()))?;
host.status()
.await
.map(|status| status.to_string())
.map_err(|error| FfiError::HostUnavailable(error.to_string()))
})
}

pub fn stop(&self) -> Result<(), FfiError> {
#[cfg(not(feature = "embedded-runtime"))]
return Err(FfiError::ServingUnsupported(
"this native library was built without embedded-runtime support".to_string(),
));
#[cfg(feature = "embedded-runtime")]
block_on(async {
let host = self.host.lock().await.take();
if let Some(host) = host {
host.stop()
.await
.map_err(|error| FfiError::HostUnavailable(error.to_string()))?;
}
Ok(())
})
}
}

#[cfg(feature = "embedded-runtime")]
fn provider_host_config(options: ProviderRuntimeOptionsNative) -> ProviderHostConfig {
ProviderHostConfig {
bundle_roots: options
.bundle_roots
.into_iter()
.filter_map(|path| non_empty_path(Some(path)))
.map(PathBuf::from)
.collect(),
release_manifest: non_empty_path(options.release_manifest).map(PathBuf::from),
cache_dir: non_empty_path(options.cache_dir).map(PathBuf::from),
allow_download: options.allow_download,
startup_timeout: Duration::from_millis(options.startup_timeout_ms.max(1)),
}
}
9 changes: 9 additions & 0 deletions crates/mesh-llm-ffi/src/request_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ pub struct ConsoleOptionsNative {
pub listen_all: bool,
}

#[derive(uniffi::Record)]
pub struct ProviderRuntimeOptionsNative {
pub bundle_roots: Vec<String>,
pub release_manifest: Option<String>,
pub cache_dir: Option<String>,
pub allow_download: bool,
pub startup_timeout_ms: u64,
}

#[derive(uniffi::Record)]
pub struct PublicMeshQuery {
pub model: Option<String>,
Expand Down
Loading
Loading