diff --git a/crates/mesh-llm-commands/src/runtime_native.rs b/crates/mesh-llm-commands/src/runtime_native.rs index 2582100aad..fa13be00f8 100644 --- a/crates/mesh-llm-commands/src/runtime_native.rs +++ b/crates/mesh-llm-commands/src/runtime_native.rs @@ -420,6 +420,7 @@ mod tests { std::fs::write(path.join("lib/libllama.so"), b"native runtime").unwrap(); NativeRuntimeManifest { runtime: NativeRuntimeArtifact { + build_id: None, id: runtime_id.to_string(), mesh_version: Some(CURRENT_MESH_VERSION.to_string()), skippy_abi: "0.1.25".to_string(), diff --git a/crates/mesh-llm-commands/src/runtime_native/setup_helpers.rs b/crates/mesh-llm-commands/src/runtime_native/setup_helpers.rs index c1ad600658..23a1f3a8d2 100644 --- a/crates/mesh-llm-commands/src/runtime_native/setup_helpers.rs +++ b/crates/mesh-llm-commands/src/runtime_native/setup_helpers.rs @@ -302,6 +302,7 @@ mod tests { fn fake_install_outcome(mesh_version: &str) -> NativeRuntimeInstallOutcome { let artifact = NativeRuntimeArtifact { + build_id: None, id: "meshllm-runtime-linux-x86_64-cpu".to_string(), mesh_version: Some(mesh_version.to_string()), skippy_abi: "0.1.25".to_string(), diff --git a/crates/mesh-llm-hardware-profile/src/lib.rs b/crates/mesh-llm-hardware-profile/src/lib.rs index 095f9de054..6b37663816 100644 --- a/crates/mesh-llm-hardware-profile/src/lib.rs +++ b/crates/mesh-llm-hardware-profile/src/lib.rs @@ -984,6 +984,7 @@ mod tests { vulkan: None, }; let artifact = |id: &str, backend: NativeRuntimeBackend| NativeRuntimeArtifact { + build_id: None, id: id.to_string(), mesh_version: Some("test".to_string()), skippy_abi: "test-abi".to_string(), diff --git a/crates/mesh-llm-host-runtime/src/system/native_runtime.rs b/crates/mesh-llm-host-runtime/src/system/native_runtime.rs index ca359da4f1..e9186f4e3b 100644 --- a/crates/mesh-llm-host-runtime/src/system/native_runtime.rs +++ b/crates/mesh-llm-host-runtime/src/system/native_runtime.rs @@ -455,6 +455,7 @@ mod dynamic { fs::write(dir.join(&library_rel_path), b"native runtime").unwrap(); let manifest = NativeRuntimeManifest { runtime: NativeRuntimeArtifact { + build_id: None, id: id.to_string(), mesh_version: version.map(ToString::to_string), skippy_abi: "0.1.25".to_string(), @@ -728,6 +729,7 @@ mod dynamic { artifacts: Vec::new(), }; let artifact = NativeRuntimeArtifact { + build_id: None, id: runtime_id.to_string(), mesh_version: Some(release_version.to_string()), skippy_abi: "0.1.25".to_string(), diff --git a/crates/mesh-llm-native-runtime/src/cache.rs b/crates/mesh-llm-native-runtime/src/cache.rs index 3103280614..e57dc66334 100644 --- a/crates/mesh-llm-native-runtime/src/cache.rs +++ b/crates/mesh-llm-native-runtime/src/cache.rs @@ -421,6 +421,7 @@ mod tests { let manifest = NativeRuntimeManifest { runtime: NativeRuntimeArtifact { id: id.to_string(), + build_id: None, mesh_version: Some(version.to_string()), skippy_abi: "0.1.25".to_string(), platform: NativeRuntimePlatform { diff --git a/crates/mesh-llm-native-runtime/src/manifest.rs b/crates/mesh-llm-native-runtime/src/manifest.rs index ec4a207be1..d17444f49b 100644 --- a/crates/mesh-llm-native-runtime/src/manifest.rs +++ b/crates/mesh-llm-native-runtime/src/manifest.rs @@ -23,6 +23,8 @@ pub struct NativeRuntimePlatform { pub struct NativeRuntimeArtifact { pub id: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub build_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub mesh_version: Option, pub skippy_abi: String, pub platform: NativeRuntimePlatform, @@ -163,6 +165,14 @@ fn validate_artifact(artifact: &NativeRuntimeArtifact) -> Result<()> { if artifact.id.trim().is_empty() { bail!("native runtime artifact id is empty"); } + if let Some(build_id) = &artifact.build_id { + validate_build_id(build_id).with_context(|| { + format!( + "native runtime artifact {} has invalid build_id", + artifact.id + ) + })?; + } if artifact.skippy_abi.trim().is_empty() { bail!( "native runtime artifact {} skippy_abi is empty", @@ -242,6 +252,20 @@ fn validate_runtime_path(relative: &str) -> Result<()> { Ok(()) } +fn validate_build_id(value: &str) -> Result<()> { + let Some(digest) = value.strip_prefix("sha256:") else { + bail!("expected sha256:<64 lowercase hex>"); + }; + if digest.len() != 64 + || !digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + bail!("expected sha256:<64 lowercase hex>"); + } + Ok(()) +} + fn normalize_sha256(value: &str) -> Result { let value = value .trim() @@ -292,6 +316,7 @@ mod tests { r#"{ "runtime": { "id": "meshllm-runtime-linux-x86_64-cuda12", + "build_id": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "mesh_version": "0.68.0", "skippy_abi": "0.1.25", "platform": { @@ -319,6 +344,10 @@ mod tests { let manifest = NativeRuntimeManifest::read_from_dir(temp.path()).unwrap(); assert_eq!(manifest.runtime.id, "meshllm-runtime-linux-x86_64-cuda12"); + assert_eq!( + manifest.runtime.build_id.as_deref(), + Some("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ); assert_eq!(manifest.runtime.skippy_abi, "0.1.25"); assert_eq!(manifest.runtime.backend.kind.as_str(), "cuda"); } @@ -345,9 +374,39 @@ mod tests { .unwrap(); assert_eq!(manifest.artifacts.len(), 1); + assert_eq!(manifest.artifacts[0].build_id, None); assert_eq!(manifest.artifacts[0].backend, NativeRuntimeBackend::cpu()); } + #[test] + fn rejects_invalid_runtime_build_ids() { + for build_id in [ + "", + &"a".repeat(64), + &format!("sha256:{}", "a".repeat(63)), + &format!("sha256:{}", "A".repeat(64)), + &format!("sha256:{}g", "a".repeat(63)), + ] { + let json = format!( + r#"{{ + "mesh_version": "0.68.0", + "skippy_abi": "0.1.25", + "artifacts": [{{ + "id": "runtime", + "build_id": "{build_id}", + "skippy_abi": "0.1.25", + "platform": {{"os": "linux", "arch": "x86_64"}}, + "backend": {{"kind": "cpu"}}, + "libraries": ["lib/runtime.so"] + }}] +}}"# + ); + + let error = NativeRuntimeReleaseManifest::from_json_str(&json).unwrap_err(); + assert!(error.to_string().contains("invalid build_id"), "{build_id}"); + } + } + #[test] fn rejects_tampered_runtime_file() { let temp = tempfile::tempdir().unwrap(); @@ -357,6 +416,7 @@ mod tests { let manifest = NativeRuntimeManifest { runtime: NativeRuntimeArtifact { id: "meshllm-runtime-linux-x86_64-cpu".to_string(), + build_id: None, mesh_version: Some("0.68.0".to_string()), skippy_abi: "0.1.25".to_string(), platform: NativeRuntimePlatform { @@ -395,6 +455,7 @@ mod tests { let manifest = NativeRuntimeManifest { runtime: NativeRuntimeArtifact { id: "meshllm-runtime-linux-x86_64-cuda12".to_string(), + build_id: None, mesh_version: Some("0.68.0".to_string()), skippy_abi: "0.1.25".to_string(), platform: NativeRuntimePlatform { @@ -478,6 +539,7 @@ mod tests { fn rejects_runtime_checksum_path_traversal() { let artifact = NativeRuntimeArtifact { id: "meshllm-runtime-linux-x86_64-cpu".to_string(), + build_id: None, mesh_version: Some("0.68.0".to_string()), skippy_abi: "0.1.25".to_string(), platform: NativeRuntimePlatform { diff --git a/crates/mesh-llm-native-runtime/src/resolver.rs b/crates/mesh-llm-native-runtime/src/resolver.rs index 4a654ff81c..c3ccf10dfb 100644 --- a/crates/mesh-llm-native-runtime/src/resolver.rs +++ b/crates/mesh-llm-native-runtime/src/resolver.rs @@ -205,7 +205,9 @@ impl NativeRuntimeResolver { artifact.mesh_version_or(&self.mesh_version), artifact.native_runtime_id(), )?; - if let Some(installed) = installed { + if let Some(installed) = installed + && artifact_identity_matches(&installed.manifest.runtime, artifact) + { return Ok(NativeRuntimeSource::Installed { path: installed.path, }); @@ -223,6 +225,10 @@ fn parse_cuda_major(value: &str) -> Option { } fn artifact_key(artifact: &NativeRuntimeArtifact) -> String { + // Candidate evaluation is lane-based: the selected release/bundle artifact + // must replace any installed candidate for the same lane. Build identity is + // enforced later by source_for_artifact when deciding whether cached bytes + // satisfy that selected artifact. format!( "{}\0{}\0{}", artifact.id, @@ -376,6 +382,10 @@ fn artifact_identity_matches( candidate.id == selected.id && candidate.mesh_version.as_deref() == selected.mesh_version.as_deref() && candidate.skippy_abi == selected.skippy_abi + && selected + .build_id + .as_ref() + .is_none_or(|build_id| candidate.build_id.as_ref() == Some(build_id)) } fn evaluate_backend_requirements( @@ -577,6 +587,7 @@ mod tests { fn artifact(id: &str, backend: NativeRuntimeBackend) -> NativeRuntimeArtifact { NativeRuntimeArtifact { id: id.to_string(), + build_id: None, mesh_version: Some("0.68.0".to_string()), skippy_abi: "0.1.25".to_string(), platform: NativeRuntimePlatform { @@ -1008,6 +1019,7 @@ mod tests { mesh_version: "0.67.0".to_string(), skippy_abi: "0.1.25".to_string(), artifacts: vec![NativeRuntimeArtifact { + build_id: None, mesh_version: Some("0.67.0".to_string()), ..cuda_runtime("meshllm-runtime-linux-x86_64-cuda12", 12, &["sm_90"]) }], @@ -1030,6 +1042,7 @@ mod tests { mesh_version: "0.67.0".to_string(), skippy_abi: "0.1.25".to_string(), artifacts: vec![NativeRuntimeArtifact { + build_id: None, mesh_version: Some("0.67.0".to_string()), ..cuda_runtime("meshllm-runtime-linux-x86_64-cuda12", 12, &["sm_90"]) }], @@ -1121,12 +1134,101 @@ mod tests { ); } + fn resolve_with_cached_and_selected_build( + cached_build_id: Option<&str>, + selected_build_id: Option<&str>, + ) -> NativeRuntimeResolution { + let bundle = tempfile::tempdir().unwrap(); + let cache_root = tempfile::tempdir().unwrap(); + let runtime_id = "meshllm-runtime-linux-x86_64-cpu"; + let mut cached = artifact(runtime_id, NativeRuntimeBackend::cpu()); + cached.build_id = cached_build_id.map(str::to_string); + write_bundle_runtime(bundle.path(), cached); + let cache = NativeRuntimeCache::new(cache_root.path()); + let installed = cache.install_from_dir(bundle.path()).unwrap(); + + let mut selected = artifact(runtime_id, NativeRuntimeBackend::cpu()); + selected.build_id = selected_build_id.map(str::to_string); + selected.url = Some("https://example.invalid/runtime.tar.gz".to_string()); + let resolution = NativeRuntimeResolver::new( + "0.68.0", + HostRuntimeProfile { + available_flavors: BTreeSet::from([NativeRuntimeBackendKind::Cpu]), + cuda: None, + ..profile() + }, + NativeRuntimeReleaseManifest { + mesh_version: "0.68.0".to_string(), + skippy_abi: "0.1.25".to_string(), + artifacts: vec![selected], + }, + cache, + ) + .with_skippy_abi_version("0.1.25") + .resolve(&RuntimeSelection::Recommended) + .unwrap(); + assert_eq!( + installed.path, + cache_root.path().join("0.68.0").join(runtime_id) + ); + resolution + } + + #[test] + fn selected_build_does_not_reuse_cached_runtime_without_build_id() { + let resolution = resolve_with_cached_and_selected_build( + None, + Some("sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ); + assert!(matches!( + resolution.source, + NativeRuntimeSource::Download { .. } + )); + } + + #[test] + fn selected_build_does_not_reuse_cached_runtime_with_different_build_id() { + let resolution = resolve_with_cached_and_selected_build( + Some("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + Some("sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), + ); + assert!(matches!( + resolution.source, + NativeRuntimeSource::Download { .. } + )); + } + + #[test] + fn selected_build_reuses_cached_runtime_with_matching_build_id() { + let resolution = resolve_with_cached_and_selected_build( + Some("sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), + Some("sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"), + ); + assert!(matches!( + resolution.source, + NativeRuntimeSource::Installed { .. } + )); + } + + #[test] + fn legacy_selected_artifact_reuses_cached_runtime_regardless_of_build_id() { + let resolution = resolve_with_cached_and_selected_build( + Some("sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"), + None, + ); + assert!(matches!( + resolution.source, + NativeRuntimeSource::Installed { .. } + )); + } + #[test] fn stale_bundle_with_same_id_does_not_satisfy_selected_artifact() { let bundle = tempfile::tempdir().unwrap(); let cache_root = tempfile::tempdir().unwrap(); let runtime_id = "meshllm-runtime-linux-x86_64-cpu"; let stale_bundle_artifact = NativeRuntimeArtifact { + build_id: None, mesh_version: Some("0.67.0".to_string()), ..artifact(runtime_id, NativeRuntimeBackend::cpu()) }; diff --git a/crates/mesh-llm-runtime-install/src/discovery.rs b/crates/mesh-llm-runtime-install/src/discovery.rs index e2f0cf3d4a..5f1b36c7b0 100644 --- a/crates/mesh-llm-runtime-install/src/discovery.rs +++ b/crates/mesh-llm-runtime-install/src/discovery.rs @@ -294,6 +294,7 @@ mod tests { fs::write(path.join("lib/libllama.so"), b"runtime").unwrap(); NativeRuntimeManifest { runtime: NativeRuntimeArtifact { + build_id: None, id: id.to_string(), mesh_version: Some("0.75.0".to_string()), skippy_abi: "0.1.25".to_string(), diff --git a/crates/mesh-llm-system/src/benchmark/tests.rs b/crates/mesh-llm-system/src/benchmark/tests.rs index 757264399e..de0ef0ee70 100644 --- a/crates/mesh-llm-system/src/benchmark/tests.rs +++ b/crates/mesh-llm-system/src/benchmark/tests.rs @@ -342,6 +342,7 @@ fn test_runtime_tool_selection_excludes_preferred_legacy_runtime_without_tool() path: PathBuf::from(format!("/test/{id}")), manifest: NativeRuntimeManifest { runtime: NativeRuntimeArtifact { + build_id: None, id: id.to_string(), mesh_version: Some("0.74.0".to_string()), skippy_abi: "0.1.0".to_string(), diff --git a/scripts/package-native-runtime.sh b/scripts/package-native-runtime.sh index 486d7db71e..0cc8bd396c 100755 --- a/scripts/package-native-runtime.sh +++ b/scripts/package-native-runtime.sh @@ -656,6 +656,22 @@ manifest = { "llama_patch_digest": "$patch_digest" or None, }, } +canonical_runtime = { + key: value + for key, value in manifest["runtime"].items() + if key not in {"url", "sha256", "signature"} +} +build_identity = { + "runtime": canonical_runtime, + "build": manifest["build"], +} +canonical_bytes = json.dumps( + build_identity, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, +).encode("utf-8") +manifest["runtime"]["build_id"] = "sha256:" + hashlib.sha256(canonical_bytes).hexdigest() with open(manifest_path, "w", encoding="utf-8") as fh: json.dump(manifest, fh, indent=2, sort_keys=True) fh.write("\\n") diff --git a/scripts/tests/test_package_native_runtime.py b/scripts/tests/test_package_native_runtime.py index 8a5477a7a8..8e3beb20a0 100644 --- a/scripts/tests/test_package_native_runtime.py +++ b/scripts/tests/test_package_native_runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os from pathlib import Path @@ -13,6 +14,44 @@ class PackageNativeRuntimeTests(unittest.TestCase): + def test_build_id_is_deterministic_and_preserves_runtime_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + build_dir = root / "build" + build_dir.mkdir() + (build_dir / "libggml.so").write_bytes(b"dependency") + (build_dir / "libllama.so").write_bytes(b"primary") + env = self.package_environment(root, build_dir) + + first = self.package_cpu(root / "first", env) + second = self.package_cpu(root / "second", env) + + runtime = first["runtime"] + self.assertEqual(runtime["id"], "meshllm-native-runtime-linux-x86_64-cpu") + self.assertRegex(runtime["build_id"], r"^sha256:[0-9a-f]{64}$") + self.assertEqual(runtime["build_id"], second["runtime"]["build_id"]) + self.assertEqual(runtime["build_id"], self.expected_build_id(first)) + self.assertTrue( + (root / "first" / f'{runtime["id"]}.tar.gz').is_file() + ) + + def test_build_id_changes_when_canonical_runtime_content_changes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + build_dir = root / "build" + build_dir.mkdir() + library = build_dir / "libllama.so" + library.write_bytes(b"first content") + env = self.package_environment(root, build_dir) + first = self.package_cpu(root / "first", env) + + library.write_bytes(b"second content") + second = self.package_cpu(root / "second", env) + + self.assertNotEqual( + first["runtime"]["build_id"], second["runtime"]["build_id"] + ) + def test_cpu_package_with_no_tools_is_safe_under_macos_bash(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -137,6 +176,13 @@ def test_rocm_benchmark_tool_uses_configured_offload_arches(self) -> None: manifest["runtime"]["backend"]["rocm"]["gpu_arches"], ["gfx90a", "gfx942", "gfx1151"], ) + self.assertEqual( + manifest["runtime"]["build_id"], self.expected_build_id(manifest) + ) + self.assertEqual( + list(manifest["runtime"]["tools"]), + ["tools/mesh-llm-gpu-benchmark"], + ) def test_cuda_flavor_uses_mesh_cuda_version_major(self) -> None: self.assertEqual( @@ -173,6 +219,56 @@ def test_explicit_cuda_toolkit_major_rejects_non_digits(self) -> None: result.stderr, ) + def expected_build_id(self, manifest: dict) -> str: + runtime = { + key: value + for key, value in manifest["runtime"].items() + if key not in {"build_id", "url", "sha256", "signature"} + } + canonical = json.dumps( + {"runtime": runtime, "build": manifest["build"]}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + def package_environment(self, root: Path, build_dir: Path) -> dict[str, str]: + tool_dir = root / "tools" + tool_dir.mkdir(exist_ok=True) + patchelf = tool_dir / "patchelf" + patchelf.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + patchelf.chmod(0o755) + env = os.environ.copy() + env["LLAMA_STAGE_BUILD_DIR"] = str(build_dir) + env["PATH"] = f"{tool_dir}{os.pathsep}{env['PATH']}" + return env + + def package_cpu(self, output: Path, env: dict[str, str]) -> dict: + result = subprocess.run( + [ + "/bin/bash", + str(SCRIPT), + "--backend", + "cpu", + "--target", + "x86_64-unknown-linux-gnu", + "--out", + str(output), + ], + env=env, + text=True, + capture_output=True, + ) + result.check_returncode() + return json.loads( + ( + output + / "meshllm-native-runtime-linux-x86_64-cpu" + / "manifest.json" + ).read_text(encoding="utf-8") + ) + def backend_flavor( self, backend: str,