diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml index a7be10c3f..aebf02958 100644 --- a/.github/workflows/publish-cli.yml +++ b/.github/workflows/publish-cli.yml @@ -273,11 +273,26 @@ jobs: shell: bash run: ${{ matrix.settings.strip }} + # Stage the binary plus, for the apple-fm arm64 build, the sidecar + # libFoundationModels.dylib that build.rs places next to it. The dylib is + # dlopen'd at runtime (never linked), so it must travel inside the npm + # package alongside tokscale. Other targets stage only the binary. + - name: Stage release artifact + shell: bash + run: | + mkdir -p dist + cp "target/${{ matrix.settings.target }}/release/${{ matrix.settings.bin_name }}" dist/ + if [ -f "target/${{ matrix.settings.target }}/release/libFoundationModels.dylib" ]; then + cp "target/${{ matrix.settings.target }}/release/libFoundationModels.dylib" dist/ + echo "staged sidecar libFoundationModels.dylib" + fi + ls -la dist + - name: Upload CLI binary artifact uses: actions/upload-artifact@v6 with: name: ${{ matrix.settings.artifact_name }} - path: target/${{ matrix.settings.target }}/release/${{ matrix.settings.bin_name }} + path: dist if-no-files-found: error prepare-release-provenance: @@ -402,6 +417,13 @@ jobs: if [ "${{ matrix.settings.binary_name }}" = "tokscale" ]; then chmod +x "packages/${{ matrix.settings.package_dir }}/bin/tokscale" fi + # Ship the apple-fm sidecar dylib next to the binary when present (only + # the arm64-darwin artifact carries it). tokscale dlopen's it at runtime + # from its own directory; "files": ["bin"] already includes it. + if [ -f "artifacts/${{ matrix.settings.package_dir }}/libFoundationModels.dylib" ]; then + cp "artifacts/${{ matrix.settings.package_dir }}/libFoundationModels.dylib" "packages/${{ matrix.settings.package_dir }}/bin/libFoundationModels.dylib" + echo "shipped sidecar libFoundationModels.dylib" + fi ls -la "packages/${{ matrix.settings.package_dir }}/bin" - name: Publish platform package diff --git a/README.ja.md b/README.ja.md index 85f68d297..2d04a9651 100644 --- a/README.ja.md +++ b/README.ja.md @@ -584,7 +584,7 @@ LLM 要約は**デフォルトで有効**になっています（`--no-summarize | バックエンド | コマンド | 備考 | |---------|---------|-------| -| `apple-fm` | （デフォルト） | ネイティブ Rust FFI 経由でオンデバイスの Apple Foundation Models を使用します（Python 不要）。`apple-fm` Cargo フィーチャーを有効にした macOS ビルドと、Apple Intelligence が有効な環境が必要です。それ以外の場合は組み込みの Rust ヒューリスティック分類器に透過的にフォールバックするため、デフォルトはどこでも動作します。 | +| `apple-fm` | （デフォルト） | ネイティブ Rust FFI 経由のオンデバイス Apple Foundation Models（Python 不要）。ビルド済みの Apple Silicon（macOS arm64）バイナリで有効化されており、Apple Intelligence を有効にした macOS 26 以降で動作します。それ以外（Intel Mac、それ以前の macOS、Linux、Windows）では組み込みの Rust ヒューリスティックに透過的にフォールバックするため、デフォルトはすべてのプラットフォームで動作します。 | | `claude` | `claude -p` | Claude Code CLI がインストールされ認証済みである必要があります。 | | `codex` | `codex --quiet` | Codex CLI がインストールされ認証済みである必要があります。 | | `gemini` | `gemini -p` | Gemini CLI がインストールされ認証済みである必要があります。 | diff --git a/README.ko.md b/README.ko.md index 70741b58a..b2d4b0dca 100644 --- a/README.ko.md +++ b/README.ko.md @@ -582,7 +582,7 @@ tokscale report --workspace my-project --client opencode | 백엔드 | 명령어 | 비고 | |---------|---------|-------| -| `apple-fm` | (기본값) | 네이티브 Rust FFI를 통해 Apple Foundation Models를 온디바이스에서 사용 (Python 불필요). `apple-fm` Cargo 피처를 활성화한 macOS 빌드와 Apple Intelligence가 필요하며, 그렇지 않은 경우 내장 Rust 휴리스틱 분류기로 투명하게 폴백합니다 (기본값은 어디서나 동작). | +| `apple-fm` | (기본값) | 네이티브 Rust FFI를 통한 온디바이스 Apple Foundation Models (Python 불필요). 사전 빌드된 Apple Silicon(macOS arm64) 바이너리에 기본 포함되어 있으며, Apple Intelligence가 켜진 macOS 26 이상에서 동작합니다. 그 외 환경(Intel Mac, 이전 macOS, Linux, Windows)에서는 내장 Rust 휴리스틱으로 투명하게 폴백하므로 기본값은 모든 플랫폼에서 동작합니다. | | `claude` | `claude -p` | Claude Code CLI가 설치되어 인증되어 있어야 함. | | `codex` | `codex --quiet` | Codex CLI가 설치되어 인증되어 있어야 함. | | `gemini` | `gemini -p` | Gemini CLI가 설치되어 인증되어 있어야 함. | diff --git a/README.md b/README.md index 87f03fae4..49fb393c4 100644 --- a/README.md +++ b/README.md @@ -681,7 +681,7 @@ tokscale report --workspace my-project --client opencode | Backend | Command | Notes | |---------|---------|-------| -| `apple-fm` | (default) | Uses Apple Foundation Models on-device via native Rust FFI (no Python). Requires a macOS build with the `apple-fm` Cargo feature and Apple Intelligence enabled; otherwise it transparently falls back to a built-in Rust heuristic classifier (so the default works everywhere). | +| `apple-fm` | (default) | On-device Apple Foundation Models via native Rust FFI (no Python). Enabled in the prebuilt Apple Silicon (macOS arm64) binary; runs on macOS 26+ with Apple Intelligence on, and transparently falls back to a built-in Rust heuristic everywhere else (Intel Macs, older macOS, Linux, Windows) — so the default works on every platform. | | `claude` | `claude -p` | Requires Claude Code CLI installed and authenticated. | | `codex` | `codex --quiet` | Requires Codex CLI installed and authenticated. | | `gemini` | `gemini -p` | Requires Gemini CLI installed and authenticated. | diff --git a/README.zh-cn.md b/README.zh-cn.md index ffb8c7a57..666423c20 100644 --- a/README.zh-cn.md +++ b/README.zh-cn.md @@ -588,7 +588,7 @@ tokscale report --workspace my-project --client opencode | 后端 | 命令 | 说明 | |---------|---------|-------| -| `apple-fm` | （默认） | 通过原生 Rust FFI 在本地使用 Apple Foundation Models（无需 Python）。需要启用 `apple-fm` Cargo 特性的 macOS 构建，且须开启 Apple Intelligence；否则将透明回退至内置 Rust 启发式分类器（因此默认配置可在所有平台正常使用）。 | +| `apple-fm` | （默认） | 通过原生 Rust FFI 在本地使用 Apple Foundation Models（无需 Python）。已在预构建的 Apple Silicon（macOS arm64）二进制中启用，在开启 Apple Intelligence 的 macOS 26 及以上系统运行；在其他环境（Intel Mac、更旧的 macOS、Linux、Windows）则透明回退至内置 Rust 启发式分类器，因此默认配置可在所有平台正常使用。 | | `claude` | `claude -p` | 需要已安装并已认证的 Claude Code CLI。 | | `codex` | `codex --quiet` | 需要已安装并已认证的 Codex CLI。 | | `gemini` | `gemini -p` | 需要已安装并已认证的 Gemini CLI。 | diff --git a/crates/tokscale-cli/build.rs b/crates/tokscale-cli/build.rs index 6c590fe2d..37bb0ee47 100644 --- a/crates/tokscale-cli/build.rs +++ b/crates/tokscale-cli/build.rs @@ -2,12 +2,28 @@ //! //! When (and only when) the optional `apple-fm` feature is enabled AND the //! target OS is macOS, this builds the vendored `foundation-models-c` SwiftPM -//! package and links the resulting `libFoundationModels.dylib`. +//! package as a DYNAMIC `libFoundationModels.dylib` and stages it next to the +//! final binary. +//! +//! The dylib is deliberately NOT linked into `tokscale`. Apple's +//! `FoundationModels.framework` only exists on macOS 26+, and the Swift runtime +//! the dylib pulls in (e.g. `libswiftSynchronization`, macOS 15+) does too; +//! hard-linking any of them would make the *whole* CLI fail to `dyld`-load on +//! older macOS — a crash-on-launch for every command, not a feature fallback. +//! Worse, `import FoundationModels` autolinks the framework as a NON-weak load +//! command, so a `-weak_framework` flag can't reliably flip it. +//! +//! Instead the binary links nothing FM/Swift (verifiable: `otool -L tokscale` +//! shows no FoundationModels and no libswift*), and the `apple-fm` code path +//! `dlopen`s this dylib lazily at runtime — only on macOS 26+, where all its +//! dependencies are present. On older macOS the `dlopen` simply fails and the +//! caller degrades to the cross-platform Rust heuristic. This keeps a SINGLE +//! arm64 binary safe to ship to every Apple Silicon Mac via npm. //! //! When the feature is off, or the target is not macOS, this build script is a //! complete no-op so that cross-platform / default builds are unaffected. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; fn main() { @@ -51,14 +67,17 @@ fn build_apple_fm() { pkg_dir.join("Sources").display() ); - // Build the SwiftPM package in release mode. + // Build the DYNAMIC `FoundationModels` product (`libFoundationModels.dylib`) + // in release mode. We do not build/link the static archive: the dylib is + // loaded at runtime via `dlopen`, so nothing FM/Swift ends up in the + // tokscale binary's load commands. let status = Command::new("swift") .args([ "build", "-c", "release", "--product", - "FoundationModelsStatic", + "FoundationModels", "--package-path", ]) .arg(&pkg_dir) @@ -78,12 +97,7 @@ fn build_apple_fm() { ); } - // Copy the STATIC archive into OUT_DIR and link it statically, so the final - // tokscale binary is self-contained — no `libFoundationModels.dylib` to ship - // alongside it. The archive's only remaining dependencies are Apple's system - // FoundationModels framework and the OS Swift runtime, both always present on - // macOS 26 (verified with `otool -L`: no non-system dylib references). - let lib_name = "libFoundationModelsStatic.a"; + let lib_name = "libFoundationModels.dylib"; let built_lib = pkg_dir.join(".build/release").join(lib_name); if !built_lib.exists() { panic!( @@ -91,25 +105,45 @@ fn build_apple_fm() { built_lib.display() ); } - let dest_lib = Path::new(&out_dir).join(lib_name); - std::fs::copy(&built_lib, &dest_lib).unwrap_or_else(|e| { + + // 1) Copy into OUT_DIR and bake its absolute path into the binary as a + // fallback. This is what `cargo test` / `cargo run` from arbitrary CWDs + // resolve to (the test harness binary lives in target//deps, so + // a sibling-of-exe copy alone would not be found there). + let out_lib = Path::new(&out_dir).join(lib_name); + copy(&built_lib, &out_lib); + println!("cargo:rustc-env=TOKSCALE_FM_DYLIB={}", out_lib.display()); + + // 2) Stage a copy NEXT TO the final binary, so the primary runtime lookup + // (`current_exe()`'s directory) succeeds for both `cargo run` and the + // shipped npm package, where the dylib travels alongside `tokscale`. + // + // OUT_DIR is `.../target///build/-/out`; + // ascending three parents lands on the profile dir that holds the binary. + if let Some(profile_dir) = profile_dir_from_out(&out_dir) { + let staged = profile_dir.join(lib_name); + copy(&built_lib, &staged); + // CI's release step copies this sibling dylib into the npm package's + // bin/ next to tokscale; surface its path for that step / debugging. + println!("cargo:warning=apple-fm: staged {}", staged.display()); + } +} + +/// `...//build/-/out` -> `.../`. +fn profile_dir_from_out(out_dir: &str) -> Option { + Path::new(out_dir) + .parent() // - + .and_then(Path::parent) // build + .and_then(Path::parent) // + .map(Path::to_path_buf) +} + +fn copy(from: &Path, to: &Path) { + std::fs::copy(from, to).unwrap_or_else(|e| { panic!( "apple-fm: failed to copy {} -> {}: {e}", - built_lib.display(), - dest_lib.display() + from.display(), + to.display() ) }); - - // Statically link the bindings archive, plus the system FoundationModels - // framework and the OS Swift runtime search path. The archive also carries - // autolink hints, but these are made explicit for a deterministic link. - println!("cargo:rustc-link-search=native={out_dir}"); - println!("cargo:rustc-link-lib=static=FoundationModelsStatic"); - println!("cargo:rustc-link-lib=framework=FoundationModels"); - println!("cargo:rustc-link-search=native=/usr/lib/swift"); - // The Swift runtime dylibs (e.g. libswift_Concurrency.dylib) are referenced - // via `@rpath`. They live in /usr/lib/swift, which is part of every macOS 26 - // install's dyld shared cache, so baking this system rpath keeps the binary - // self-contained (it needs only OS-provided libraries at runtime). - println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); } diff --git a/crates/tokscale-cli/src/commands/apple_fm.rs b/crates/tokscale-cli/src/commands/apple_fm.rs index bb7643915..cabc4afee 100644 --- a/crates/tokscale-cli/src/commands/apple_fm.rs +++ b/crates/tokscale-cli/src/commands/apple_fm.rs @@ -10,8 +10,36 @@ //! back to the Rust heuristic ([`heuristic_classify`]), which is always //! available and cross-platform. //! -//! Availability gate: [`summarize`] returns `None` (never errors) when Apple -//! Intelligence is unavailable, so the caller degrades to the heuristic. +//! ## Why `dlopen` instead of linking +//! +//! `FoundationModels.framework` only exists on macOS 26+, and the Swift runtime +//! the shim drags in (`libswiftSynchronization`, macOS 15+, etc.) does too. If +//! the `tokscale` binary hard-linked any of them it would fail to `dyld`-load on +//! older macOS — a crash-on-launch for EVERY command, not a feature fallback. +//! And `import FoundationModels` autolinks the framework as a non-weak load +//! command, so a `-weak_framework` flag can't reliably flip it. +//! +//! So the binary links nothing FM/Swift (verifiable: `otool -L tokscale` shows +//! no FoundationModels and no `libswift*`). The vendored shim is built as a +//! DYNAMIC `libFoundationModels.dylib` (see `build.rs`) staged next to the +//! binary, and this module `dlopen`s it lazily — only on macOS 26+, where all +//! its dependencies exist. On older macOS the `dlopen` simply fails and the +//! caller degrades to the heuristic. This keeps a SINGLE arm64 binary safe to +//! ship to every Apple Silicon Mac via npm, regardless of their macOS version. +//! +//! Availability gate: [`summarize`] returns `None` (never errors) when the +//! dylib can't be loaded (old macOS / missing file) OR Apple Intelligence is +//! unavailable, so the caller degrades to the heuristic. +//! +//! Smoke-testing note: the vendored `fm-c-example` binary's streaming path +//! (`FMLanguageModelSessionStreamResponse`) hard-segfaults (EXC_BAD_ACCESS in +//! `objc_retain`) on macOS 26.2, so it is NOT a valid liveness check for "is FM +//! working on this box". This module uses only the non-streaming +//! `FMLanguageModelSessionRespondWithSchema` path (a PROGRAMMATIC +//! GenerationSchema built via [`imp::build_schema`], NOT the JSON-Schema-string +//! `...FromJSON` variant) and is unaffected. For end-to-end verification use the +//! `#[ignore]`d live test in this module (`live_summarize_smoke`), not the +//! streaming example. /// Input metadata for one coding session to be summarized. /// @@ -112,7 +140,10 @@ mod imp { use super::{heuristic_classify, SessionInput, SessionSummary}; use super::{VALID_CATEGORIES, VALID_COMPLEXITIES}; use std::ffi::{c_char, c_int, c_void, CStr, CString}; + use std::os::unix::ffi::OsStrExt; + use std::path::PathBuf; use std::sync::mpsc; + use std::sync::OnceLock; use std::time::Duration; /// Upper bound on a single on-device generation. A short classification @@ -121,58 +152,276 @@ mod imp { /// timeout the session falls back to the heuristic. const FM_GENERATION_TIMEOUT: Duration = Duration::from_secs(60); + /// Upper bound on the first-user-message text appended to the prompt. The + /// on-device model has a small context window, so a large pasted message + /// (stack trace, file dump, multi-KB prompt) is truncated here. Larger than + /// the CLI backend's 200-char cap since the FM prompt carries only one + /// session at a time. + const MAX_FIRST_USER_MESSAGE_CHARS: usize = 1000; + + /// First macOS major version that ships `FoundationModels.framework`. + const FM_MIN_MACOS_MAJOR: u32 = 26; + /// Verbatim system instructions for the classifier (matches the former /// Python backend exactly). const SYSTEM_INSTRUCTIONS: &str = "You are a coding session classifier. Given metadata about an AI coding session, produce a structured summary.\n\nRules:\n- title: 3-8 word description of what was done (imperative mood, e.g. \"Add JWT auth middleware\")\n- task_category: exactly one of: feature, bugfix, refactor, research, debug, review, docs, config, other\n- description: 1-2 sentences explaining what happened in the session\n- complexity: exactly one of: trivial, moderate, complex\n\nBase your classification on:\n- The first user message (primary signal)\n- The workspace name (project context)\n- Token count and duration (complexity signal)\n- Models used (opus = likely complex, haiku = likely trivial)\n\nRespond ONLY with valid JSON matching the schema."; - /// JSON-schema string passed to `RespondWithSchemaFromJSON`. A simple - /// object-with-properties; the result is re-validated in Rust regardless. - const SCHEMA_JSON: &str = r#"{ - "type": "object", - "properties": { - "title": { "type": "string" }, - "task_category": { - "type": "string", - "enum": ["feature", "bugfix", "refactor", "research", "debug", "review", "docs", "config", "other"] - }, - "description": { "type": "string" }, - "complexity": { - "type": "string", - "enum": ["trivial", "moderate", "complex"] - } - }, - "required": ["title", "task_category", "description", "complexity"] -}"#; - // Opaque FoundationModels handles. All are `const void*` in the C ABI. type FMRef = *const c_void; /// Callback signature: `void (*)(int status, FMGeneratedContentRef content, void* userInfo)`. type StructuredCallback = extern "C" fn(status: c_int, content: FMRef, user_info: *mut c_void); - #[allow(non_snake_case)] + // --- dl* / sysctl: libSystem symbols, ALWAYS present, no FM/Swift linkage. extern "C" { - fn FMSystemLanguageModelGetDefault() -> FMRef; - fn FMSystemLanguageModelIsAvailable(model: FMRef, unavailable_reason: *mut c_int) -> bool; - fn FMLanguageModelSessionCreateFromSystemLanguageModel( - model: FMRef, - instructions: *const c_char, - tools: *mut FMRef, - tool_count: c_int, - ) -> FMRef; - fn FMComposedPromptInitialize() -> FMRef; - fn FMComposedPromptAddText(composed_prompt: FMRef, text: *const c_char); - fn FMLanguageModelSessionRespondWithSchemaFromJSON( - session: FMRef, - composed_prompt: FMRef, - schema_json: *const c_char, - options_json: *const c_char, - user_info: *mut c_void, - callback: StructuredCallback, - ) -> FMRef; - fn FMGeneratedContentGetJSONString(content: FMRef) -> *mut c_char; - fn FMRelease(object: FMRef); - fn FMFreeString(s: *mut c_char); + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + fn dlerror() -> *const c_char; + fn sysctlbyname( + name: *const c_char, + oldp: *mut c_void, + oldlenp: *mut usize, + newp: *mut c_void, + newlen: usize, + ) -> c_int; + } + const RTLD_NOW: c_int = 2; + const RTLD_LOCAL: c_int = 4; + + // --- Resolved FoundationModels C-ABI entry points (loaded via dlsym). The + // signatures mirror the vendored `foundation-models-c` header exactly. + type FnGetDefault = unsafe extern "C" fn() -> FMRef; + type FnIsAvailable = unsafe extern "C" fn(FMRef, *mut c_int) -> bool; + type FnSessionCreate = unsafe extern "C" fn(FMRef, *const c_char, *mut FMRef, c_int) -> FMRef; + type FnPromptInit = unsafe extern "C" fn() -> FMRef; + type FnPromptAddText = unsafe extern "C" fn(FMRef, *const c_char); + type FnSchemaCreate = unsafe extern "C" fn(*const c_char, *const c_char) -> FMRef; + type FnPropertyCreate = + unsafe extern "C" fn(*const c_char, *const c_char, *const c_char, bool) -> FMRef; + type FnPropertyAddAnyOf = unsafe extern "C" fn(FMRef, *const *const c_char, c_int, bool); + type FnSchemaAddProperty = unsafe extern "C" fn(FMRef, FMRef); + type FnRespondWithSchema = unsafe extern "C" fn( + FMRef, + FMRef, + FMRef, + *const c_char, + *mut c_void, + StructuredCallback, + ) -> FMRef; + type FnContentGetJSON = unsafe extern "C" fn(FMRef) -> *mut c_char; + type FnRelease = unsafe extern "C" fn(FMRef); + type FnFreeString = unsafe extern "C" fn(*mut c_char); + + /// The resolved FoundationModels API plus the (never-closed) dlopen handle. + /// + /// `FMGenerationSchemaCreate` / `...PropertyCreate` return a +1-retained ref + /// (`Unmanaged.passRetained`), so each must be `release`d. The builder's + /// `addProperty` copies the property into a Swift array (it holds its own + /// strong reference), so a property ref may be released immediately after + /// `schema_add_property`. The structured-response callback receives a + /// `content` handle the shim hands over with a +1 retain on EVERY invocation + /// (success AND error/cancel); `content_get_json` only borrows it, so the + /// callback owns that +1 and must `release(content)` exactly once on every + /// path or one generated-content wrapper leaks per generation. + #[allow(dead_code)] + struct Api { + /// Kept alive for the process lifetime; the dylib is never `dlclose`d. + handle: *mut c_void, + get_default: FnGetDefault, + is_available: FnIsAvailable, + session_create: FnSessionCreate, + prompt_init: FnPromptInit, + prompt_add_text: FnPromptAddText, + schema_create: FnSchemaCreate, + property_create: FnPropertyCreate, + property_add_anyof: FnPropertyAddAnyOf, + schema_add_property: FnSchemaAddProperty, + respond_with_schema: FnRespondWithSchema, + content_get_json: FnContentGetJSON, + release: FnRelease, + free_string: FnFreeString, + } + + // `Api` holds only function pointers and an opaque handle; the function + // pointers are immutable after load and safe to call from any thread (the + // background structured callback reads them via `api()`). + unsafe impl Send for Api {} + unsafe impl Sync for Api {} + + impl Api { + /// dlsym every entry point off `handle`. Returns `None` if any symbol is + /// missing (treated as "FM unavailable" -> heuristic). + unsafe fn load(handle: *mut c_void) -> Option { + // dlsym + transmute one symbol. `transmute_copy` because `T` is a + // (pointer-sized) fn-pointer type and plain `transmute` can't prove + // size equality for a generic. + unsafe fn sym(handle: *mut c_void, name: &[u8]) -> Option { + debug_assert_eq!( + name.last(), + Some(&0u8), + "symbol name must be NUL-terminated" + ); + let p = dlsym(handle, name.as_ptr() as *const c_char); + if p.is_null() { + None + } else { + Some(std::mem::transmute_copy::<*mut c_void, T>(&p)) + } + } + + Some(Api { + handle, + get_default: sym(handle, b"FMSystemLanguageModelGetDefault\0")?, + is_available: sym(handle, b"FMSystemLanguageModelIsAvailable\0")?, + session_create: sym( + handle, + b"FMLanguageModelSessionCreateFromSystemLanguageModel\0", + )?, + prompt_init: sym(handle, b"FMComposedPromptInitialize\0")?, + prompt_add_text: sym(handle, b"FMComposedPromptAddText\0")?, + schema_create: sym(handle, b"FMGenerationSchemaCreate\0")?, + property_create: sym(handle, b"FMGenerationSchemaPropertyCreate\0")?, + property_add_anyof: sym(handle, b"FMGenerationSchemaPropertyAddAnyOfGuide\0")?, + schema_add_property: sym(handle, b"FMGenerationSchemaAddProperty\0")?, + respond_with_schema: sym(handle, b"FMLanguageModelSessionRespondWithSchema\0")?, + content_get_json: sym(handle, b"FMGeneratedContentGetJSONString\0")?, + release: sym(handle, b"FMRelease\0")?, + free_string: sym(handle, b"FMFreeString\0")?, + }) + } + } + + /// Read the macOS major version via `sysctl kern.osproductversion` + /// (e.g. `"26.1"` -> `26`). `None` if it can't be determined. + fn macos_major() -> Option { + let name = b"kern.osproductversion\0"; + let mut size: usize = 0; + // Probe the buffer size. + let rc = unsafe { + sysctlbyname( + name.as_ptr() as *const c_char, + std::ptr::null_mut(), + &mut size, + std::ptr::null_mut(), + 0, + ) + }; + if rc != 0 || size == 0 { + return None; + } + let mut buf = vec![0u8; size]; + let rc = unsafe { + sysctlbyname( + name.as_ptr() as *const c_char, + buf.as_mut_ptr() as *mut c_void, + &mut size, + std::ptr::null_mut(), + 0, + ) + }; + if rc != 0 { + return None; + } + let s = CStr::from_bytes_until_nul(&buf).ok()?.to_str().ok()?; + s.split('.').next()?.parse::().ok() + } + + /// Candidate `libFoundationModels.dylib` locations, in priority order: + /// 1. next to the running binary (npm package layout, `cargo run`), + /// 2. the absolute OUT_DIR copy baked in at build time (`cargo test`, where + /// the test harness binary lives in `target//deps`). + fn candidate_paths() -> Vec { + let mut v = Vec::new(); + if let Ok(exe) = std::env::current_exe() { + if let Some(dir) = exe.parent() { + v.push(dir.join("libFoundationModels.dylib")); + } + } + if let Some(p) = option_env!("TOKSCALE_FM_DYLIB") { + v.push(PathBuf::from(p)); + } + v + } + + /// Load (once) the FoundationModels dylib and resolve its entry points. + /// + /// Returns `None` — caller falls back to the heuristic — when: + /// - the OS is older than macOS 26 (no `FoundationModels.framework`), or + /// - the dylib isn't found / can't be loaded (e.g. dependencies absent), or + /// - an expected symbol is missing. + fn load_api() -> Option { + // Set `TOKSCALE_FM_DEBUG=1` to trace why apple-fm did/didn't engage + // (OS gate, which dylib path loaded, dlopen errors, symbol resolution). + let debug = std::env::var_os("TOKSCALE_FM_DEBUG").is_some(); + + // Fast OS gate. The dylib's transitive deps (FoundationModels.framework + // + macOS-26 Swift runtime) only exist on macOS 26+, so on older systems + // the dlopen below would fail anyway; this documents the contract and + // avoids a doomed load attempt. + let major = macos_major(); + if debug { + eprintln!(" apple-fm[debug]: macos_major={major:?}"); + } + if let Some(major) = major { + if major < FM_MIN_MACOS_MAJOR { + if debug { + eprintln!( + " apple-fm[debug]: OS gate {major} < {FM_MIN_MACOS_MAJOR} -> heuristic" + ); + } + return None; + } + } + + for path in candidate_paths() { + let exists = path.exists(); + let c = match CString::new(path.as_os_str().as_bytes()) { + Ok(c) => c, + Err(_) => continue, + }; + let handle = unsafe { dlopen(c.as_ptr(), RTLD_NOW | RTLD_LOCAL) }; + if handle.is_null() { + if debug { + let err = unsafe { dlerror() }; + let msg = if err.is_null() { + "(no dlerror)".to_string() + } else { + unsafe { CStr::from_ptr(err) } + .to_string_lossy() + .into_owned() + }; + eprintln!( + " apple-fm[debug]: dlopen failed (exists={exists}) {} :: {msg}", + path.display() + ); + } + continue; + } + if debug { + eprintln!(" apple-fm[debug]: dlopen ok {}", path.display()); + } + // Leave the handle open for the process lifetime on success; on a + // (very unexpected) missing symbol, move on to the next candidate. + if let Some(api) = unsafe { Api::load(handle) } { + return Some(api); + } + if debug { + eprintln!( + " apple-fm[debug]: symbol resolution failed for {}", + path.display() + ); + } + } + if debug { + eprintln!(" apple-fm[debug]: no usable FoundationModels dylib -> heuristic"); + } + None + } + + /// Process-wide resolved API, or `None` if FM is unavailable on this box. + fn api() -> Option<&'static Api> { + static API: OnceLock> = OnceLock::new(); + API.get_or_init(load_api).as_ref() } /// What the background callback ships back to the blocked calling thread: @@ -194,23 +443,39 @@ mod imp { } let cb: Box = unsafe { Box::from_raw(user_info as *mut CallbackBox) }; - let result: CallbackResult = if status != 0 || content.is_null() { - Err(status) - } else { - // SAFETY: content is non-null per the check above; the returned - // string is malloc'd and must be freed via FMFreeString. - let json_ptr = unsafe { FMGeneratedContentGetJSONString(content) }; - if json_ptr.is_null() { - Err(status) - } else { - let json = unsafe { CStr::from_ptr(json_ptr) } - .to_string_lossy() - .into_owned(); - unsafe { FMFreeString(json_ptr) }; - Ok(json) + // The callback can only fire after a successful `respond_with_schema` + // call, which required `api()` to be `Some`; so this lookup never fails + // in practice, but we degrade to `Err(status)` if it somehow does. + let api = api(); + + let result: CallbackResult = match api { + Some(api) if status == 0 && !content.is_null() => { + // SAFETY: content is non-null; the returned string is malloc'd + // and must be freed via `free_string`. + let json_ptr = unsafe { (api.content_get_json)(content) }; + if json_ptr.is_null() { + Err(status) + } else { + let json = unsafe { CStr::from_ptr(json_ptr) } + .to_string_lossy() + .into_owned(); + unsafe { (api.free_string)(json_ptr) }; + Ok(json) + } } + _ => Err(status), }; + // The shim hands us a +1-retained `content` on EVERY callback path + // (success and error/cancel) and `content_get_json` only borrows it, so + // we own that retain and must release it here exactly once or one + // generated-content wrapper leaks per generation. + if !content.is_null() { + if let Some(api) = api { + unsafe { (api.release)(content) }; + } + } + // Best-effort send; if the receiver is gone there is nothing to do. let _ = cb.tx.send(result); } @@ -237,7 +502,11 @@ mod imp { match &input.first_user_message { Some(msg) if !msg.is_empty() => { s.push_str("\n\nFirst user message:\n"); - s.push_str(msg); + // Cap the (possibly multi-KB) pasted message: on-device FM has a + // small context window, so an oversized prompt risks truncation, + // refusal, or latency. `chars().take` is char-boundary-safe. + let capped: String = msg.chars().take(MAX_FIRST_USER_MESSAGE_CHARS).collect(); + s.push_str(&capped); } _ => { s.push_str("\n\nNo user message content available."); @@ -298,29 +567,102 @@ mod imp { }) } + /// Build the programmatic `SessionSummary` GenerationSchema once, enforcing + /// the category/complexity enums on-device via `anyOf` guides. Returns a + /// +1-retained schema ref the caller must `release` after the per-session + /// loop, or `None` if any CString conversion fails. + /// + /// typeName is the lowercase `"string"` literal: the shim matches it with + /// `case "string":` (FoundationModelsCBindings.swift) to produce a + /// `String`-typed property. Any other casing (e.g. "String") falls through + /// to the "reference to another schema" branch and fails to build. The + /// `anyOf` guide is added UNWRAPPED (`wrapped=false`): for a scalar String + /// the shim's `resolveStringGuides` handles `.anyOf` directly, whereas a + /// wrapped (`.element`) guide is only valid for array types and would throw + /// `unsupportedGuide`. + fn build_schema(api: &Api) -> Option { + // typeName literal the shim maps to a Swift `String` property. + let type_string = CString::new("string").ok()?; + let schema_name = CString::new("SessionSummary").ok()?; + let schema = unsafe { (api.schema_create)(schema_name.as_ptr(), std::ptr::null()) }; + if schema.is_null() { + return None; + } + + // Helper: create a property, optionally constrain it to an enum set via + // an unwrapped anyOf guide, add it to the schema, then release the + // property's +1 retain (the builder copied it into its own array). + let add_prop = |name: &str, choices: Option<&[&str]>| -> Option<()> { + let name_c = CString::new(name).ok()?; + let prop = unsafe { + (api.property_create)( + name_c.as_ptr(), + std::ptr::null(), + type_string.as_ptr(), + false, + ) + }; + if prop.is_null() { + return None; + } + if let Some(choices) = choices { + // Keep the CStrings alive until after the FFI call. + let owned: Vec = choices + .iter() + .map(|c| CString::new(*c)) + .collect::>() + .ok()?; + let ptrs: Vec<*const c_char> = owned.iter().map(|c| c.as_ptr()).collect(); + unsafe { + (api.property_add_anyof)(prop, ptrs.as_ptr(), ptrs.len() as c_int, false); + } + } + unsafe { + (api.schema_add_property)(schema, prop); + // Builder holds its own strong ref; release our creation +1. + (api.release)(prop); + } + Some(()) + }; + + // On any property failure, release the schema and bail. + let built = (|| { + add_prop("title", None)?; + add_prop("description", None)?; + add_prop("task_category", Some(VALID_CATEGORIES))?; + add_prop("complexity", Some(VALID_COMPLEXITIES))?; + Some(()) + })(); + if built.is_none() { + unsafe { (api.release)(schema) }; + return None; + } + + Some(schema) + } + /// Run a single structured generation for `input`, blocking the calling /// thread until the background callback fires. Returns the parsed summary, /// or `None` on any error (caller falls back to the heuristic). /// + /// `schema` is the prebuilt, shared GenerationSchema ref (see + /// [`build_schema`]); the shim borrows it unretained per call, so it stays + /// owned by the caller across the loop. + /// /// A FRESH `LanguageModelSession` is created per input: the session is /// stateful (it accumulates a transcript), so reusing one across sessions /// would condition later summaries on earlier prompts/responses — and a /// timed-out generation could leave a shared session busy. This mirrors the /// former Python backend, which built a new session inside its loop. fn respond_one( + api: &Api, model: FMRef, instructions: &CStr, - schema: &CStr, + schema: FMRef, input: &SessionInput, ) -> Option { - let session_ref = unsafe { - FMLanguageModelSessionCreateFromSystemLanguageModel( - model, - instructions.as_ptr(), - std::ptr::null_mut(), - 0, - ) - }; + let session_ref = + unsafe { (api.session_create)(model, instructions.as_ptr(), std::ptr::null_mut(), 0) }; if session_ref.is_null() { return None; } @@ -330,26 +672,26 @@ mod imp { let prompt_text = match CString::new(build_prompt(input)) { Ok(c) => c, Err(_) => { - unsafe { FMRelease(session_ref) }; + unsafe { (api.release)(session_ref) }; return None; } }; - let prompt_ref = unsafe { FMComposedPromptInitialize() }; + let prompt_ref = unsafe { (api.prompt_init)() }; if prompt_ref.is_null() { - unsafe { FMRelease(session_ref) }; + unsafe { (api.release)(session_ref) }; return None; } - unsafe { FMComposedPromptAddText(prompt_ref, prompt_text.as_ptr()) }; + unsafe { (api.prompt_add_text)(prompt_ref, prompt_text.as_ptr()) }; let (tx, rx) = mpsc::channel::(); let cb_box = Box::new(CallbackBox { tx }); let user_info = Box::into_raw(cb_box) as *mut c_void; let task_ref = unsafe { - FMLanguageModelSessionRespondWithSchemaFromJSON( + (api.respond_with_schema)( session_ref, prompt_ref, - schema.as_ptr(), + schema, std::ptr::null(), user_info, structured_callback, @@ -357,20 +699,40 @@ mod imp { }; // Block on the background callback (bounded). The callback reclaims - // `user_info`; on timeout the box is intentionally leaked rather than - // risk a use-after-free if the callback fires later. + // `user_info` (the boxed sender). On timeout we deliberately do NOT + // reclaim it here: the detached Swift task may still fire the callback + // later, so freeing the box now would risk a use-after-free. The box + // (one channel sender) is leaked instead — a bounded, rare cost paid + // only when a generation exceeds the 60s timeout. let received = rx.recv_timeout(FM_GENERATION_TIMEOUT); // Release the task handle, composed prompt, and this input's session. if !task_ref.is_null() { - unsafe { FMRelease(task_ref) }; + unsafe { (api.release)(task_ref) }; } - unsafe { FMRelease(prompt_ref) }; - unsafe { FMRelease(session_ref) }; + unsafe { (api.release)(prompt_ref) }; + unsafe { (api.release)(session_ref) }; match received { Ok(Ok(json)) => parse_summary(&input.session_id, &json), - _ => None, + // Surface the failure mode so silent degradation to the heuristic is + // diagnosable (the FM-vs-heuristic breakdown reports the count; this + // names the cause). Non-success status codes flow through here. + Ok(Err(status)) => { + eprintln!( + " apple-fm: generation failed for {} (status {}); using heuristic", + input.session_id, status + ); + None + } + Err(_) => { + eprintln!( + " apple-fm: generation timed out for {} after {}s; using heuristic", + input.session_id, + FM_GENERATION_TIMEOUT.as_secs() + ); + None + } } } @@ -380,14 +742,18 @@ mod imp { return Some(Vec::new()); } + // 0) Resolve the dylib + entry points (OS gate + dlopen happen here). + // `None` => old macOS / dylib missing => caller uses the heuristic. + let api = api()?; + // 1) Default model + availability gate. NEVER generate if unavailable. - let model = unsafe { FMSystemLanguageModelGetDefault() }; + let model = unsafe { (api.get_default)() }; if model.is_null() { return None; } - let available = unsafe { FMSystemLanguageModelIsAvailable(model, std::ptr::null_mut()) }; + let available = unsafe { (api.is_available)(model, std::ptr::null_mut()) }; if !available { - unsafe { FMRelease(model) }; + unsafe { (api.release)(model) }; return None; } @@ -396,14 +762,17 @@ mod imp { let instructions = match CString::new(SYSTEM_INSTRUCTIONS) { Ok(c) => c, Err(_) => { - unsafe { FMRelease(model) }; + unsafe { (api.release)(model) }; return None; } }; - let schema = match CString::new(SCHEMA_JSON) { - Ok(c) => c, - Err(_) => { - unsafe { FMRelease(model) }; + // Build the programmatic output schema ONCE and share it across the + // per-session loop (the shim borrows it unretained per call). Released + // after the loop. If schema construction fails, fall back entirely. + let schema = match build_schema(api) { + Some(s) => s, + None => { + unsafe { (api.release)(model) }; return None; } }; @@ -412,13 +781,16 @@ mod imp { // back to the heuristic for that single session. let mut results = Vec::with_capacity(sessions.len()); for input in sessions { - match respond_one(model, instructions.as_c_str(), schema.as_c_str(), input) { + match respond_one(api, model, instructions.as_c_str(), schema, input) { Some(summary) => results.push(summary), None => results.push(heuristic_classify(input)), } } - unsafe { FMRelease(model) }; + unsafe { + (api.release)(schema); + (api.release)(model); + } Some(results) } @@ -537,4 +909,65 @@ mod tests { // return None (unavailable) or Some. Either way it must not panic. let _ = summarize(&[input(1, 1, "/x/p", &["m"])]); } + + /// Live end-to-end check against the real on-device model. Kept `#[ignore]`d + /// so it never runs in CI (it requires Apple Intelligence enabled + the + /// on-device model READY). Run manually with: + /// cargo test -p tokscale-cli --features apple-fm -- --ignored live_summarize_smoke + /// Documents the live path; use THIS, not the segfaulting fm-c-example + /// streaming binary, as the on-device smoke test on macOS 26.x. + #[cfg(all(target_os = "macos", feature = "apple-fm"))] + #[test] + #[ignore] + fn live_summarize_smoke() { + let sessions = vec![ + SessionInput { + session_id: "ses_live_1".to_string(), + client: "claude".to_string(), + workspace: "/Users/x/payments-api".to_string(), + first_user_message: Some( + "Add JWT auth middleware to the payments API and write tests.".to_string(), + ), + models_used: vec!["claude-opus-4".to_string()], + total_tokens: 120_000, + duration_minutes: 45, + message_count: 12, + }, + SessionInput { + session_id: "ses_live_2".to_string(), + client: "claude".to_string(), + workspace: "/Users/x/dashboard".to_string(), + first_user_message: Some( + "The settings page crashes with a null pointer when the avatar URL is empty; \ + find and fix the bug." + .to_string(), + ), + models_used: vec!["claude-haiku-4".to_string()], + total_tokens: 8_000, + duration_minutes: 10, + message_count: 4, + }, + ]; + + let out = summarize(&sessions).expect("FM should be available on this box"); + assert_eq!(out.len(), 2); + for s in &out { + eprintln!( + "live[{}]: title={:?} category={:?} complexity={:?} fm_version={:?}\n desc={:?}", + s.session_id, s.title, s.task_category, s.complexity, s.fm_version, s.description + ); + } + // After a working schema/generation, every summary must be FM-produced, + // not the heuristic backfill. + for s in &out { + assert_eq!( + s.fm_version.as_deref(), + Some("apple-fm-on-device"), + "expected FM-generated provenance, got heuristic fallback for {}", + s.session_id + ); + assert!(VALID_CATEGORIES.contains(&s.task_category.as_str())); + assert!(VALID_COMPLEXITIES.contains(&s.complexity.as_str())); + } + } } diff --git a/crates/tokscale-cli/src/commands/report.rs b/crates/tokscale-cli/src/commands/report.rs index cd4dcb3d5..dafecec6a 100644 --- a/crates/tokscale-cli/src/commands/report.rs +++ b/crates/tokscale-cli/src/commands/report.rs @@ -219,12 +219,25 @@ fn run_summarizer(db: &WikiDb, session_ids: &[String], backend: &str) -> Result< backend.cyan() ); + // apple-fm runs each session as a self-contained on-device generation, so a + // single giant chunk would suppress the per-batch progress indicator below + // (it's gated on `batch_size < payloads.len()`). Use a modest batch size so + // the "\r Batch i/total" line fires and 100+ sequential on-device calls + // show visible progress instead of hanging silent until the very end. + // Re-fetching the model + rebuilding the schema per small batch is cheap; + // the generation dominates. let batch_size = match backend { - "apple-fm" => payloads.len(), + "apple-fm" => 8, _ => 20, }; let mut total_summarized = 0; + // Count how many summaries actually came from Apple FM vs the heuristic + // fallback, so a silent total-fallback (e.g. FM unavailable, or every + // generation erroring) is visible rather than reported as plain "N + // summarized". Only meaningful for the apple-fm backend; CLI backends leave + // fm_version null by design. + let mut fm_generated = 0; for (batch_idx, chunk) in payloads.chunks(batch_size).enumerate() { if batch_size < payloads.len() { eprint!( @@ -253,6 +266,9 @@ fn run_summarizer(db: &WikiDb, session_ids: &[String], backend: &str) -> Result< let description = result["description"].as_str().unwrap_or(""); let complexity = result["complexity"].as_str().unwrap_or("moderate"); let fm_version = result["fm_version"].as_str(); + if fm_version == Some("apple-fm-on-device") { + fm_generated += 1; + } db.update_summary( session_id, @@ -268,11 +284,22 @@ fn run_summarizer(db: &WikiDb, session_ids: &[String], backend: &str) -> Result< total_summarized += results.len(); } - eprintln!( - "\n {} {} sessions summarized", - "✓".green(), - total_summarized - ); + if backend == "apple-fm" { + let heuristic = total_summarized.saturating_sub(fm_generated); + eprintln!( + "\n {} {} sessions summarized ({} via Apple FM, {} heuristic)", + "✓".green(), + total_summarized, + fm_generated, + heuristic + ); + } else { + eprintln!( + "\n {} {} sessions summarized", + "✓".green(), + total_summarized + ); + } Ok(()) } @@ -297,6 +324,33 @@ fn run_task_grouping(db: &WikiDb, entries: &[WikiEntry], backend: &str) -> Resul return Ok(()); } + // Non-CLI backends (apple-fm and any future on-device backend) have no LLM + // grouping path. Rather than skip — which leaves every task_group null and + // makes the report collapse sessions by EXACT title — cluster titles + // deterministically in Rust. This merges near-duplicate titles ("Enhance API + // Security" / "Enhance API security with JWT auth middleware") into a single + // labeled group while keeping unrelated titles apart. + if !matches!(backend, "claude" | "codex" | "gemini" | "kiro") { + let assignments = cluster_titles(&summarized); + let group_count = assignments + .iter() + .map(|(_, label)| label.as_str()) + .collect::>() + .len(); + for (session_id, label) in &assignments { + db.update_task_group(session_id, label).map_err(|e| { + anyhow::anyhow!("Failed to save task_group for {}: {}", session_id, e) + })?; + } + eprintln!( + " {} grouped {} sessions into {} tasks", + "✓".green(), + summarized.len(), + group_count + ); + return Ok(()); + } + eprint!( " Grouping {} sessions into tasks...", summarized.len().to_string().cyan() @@ -341,12 +395,9 @@ fn run_task_grouping(db: &WikiDb, entries: &[WikiEntry], backend: &str) -> Resul .arg(format!("{}\n\n{}", GROUPING_SYSTEM_PROMPT, prompt)); c } - _ => { - eprintln!( - " skipped (task grouping requires a CLI backend: claude, codex, gemini, or kiro)" - ); - return Ok(()); - } + // Non-CLI backends were already handled by the title-clustering path + // above (which early-returns), so only the four CLI backends reach here. + other => unreachable!("non-CLI backend '{}' must be handled by clustering", other), }; // A timed-out (or otherwise un-spawnable) backend must degrade gracefully: @@ -393,6 +444,204 @@ fn run_task_grouping(db: &WikiDb, entries: &[WikiEntry], backend: &str) -> Resul Ok(()) } +/// Generic verbs and stopwords stripped from titles before clustering. These +/// carry no signal about *which* project/feature a session touched (every other +/// session "adds" or "fixes" something), so keeping them would make unrelated +/// titles look similar. +const CLUSTER_STOPWORDS: &[&str] = &[ + "add", + "fix", + "fixes", + "fixed", + "update", + "updates", + "refactor", + "improve", + "implement", + "enhance", + "create", + "remove", + "the", + "a", + "an", + "to", + "for", + "with", + "and", + "of", + "in", + "on", + "via", +]; + +/// Reduce a title to its set of SIGNIFICANT tokens: lowercase, strip +/// punctuation/ellipsis, collapse whitespace, drop generic verbs/stopwords. +/// The returned tokens are deduplicated and sorted so two titles with the same +/// significant words (in any order) produce equal sets. +fn significant_tokens(title: &str) -> Vec { + let mut tokens: Vec = title + .chars() + // Map punctuation/ellipsis to spaces; keep alphanumerics. This also + // strips a trailing "…" or "..." that the on-device model often emits. + .map(|c| { + if c.is_alphanumeric() { + c.to_ascii_lowercase() + } else { + ' ' + } + }) + .collect::() + .split_whitespace() + .map(|t| t.to_string()) + .filter(|t| !CLUSTER_STOPWORDS.contains(&t.as_str())) + .collect(); + tokens.sort(); + tokens.dedup(); + tokens +} + +/// Two token sets are considered the same task when they overlap strongly: +/// Jaccard similarity ≥ 0.6, OR they share at least two significant tokens. +/// The two-shared-tokens rule lets a long title ("Enhance API security with JWT +/// auth middleware") merge with a short one ("Enhance API Security") even though +/// the length gap drags Jaccard below 0.6. +fn tokens_overlap(a: &[String], b: &[String]) -> bool { + if a.is_empty() || b.is_empty() { + return false; + } + let shared = a.iter().filter(|t| b.contains(t)).count(); + if shared >= 2 { + return true; + } + let union = a.len() + b.len() - shared; + union > 0 && (shared as f64 / union as f64) >= 0.6 +} + +/// Deterministically cluster summarized entries by title similarity and return +/// `(session_id, group_label)` for every entry. Greedy O(n²) clustering — n is +/// small (one report's worth of sessions). Each cluster is labeled with its most +/// frequent original title, tie-broken by shortest, so the label is a real +/// human-readable title rather than a synthetic key. +fn cluster_titles(entries: &[&WikiEntry]) -> Vec<(String, String)> { + struct Cluster { + tokens: Vec, + members: Vec, + } + + // Precompute significant tokens once per entry. + let prepared: Vec<(usize, Vec)> = entries + .iter() + .enumerate() + .map(|(i, e)| (i, significant_tokens(e.title.as_deref().unwrap_or("")))) + .collect(); + + let mut clusters: Vec = Vec::new(); + for (idx, tokens) in &prepared { + // Find the first existing cluster this entry overlaps strongly with. + // An entry with no significant tokens (all stopwords) only matches a + // cluster that is itself token-empty, so generic titles still group. + let mut placed = false; + for cluster in clusters.iter_mut() { + let matches = if tokens.is_empty() { + cluster.tokens.is_empty() + } else { + tokens_overlap(tokens, &cluster.tokens) + }; + if matches { + cluster.members.push(*idx); + // Grow the cluster signature with this entry's tokens so later + // entries can match on the union of what's been seen. + for t in tokens { + if !cluster.tokens.contains(t) { + cluster.tokens.push(t.clone()); + } + } + cluster.tokens.sort(); + placed = true; + break; + } + } + if !placed { + clusters.push(Cluster { + tokens: tokens.clone(), + members: vec![*idx], + }); + } + } + + // Consolidation pass: the single greedy pass above is order-dependent — an + // entry compared before a cluster grew its signature can land in its own + // cluster even though it overlaps the grown signature. Repeatedly merge any + // two clusters whose signatures overlap until a fixpoint, making the final + // grouping independent of input order. n is small, so the O(n²)-per-round + // loop is cheap. + loop { + let mut merged_any = false; + 'outer: for i in 0..clusters.len() { + for j in (i + 1)..clusters.len() { + let overlap = if clusters[i].tokens.is_empty() || clusters[j].tokens.is_empty() { + // Token-empty clusters (all-stopword titles) only merge with + // each other, never with a tokened cluster. + clusters[i].tokens.is_empty() && clusters[j].tokens.is_empty() + } else { + tokens_overlap(&clusters[i].tokens, &clusters[j].tokens) + }; + if overlap { + let other = clusters.remove(j); + clusters[i].members.extend(other.members); + for t in other.tokens { + if !clusters[i].tokens.contains(&t) { + clusters[i].tokens.push(t); + } + } + clusters[i].tokens.sort(); + merged_any = true; + break 'outer; + } + } + } + if !merged_any { + break; + } + } + + let mut assignments = Vec::new(); + for cluster in &clusters { + let label = cluster_label(entries, &cluster.members); + for &idx in &cluster.members { + assignments.push((entries[idx].session_id.clone(), label.clone())); + } + } + assignments +} + +/// Pick a human-readable label for a cluster: the most frequent original title, +/// tie-broken by shortest (char count), then lexicographically for full +/// determinism. +fn cluster_label(entries: &[&WikiEntry], members: &[usize]) -> String { + let mut counts: HashMap<&str, usize> = HashMap::new(); + for &idx in members { + let title = entries[idx] + .title + .as_deref() + .map(str::trim) + .filter(|t| !t.is_empty()) + .unwrap_or("(unsummarized)"); + *counts.entry(title).or_insert(0) += 1; + } + counts + .into_iter() + .max_by(|(at, ac), (bt, bc)| { + ac.cmp(bc) + // Higher frequency wins; on a tie prefer the SHORTER title, then + // lexicographically smaller, so the label is stable run-to-run. + .then_with(|| bt.chars().count().cmp(&at.chars().count())) + .then_with(|| bt.cmp(at)) + }) + .map(|(title, _)| title.to_string()) + .unwrap_or_else(|| "(unsummarized)".to_string()) +} + fn run_apple_fm_summarizer(payloads: &[serde_json::Value]) -> Result> { // Build typed inputs from the JSON payloads. let inputs: Vec = payloads @@ -1032,4 +1281,125 @@ mod tests { let msg = parsed_message("claude-haiku-4"); assert_eq!(compute_msg_cost(&msg, None), 0.0); } + + fn titled_entry(session_id: &str, title: &str) -> WikiEntry { + WikiEntry { + session_id: session_id.to_string(), + client: "apple-fm".to_string(), + workspace: None, + workspace_label: None, + created_at: 0, + last_active: 0, + title: Some(title.to_string()), + task_category: None, + description: None, + complexity: None, + task_group: None, + total_input_tokens: 0, + total_output_tokens: 0, + total_cache_read: 0, + total_cost: 0.0, + models_used: Vec::new(), + message_count: 0, + duration_minutes: 0, + summarized_at: None, + fm_version: None, + } + } + + #[test] + fn significant_tokens_normalizes_and_strips_stopwords() { + // Lowercase, punctuation/ellipsis stripped, generic verbs dropped, + // result sorted + deduped. + assert_eq!( + significant_tokens("Add JWT auth middleware…"), + vec!["auth", "jwt", "middleware"] + ); + assert_eq!( + significant_tokens("Enhance API Security"), + vec!["api", "security"] + ); + // Trailing "..." and mixed case collapse to the same key. + assert_eq!( + significant_tokens("Fix the API Security..."), + vec!["api", "security"] + ); + } + + #[test] + fn cluster_titles_merges_near_duplicates() { + let entries = vec![ + titled_entry("a", "Enhance API Security"), + titled_entry("b", "Enhance API security with JWT auth middleware"), + titled_entry("c", "Add JWT auth middleware"), + ]; + let refs: Vec<&WikiEntry> = entries.iter().collect(); + let assignments = cluster_titles(&refs); + + let label_of = |sid: &str| { + assignments + .iter() + .find(|(s, _)| s == sid) + .map(|(_, l)| l.clone()) + .unwrap() + }; + + // a & b share "api" + "security" → same cluster. + assert_eq!(label_of("a"), label_of("b")); + // b & c share "auth" + "jwt" + "middleware" → all three collapse via b. + assert_eq!(label_of("b"), label_of("c")); + + let distinct: std::collections::HashSet<_> = + assignments.iter().map(|(_, l)| l.clone()).collect(); + assert_eq!(distinct.len(), 1, "all three should merge into one task"); + } + + #[test] + fn cluster_titles_keeps_unrelated_apart() { + let entries = vec![ + titled_entry("a", "Add JWT auth middleware"), + titled_entry("b", "Update database migration scripts"), + titled_entry("c", "Refactor pricing service cache"), + ]; + let refs: Vec<&WikiEntry> = entries.iter().collect(); + let assignments = cluster_titles(&refs); + + let distinct: std::collections::HashSet<_> = + assignments.iter().map(|(_, l)| l.clone()).collect(); + assert_eq!( + distinct.len(), + 3, + "unrelated titles must stay in separate groups" + ); + } + + #[test] + fn cluster_label_prefers_most_frequent_then_shortest() { + // Two identical long titles + one shorter variant: frequency wins, so the + // repeated long title is the label even though a shorter one exists. + let entries = vec![ + titled_entry("a", "Add JWT auth middleware"), + titled_entry("b", "Add JWT auth middleware"), + titled_entry("c", "JWT auth"), + ]; + let refs: Vec<&WikiEntry> = entries.iter().collect(); + let assignments = cluster_titles(&refs); + let label = &assignments[0].1; + assert_eq!(label, "Add JWT auth middleware"); + } + + #[test] + fn cluster_titles_groups_exact_duplicates() { + // The degenerate case from the report: many identical titles must + // collapse into exactly one group. + let entries: Vec = (0..51) + .map(|i| titled_entry(&format!("s{i}"), "Add JWT auth middleware")) + .collect(); + let refs: Vec<&WikiEntry> = entries.iter().collect(); + let assignments = cluster_titles(&refs); + let distinct: std::collections::HashSet<_> = + assignments.iter().map(|(_, l)| l.clone()).collect(); + assert_eq!(distinct.len(), 1); + assert_eq!(assignments.len(), 51); + } }