feat(radix-tree): chain-native prefix-membership index

.github/workflows/radix-tree-fuzz.yml

@@ -0,0 +1,44 @@
+name: radix-tree fuzz campaign
+
+# The always-on `fuzz_quick` in the unit-test lane is a triage gate (8+3
+# seeds, debug build). The differential campaign — seeded runs against
+# the reference model, every 4th at the wide H<=64 sharing width, plus
+# the out-of-contract chaos runs — is release-only and runs here nightly
+# so the data structure's correctness gate does not depend on someone
+# remembering to run it by hand. Budget: 1000 seeds took 52 min in
+# release on an M-series laptop; 600 fits a 2x slower hosted runner
+# inside the 120-minute job limit with headroom. Dispatch for more.
+
+on:
+  schedule:
+    - cron: "17 6 * * *"
+  workflow_dispatch:
+    inputs:
+      seeds:
+        description: "RADIX_FUZZ_SEEDS"
+        default: "600"
+      start:
+        description: "RADIX_FUZZ_START"
+        default: "1000"
+
+concurrency:
+  group: radix-tree-fuzz
+  cancel-in-progress: true
+
+jobs:
+  campaign:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: radix-tree-fuzz
+      - name: Run the differential campaign (release)
+        env:
+          RADIX_FUZZ_SEEDS: ${{ github.event.inputs.seeds || '600' }}
+          RADIX_FUZZ_START: ${{ github.event.inputs.start || '1000' }}
+        run: |
+          cargo test -p smg-radix-tree --release --test fuzz_differential \
+            -- --ignored --nocapture fuzz_campaign

.github/workflows/release-crates.yml

@@ -41,6 +41,8 @@ jobs:
             path: crates/mm_rdma
           - crate: engine-zmq-client
             path: crates/engine_zmq_client
+          - crate: smg-radix-tree
+            path: crates/radix_tree
     steps:
       - uses: actions/checkout@v7
       - uses: ./.github/actions/publish-crate

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 members = ["model_gateway", "crates/protocols", "crates/reasoning_parser", "crates/tool_parser", "crates/workflow", "crates/tokenizer", "crates/auth", "crates/mcp",
-    "crates/external_router", "crates/kv_index", "crates/data_connector", "crates/multimodal", "crates/mm_rdma", "crates/wasm", "crates/mesh", "crates/grpc_client", "crates/engine_zmq_client", "bindings/python", "bindings/golang", "clients/rust", "clients/openapi-gen", "crates/mock_worker", "crates/rl"]
+    "crates/external_router", "crates/kv_index", "crates/data_connector", "crates/multimodal", "crates/mm_rdma", "crates/wasm", "crates/mesh", "crates/grpc_client", "crates/engine_zmq_client", "bindings/python", "bindings/golang", "clients/rust", "clients/openapi-gen", "crates/mock_worker", "crates/rl", "crates/radix_tree"]
 resolver = "2"
 
 [workspace.dependencies]
@@ -15,6 +15,7 @@ smg-auth = { version = "1.2.3", path = "crates/auth" }
 smg-mcp = { version = "2.3.3", path = "crates/mcp" }
 smg-external-router = { version = "0.1.0", path = "crates/external_router" }
 kv-index = { version = "1.4.0", path = "crates/kv_index" }
+smg-radix-tree = { version = "0.1.0", path = "crates/radix_tree" }
 smg-data-connector = { version = "2.3.4", path = "crates/data_connector", package = "data-connector" }
 llm-multimodal = { version = "1.11.0", path = "crates/multimodal" }
 smg-wasm = { version = "1.1.4", path = "crates/wasm", package = "smg-wasm" }

crates/kv_index/src/event_tree.rs

@@ -165,6 +165,34 @@ pub fn compute_content_hash(token_ids: &[u32]) -> ContentHash {
     ContentHash(hasher.finish())
 }
 
+/// Rolling prefix hash over content hashes: `XXH3(prev || current)`, the
+/// same chaining `PositionalIndexer` computes internally. Exported so
+/// out-of-process publishers (the radix index service's placement feed)
+/// can synthesize byte-identical position chains for identical prefixes.
+///
+/// Note the base case: position 0's `SequenceHash` is the bare
+/// `ContentHash` value (`SequenceHash(c0.0)`), NOT
+/// `chain_prefix_hash(SequenceHash(0), c0)`. Callers must seed with the
+/// first content hash and chain from position 1, or the whole chain
+/// silently diverges from the indexer's (zero prefix matches, no error):
+///
+/// ```
+/// use kv_index::{chain_prefix_hash, ContentHash, SequenceHash};
+/// let contents = [ContentHash(11), ContentHash(22), ContentHash(33)];
+/// let mut chain = vec![SequenceHash(contents[0].0)];
+/// for &c in &contents[1..] {
+///     let prev = *chain.last().unwrap();
+///     chain.push(chain_prefix_hash(prev, c));
+/// }
+/// assert_eq!(chain.len(), 3);
+/// ```
+pub fn chain_prefix_hash(prev: SequenceHash, current: ContentHash) -> SequenceHash {
+    let mut bytes = [0u8; 16];
+    bytes[..8].copy_from_slice(&prev.0.to_le_bytes());
+    bytes[8..].copy_from_slice(&current.0.to_le_bytes());
+    SequenceHash(xxhash_rust::xxh3::xxh3_64_with_seed(&bytes, XXH3_SEED))
+}
+
 /// Chunk request tokens by block size and compute a [`ContentHash`] per full block.
 ///
 /// This is the entry point for the **query path**: given a request's token IDs and

crates/kv_index/src/lib.rs

@@ -20,9 +20,9 @@ mod token_tree;
 
 pub use common::{MatchResult, TenantId};
 pub use event_tree::{
-    compute_content_hash, compute_request_content_hashes, ApplyError, ContentHash, OverlapScores,
-    PositionalIndexer, PruneStats, SequenceHash, StoredBlock, WorkerBlockMap, WorkerId,
-    WorkerIdExhausted,
+    chain_prefix_hash, compute_content_hash, compute_request_content_hashes, ApplyError,
+    ContentHash, OverlapScores, PositionalIndexer, PruneStats, SequenceHash, StoredBlock,
+    WorkerBlockMap, WorkerId, WorkerIdExhausted, XXH3_SEED,
 };
 pub use path_hash::{hash_node_path, hash_token_path, GLOBAL_EVICTION_HASH};
 // Re-export under names matching old tree.rs API for easier migration

crates/radix_tree/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "smg-radix-tree"
+version = "0.1.0"
+edition = "2021"
+description = "Generic prefix-membership index: chain-native radix tree answering which holders share the longest prefix of a block chain, and how deep"
+license = "Apache-2.0"
+repository = "https://github.com/smg-project/smg"
+readme = "README.md"
+authors = ["Simo Lin <linsimo.mark@gmail.com>"]
+keywords = ["radix-tree", "prefix-matching", "cache-routing", "kv-cache", "llm"]
+categories = ["data-structures", "caching"]
+
+# The Rust import path stays `radix_tree`; only the crates.io package name
+# carries the `smg-` prefix (the bare `radix-tree` name is taken).
+[lib]
+name = "radix_tree"
+path = "src/lib.rs"
+
+[dependencies]
+# Zero SMG dependencies. External utility crates only.
+rustc-hash = "2"
+
+
+[dev-dependencies]
+# The differential oracle. Dev-only AND path-only (no version): cargo
+# drops a version-less path dev-dependency from the published manifest,
+# so the crate on crates.io carries no SMG dependency at all and can be
+# published in tier 1 without waiting on a kv-index release (a
+# versioned dev-dep would have to exist on the registry first — a race
+# whenever both crates bump in one release).
+kv-index = { path = "../kv_index" }

crates/radix_tree/README.md

@@ -0,0 +1,98 @@
+# radix-tree
+
+Published on crates.io as **`smg-radix-tree`** (the bare `radix-tree` name
+belongs to an unrelated router crate); the Rust import path is `radix_tree`.
+No SMG dependencies — it is a tier-1 crate in the release workflow.
+
+A generic prefix-membership index: given per-holder chains of
+content-addressed blocks, answer *"which holders already hold the
+longest prefix of this chain, and how deep?"* — plus the write and
+lifecycle operations a long-lived, multi-tenant index service needs
+as first-class API. Zero SMG dependencies; everything it sees is a
+hash.
+
+Built as the ground-up replacement for the gateway's
+`kv_index::PositionalIndexer` as a fleet-wide index core; the shared
+radix-index service (`smg-radix-index`, a separate crate) is built on
+it.
+
+## API in one glance
+
+```rust
+let mut tree = RadixTree::new(Config::default());
+let w = tree.create_holder("worker-7");            // generational id
+tree.store(w, None, &[(key, content), ...])?;      // anchor a chain
+tree.store(w, Some(parent_key), &more)?;           // extend it
+tree.remove(w, &[key]);                            // event-feed eviction
+tree.truncate_tail(w, keep);                       // prefix-closed capacity cut
+tree.clear(w);                                     // epoch bump
+tree.retire_holder(w);                             // frees everything, id recycled
+
+let mut scratch = OverlapScratch::default();
+let mut out = Vec::new();
+tree.overlap(&chain_hashes, &mut scratch, &mut out);
+// out: [{ holder, depth, total_blocks }]
+tree.enumerate(w);                                 // (pos, key, content) for snapshots
+```
+
+The contract — exact matching semantics, convergence scope, and the
+§4 alias rules — is what `tests/differential.rs` enforces: every core
+must equal the reference model on every run.
+
+## Structure
+
+Prefixes form a trie of **chains**. A chain's contents are one
+contiguous array, stored once no matter how many holders cover it;
+membership is a short list of maximal runs pointing at interned
+(hash-consed) holder sets; a query is one hash probe to the root
+chain, a linear scan of contiguous contents to the divergence point,
+a child-fork hop if needed, and a handful of span reads. Matching is
+exact and content-verified — fingerprint collisions cannot
+cross-credit.
+
+`FlatTree` is the first-generation flat layout, kept as a second,
+independently verified implementation: the test harness asserts BOTH
+cores equal the reference model on every run.
+
+## Verification
+
+The crate was built harness-first:
+
+- `tests/differential.rs` — both cores vs a representationally
+  complete reference model AND the production `kv_index` oracle
+  (dev-dependency), with full-state `audit()` at every checkpoint.
+- `tests/fuzz_differential.rs` — wide-config fuzz plus a CHAOS mode
+  that violates every contract precondition under a no-panic,
+  audit-green, model-equal, deterministic-replay contract.
+  `RADIX_FUZZ_SEEDS=10000` ran green.
+- `tests/api.rs` — lifecycle, boundary, and exactness cases the model
+  deliberately doesn't express.
+- `tests/alloc_gate.rs` — counting-allocator gate: single-holder
+  stores amortize to zero allocations.
+- `tests/pinned_bench.rs` — the normative performance workload
+  (`RADIX_BENCH_SIDE=oracle|r1|r3`, `RADIX_BENCH_SCALE=large`,
+  `RADIX_BENCH_SOAK_SECS=n`; `RADIX_BENCH_PROFILE=agentic|churn|fleet|
+  fragmented` are diagnostics, not gates).
+
+## Measured (pinned workload: 12.8M holder-blocks, 256 holders)
+
+| | old `PositionalIndexer` + glue | `RadixTree` |
+|---|---|---|
+| bytes / holder-block | 166.7 | **26.9** |
+| worst cell (d78, 64 holders) cold p99 | 6.0 µs (unsound skip) | **7.6 µs exact** |
+| single-holder query p50 | 917 ns | **292 ns** |
+| writes, mixed stream | 5.5M blocks/s | 4.6M blocks/s |
+| at 128M blocks: worst-cell p99 | 29 µs | **8.0 µs** |
+
+Known sensitivity: interior holes fragment a holder's chain into
+segments, and the query walk pays per segment where the positional
+map does not. On the `fragmented` profile (the pinned shape with 2%
+of every instance's interior blocks evicted at random, ~10 holes in a
+512-block prefix) the worst cell's p99 is 18.5 µs against the
+incumbent's 8.3 µs, at the same 27.5 B/block. Production eviction is
+tail-heavy (the pinned shape); a workload with dense interior holes
+should be measured on this profile before the tree is adopted for it.
+
+No timestamps live in the tree: freshness policy (idle TTL per
+holder) belongs to the consumer; chain data is freed by reference
+counting.