diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79b5a6e..582c506 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,26 @@ concurrency: cancel-in-progress: true jobs: + rust-core: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Install pinned Rust toolchain + run: >- + rustup toolchain install 1.97.1 --profile minimal + --component clippy --component rustfmt --component llvm-tools-preview + - name: Install pinned Rust coverage tool + run: cargo +1.97.1 install cargo-llvm-cov --locked --version 0.8.6 + - run: cargo +1.97.1 fmt --all -- --check + - run: cargo +1.97.1 clippy --workspace --all-targets -- -D warnings + - run: >- + cargo +1.97.1 llvm-cov --package rankweave-core + --fail-under-lines 100 --fail-under-functions 100 + --fail-under-regions 100 + test: runs-on: ubuntu-latest timeout-minutes: 10 @@ -27,6 +47,8 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} + - name: Install pinned Rust toolchain + run: rustup toolchain install 1.97.1 --profile minimal - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: version: "0.11.29" @@ -59,6 +81,8 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" + - name: Install pinned Rust toolchain + run: rustup toolchain install 1.97.1 --profile minimal - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: version: "0.11.29" diff --git a/.github/workflows/hourly-commercialization-loop.yml b/.github/workflows/hourly-commercialization-loop.yml index 7e85598..5f4addc 100644 --- a/.github/workflows/hourly-commercialization-loop.yml +++ b/.github/workflows/hourly-commercialization-loop.yml @@ -141,9 +141,10 @@ jobs: set -euo pipefail venv="/tmp/rankweave-automation-venv-${GITHUB_RUN_ID}" sudo rm -rf "$venv" + rustup toolchain install 1.97.1 --profile minimal python -m venv "$venv" "$venv/bin/python" -m pip install --upgrade pip - "$venv/bin/python" -m pip install -e ".[dev]" hatchling + "$venv/bin/python" -m pip install -e ".[dev]" sudo chown -R root:root "$venv" sudo chmod -R a-w "$venv" sandbox_uid="$(id -u nobody)" @@ -653,7 +654,7 @@ jobs: ) metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) - if metadata["build-system"]["build-backend"] != "hatchling.build": + if metadata["build-system"]["build-backend"] != "maturin": raise SystemExit("build backend may not change") if metadata["project"].get("dependencies") != []: raise SystemExit("RankWeave runtime dependencies must remain empty") diff --git a/.gitignore b/.gitignore index 53d91cd..36b82ca 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ dist/ venv/ .coverage htmlcov/ +.codegraph/ +target/ +src/rankweave/*.so diff --git a/AGENTS.md b/AGENTS.md index 103242e..869af1c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,8 @@ Operating guide for automated agents working in this repo. ## What this is -`rankweave` is a **pure-Python, stdlib-only** library and command-line tool for +`rankweave` is a Python library and command-line tool backed by one Rust +calculation core for language-agnostic hybrid-retrieval fusion, effectiveness evaluation, paired and family-wise statistical comparison, offline policy tuning, strict TREC interchange, and direct TREC benchmark comparison. It was extracted from @@ -14,9 +15,10 @@ Context Search under the lab's ONE SOURCE MULTI USE convention ## Hard rules -- **No dependencies.** The runtime imports only the Python standard library. - Do not add a runtime dependency; if you think you need one, the feature - probably belongs in the consumer, not here. +- **No third-party Python runtime dependencies.** Python adapters import only + the standard library and the packaged `rankweave._rankweave_core` extension. + Calculation belongs in `rankweave-core`; do not add a Python fallback or a + second arithmetic implementation. - **Store-agnostic.** RankWeave never talks to a database, embedding provider, search index, or benchmark download service. It fuses scores, evaluates and compares rankings, selects offline policies, parses interchange artifacts, diff --git a/CHANGELOG.md b/CHANGELOG.md index 58ef277..b7ec740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to rankweave are documented here. The format follows [Keep a ## [Unreleased] ### Added +- Introduced the first `rankweave-core` Rust vertical slice and thin PyO3 + adapter for theoretical min-max normalization and unweighted Reciprocal Rank + Fusion, with exact public-Python parity and complete core coverage. - ADR 0005 defines the versioned public-API compatibility policy: names in `rankweave.__all__` frozen as of `0.18.0` are not removed or renamed within a minor version, enforced by `tests/test_public_api_compatibility.py`. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..120aa07 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,137 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rankweave-core" +version = "0.18.0" + +[[package]] +name = "rankweave-python" +version = "0.18.0" +dependencies = [ + "pyo3", + "rankweave-core", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..eea6076 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +members = ["crates/rankweave-core", "crates/rankweave-python"] +resolver = "2" + +[workspace.package] +edition = "2024" +license = "Apache-2.0" +rust-version = "1.97.1" +version = "0.18.0" diff --git a/crates/rankweave-core/Cargo.toml b/crates/rankweave-core/Cargo.toml new file mode 100644 index 0000000..e1f9bcb --- /dev/null +++ b/crates/rankweave-core/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "rankweave-core" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +path = "src/lib.rs" diff --git a/crates/rankweave-core/src/lib.rs b/crates/rankweave-core/src/lib.rs new file mode 100644 index 0000000..29db3b3 --- /dev/null +++ b/crates/rankweave-core/src/lib.rs @@ -0,0 +1,34 @@ +//! Deterministic calculation primitives for RankWeave. + +/// Scale a finite score with finite theoretical bounds and clamp it to `[0, 1]`. +#[must_use] +pub fn theoretical_min_max_normalize(score: f64, lower: f64, upper: f64) -> f64 { + ((score - lower) / (upper - lower)).clamp(0.0, 1.0) +} + +/// Sum Reciprocal Rank Fusion contributions in caller-provided channel order. +#[must_use] +pub fn reciprocal_rank_fusion_score(ranks: &[u64], rank_constant_eta: u64) -> f64 { + ranks.iter().fold(0.0, |score, rank| { + score + 1.0 / (rank_constant_eta + rank) as f64 + }) +} + +#[cfg(test)] +mod tests { + use super::{reciprocal_rank_fusion_score, theoretical_min_max_normalize}; + + #[test] + fn normalization_uses_theoretical_bounds_and_clamps() { + assert_eq!(theoretical_min_max_normalize(0.5, 0.0, 2.0), 0.25); + assert_eq!(theoretical_min_max_normalize(3.0, 0.0, 2.0), 1.0); + } + + #[test] + fn rrf_preserves_input_order_for_the_reduction() { + assert_eq!( + reciprocal_rank_fusion_score(&[1, 3], 60), + 1.0 / 61.0 + 1.0 / 63.0 + ); + } +} diff --git a/crates/rankweave-python/Cargo.toml b/crates/rankweave-python/Cargo.toml new file mode 100644 index 0000000..c4d5da4 --- /dev/null +++ b/crates/rankweave-python/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rankweave-python" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +name = "_rankweave_core" +crate-type = ["cdylib"] +path = "src/lib.rs" + +[dependencies] +pyo3 = { version = "0.29.2", features = ["extension-module"] } +rankweave-core = { path = "../rankweave-core" } diff --git a/crates/rankweave-python/src/lib.rs b/crates/rankweave-python/src/lib.rs new file mode 100644 index 0000000..614ef1c --- /dev/null +++ b/crates/rankweave-python/src/lib.rs @@ -0,0 +1,20 @@ +//! Python bindings for the RankWeave calculation core. + +use pyo3::prelude::*; + +#[pyfunction] +fn theoretical_min_max_normalize(score: f64, lower: f64, upper: f64) -> f64 { + rankweave_core::theoretical_min_max_normalize(score, lower, upper) +} + +#[pyfunction] +fn reciprocal_rank_fusion_score(ranks: Vec, rank_constant_eta: u64) -> f64 { + rankweave_core::reciprocal_rank_fusion_score(&ranks, rank_constant_eta) +} + +#[pymodule] +fn _rankweave_core(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(theoretical_min_max_normalize, module)?)?; + module.add_function(wrap_pyfunction!(reciprocal_rank_fusion_score, module)?)?; + Ok(()) +} diff --git a/docs/product-requirements.md b/docs/product-requirements.md index cd88ea0..83c1b09 100644 --- a/docs/product-requirements.md +++ b/docs/product-requirements.md @@ -130,10 +130,11 @@ flowchart LR reimplement those models. - contextual-orchestrator owns LLM and model-routing decisions. -The current released implementation is dependency-free Python. Issue #45 -tracks migration of production fusion/evaluation arithmetic to a Rust core -behind the existing public contract under ADR 0006. Until that work is released and pinned, -consumers must not claim a Rust production engine. +The current release remains dependency-free Python. This development head +moves theoretical min-max normalization and unweighted RRF into the Rust core; +issue #45 tracks the remaining fusion and evaluation migration under ADR 0006. +Until a Rust-backed release is published and pinned, consumers must not claim +the engine is available to them. ## 7. Explicit non-goals diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e87d832..1d99613 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -14,8 +14,8 @@ list` reality. ## 1. Product identity and responsibility boundary -RankWeave is a **leaf product**: a dependency-free, pure-Python, -standard-library-only library and CLI for hybrid-retrieval score fusion, +RankWeave is a **leaf product**: a Python library and CLI with no third-party +Python runtime dependencies and one packaged Rust calculation core for hybrid-retrieval score fusion, ranking evaluation, paired/family statistical comparison, offline policy tuning (including caller-owned blocked-fold cross-validation and availability-time backtesting), and strict TREC interchange @@ -181,10 +181,13 @@ Severity: ๐Ÿ”ด blocks a user today ยท ๐ŸŸก user-visible friction ยท ๐ŸŸข hardeni file a follow-up issue tracking the re-pin once the central scheduler's current commit is confirmed stable, rather than merging it unreviewed inside an unrelated PR. -5. ๐Ÿ”ด **Production fusion arithmetic is still Python-only** (issue #45). +5. ๐ŸŸก **The first production fusion primitives are Rust-backed; migration is + incomplete** (issue #45). LineageWeave ADR 0225 names RankWeave as the sole fusion owner, while the - current package implements the arithmetic in Python. The required migration - keeps the public Python surface as an adapter over one Rust core, preserves + development head now routes theoretical normalization and unweighted RRF + through `rankweave-core`; convex, weighted, evaluation, comparison, and + tuning arithmetic still execute in Python. The remaining migration keeps + the public Python surface as an adapter over one Rust core, preserves exact documented semantics, and publishes provenance and limitations with every calculation envelope. CPU multithreading and an optional GPU path require exact-workload parity and benchmark evidence; no throughput claim diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..19305ad --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +rust = "1.97.1" diff --git a/pyproject.toml b/pyproject.toml index e4e6b2e..cfdefa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [build-system] -requires = ["hatchling==1.31.0"] -build-backend = "hatchling.build" +requires = ["maturin==1.14.1"] +build-backend = "maturin" [project] name = "rankweave" version = "0.18.0" -description = "Dependency-free retrieval fusion, evaluation, family-wise statistical comparison, policy tuning, TREC benchmarking, and auditable CLI workflows." +description = "Rust-backed retrieval fusion, evaluation, family-wise statistical comparison, policy tuning, TREC benchmarking, and auditable CLI workflows." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" @@ -41,7 +41,7 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Rust", "Topic :: Text Processing :: Indexing", "Topic :: Scientific/Engineering :: Information Analysis", "Typing :: Typed", @@ -68,8 +68,10 @@ dev = [ [tool.uv] required-version = "==0.11.29" -[tool.hatch.build.targets.wheel] -packages = ["src/rankweave"] +[tool.maturin] +manifest-path = "crates/rankweave-python/Cargo.toml" +module-name = "rankweave._rankweave_core" +python-source = "src" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/rankweave/score_fusion.py b/src/rankweave/score_fusion.py index d75e193..31038d4 100644 --- a/src/rankweave/score_fusion.py +++ b/src/rankweave/score_fusion.py @@ -40,6 +40,7 @@ from collections.abc import Mapping from dataclasses import dataclass +from rankweave import _rankweave_core from rankweave._validation import ( _require_finite, _require_positive_integer, @@ -113,8 +114,9 @@ def theoretical_min_max_normalize( raise ValueError("bounds must be finite") from exc if upper_bound <= lower_bound: raise ValueError("bounds must satisfy upper > lower") - normalized = (score - lower_bound) / (upper_bound - lower_bound) - return min(1.0, max(0.0, normalized)) + return _rankweave_core.theoretical_min_max_normalize( + score, lower_bound, upper_bound + ) def convex_combination_score( @@ -183,13 +185,16 @@ def reciprocal_rank_fusion_score( validated_eta = _require_positive_integer( rank_constant_eta, "rank_constant_eta" ) - fused_score = 0.0 + validated_ranks = [] for channel_name, one_based_rank in channel_ranks.items(): - validated_rank = _require_positive_integer( - one_based_rank, f"rank for channel {channel_name!r}" + validated_ranks.append( + _require_positive_integer( + one_based_rank, f"rank for channel {channel_name!r}" + ) ) - fused_score += 1.0 / (validated_eta + validated_rank) - return fused_score + return _rankweave_core.reciprocal_rank_fusion_score( + validated_ranks, validated_eta + ) def weighted_reciprocal_rank_fusion_score( diff --git a/tests/test_ci_supply_chain.py b/tests/test_ci_supply_chain.py index 516d16a..3ba0deb 100644 --- a/tests/test_ci_supply_chain.py +++ b/tests/test_ci_supply_chain.py @@ -28,7 +28,7 @@ def _references(workflow: str) -> tuple[tuple[str, str], ...]: def test_ci_pins_reviewed_node24_action_commits(): workflow = _workflow_text() - assert workflow.count(f"actions/checkout@{CHECKOUT_SHA}") == 2 + assert workflow.count(f"actions/checkout@{CHECKOUT_SHA}") == 3 assert workflow.count(f"actions/setup-python@{SETUP_PYTHON_SHA}") == 2 assert workflow.count(f"astral-sh/setup-uv@{SETUP_UV_SHA}") == 2