diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9af52bf..5738d68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,26 @@ concurrency: cancel-in-progress: true jobs: + rust-core: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Install pinned Rust toolchain + run: >- + rustup toolchain install 1.97.1 --profile minimal + --component clippy --component rustfmt --component llvm-tools-preview + - name: Install pinned Rust coverage tool + run: cargo +1.97.1 install cargo-llvm-cov --locked --version 0.8.6 + - run: cargo +1.97.1 fmt --all -- --check + - run: cargo +1.97.1 clippy --workspace --all-targets -- -D warnings + - run: >- + cargo +1.97.1 llvm-cov --package rankweave-core + --fail-under-lines 100 --fail-under-functions 100 + --fail-under-regions 100 + test: runs-on: ubuntu-latest timeout-minutes: 10 @@ -27,6 +47,10 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} + - name: Install pinned Rust toolchain + run: | + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: version: "0.12.1" @@ -59,90 +83,28 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" + - name: Install pinned Rust toolchain + run: | + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: version: "0.12.1" enable-cache: false - run: uv sync --frozen --extra dev --python 3.13 - run: uv build --wheel --sdist --out-dir dist - - name: Verify wheel contents - run: | - uv run --frozen --extra dev --python 3.13 python - <<'PY' - from pathlib import Path - from zipfile import ZipFile - - wheel_path = next(Path("dist").glob("rankweave-*.whl")) - with ZipFile(wheel_path) as wheel_file: - wheel_members = set(wheel_file.namelist()) - required_members = { - "rankweave/__init__.py", - "rankweave/__main__.py", - "rankweave/_validation.py", - "rankweave/artifact_verification.py", - "rankweave/cli.py", - "rankweave/comparison.py", - "rankweave/cross_validation.py", - "rankweave/evaluation.py", - "rankweave/query_normalization.py", - "rankweave/ranked_list_fusion.py", - "rankweave/report_schemas.py", - "rankweave/score_fusion.py", - "rankweave/schemas/__init__.py", - "rankweave/schemas/artifact-verification-v1.schema.json", - "rankweave/schemas/trec-comparison-v1.schema.json", - "rankweave/schemas/trec-comparison-v2.schema.json", - "rankweave/schemas/trec-family-comparison-v1.schema.json", - "rankweave/schemas/trec-family-comparison-v2.schema.json", - "rankweave/temporal_backtesting.py", - "rankweave/trec.py", - "rankweave/trec_comparison.py", - "rankweave/trec_family_comparison.py", - "rankweave/tuning.py", - "rankweave/py.typed", - } - missing_members = required_members - wheel_members - if missing_members: - raise SystemExit(f"wheel is missing: {sorted(missing_members)!r}") - PY - - name: Verify source distribution contents - run: | - uv run --frozen --extra dev --python 3.13 python - <<'PY' - from pathlib import Path - from tarfile import open as open_tarfile - - source_distributions = tuple( - Path("dist").glob("rankweave-*.tar.gz") - ) - if len(source_distributions) != 1: - raise SystemExit( - "package job requires exactly one source distribution" - ) - source_distribution = source_distributions[0] - source_root = source_distribution.name.removesuffix(".tar.gz") + "/" - with open_tarfile(source_distribution, "r:gz") as archive: - source_members = set(archive.getnames()) - required_source_members = { - source_root + "pyproject.toml", - source_root + "README.md", - source_root + "CHANGELOG.md", - source_root + "LICENSE", - source_root + "src/rankweave/__init__.py", - source_root + "tests/test_version.py", - } - missing_source_members = required_source_members - source_members - if missing_source_members: - raise SystemExit( - "source distribution is missing: " - f"{sorted(missing_source_members)!r}" - ) - PY + - name: Verify wheel and source distribution contents + run: >- + uv run --frozen --extra dev --python 3.13 python + scripts/verify_release_archives.py --dist-dir dist --version 0.18.0 + --wheel-tag linux --require-sdist - name: Exercise release checksum handoff run: | set -euo pipefail mkdir -p release-handoff ( cd dist - sha256sum *.whl *.tar.gz + sha256sum ./*.whl ./*.tar.gz ) > release-handoff/SHA256SUMS manifest_sha256="$( sha256sum release-handoff/SHA256SUMS | cut -d ' ' -f1 diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 456177f..41fa01e 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -37,6 +37,10 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" + - name: Install pinned Rust toolchain + run: | + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: version: "0.12.1" @@ -224,24 +228,12 @@ jobs: uv run --frozen --extra dev --python 3.13 \ python -m coverage report uv build --wheel --sdist --out-dir dist - - version="$RELEASE_VERSION" - wheel_paths=(dist/rankweave-*.whl) - source_paths=(dist/rankweave-*.tar.gz) - if [[ ${#wheel_paths[@]} -ne 1 || ${#source_paths[@]} -ne 1 ]]; then - echo "release requires exactly one wheel and one source distribution" >&2 - exit 1 - fi - expected_wheel="rankweave-${version}-py3-none-any.whl" - expected_source="rankweave-${version}.tar.gz" - if [[ "${wheel_paths[0]##*/}" != "$expected_wheel" ]]; then - echo "unexpected wheel name: ${wheel_paths[0]##*/}" >&2 - exit 1 - fi - if [[ "${source_paths[0]##*/}" != "$expected_source" ]]; then - echo "unexpected source distribution: ${source_paths[0]##*/}" >&2 - exit 1 - fi + uv run --frozen --extra dev --python 3.13 python \ + scripts/verify_release_archives.py \ + --dist-dir dist \ + --version "$RELEASE_VERSION" \ + --wheel-tag linux \ + --require-sdist - name: Extract deterministic release notes env: RELEASE_VERSION: ${{ steps.identity.outputs.release_version }} diff --git a/.github/workflows/hourly-commercialization-loop.yml b/.github/workflows/hourly-commercialization-loop.yml index ad4317b..663451d 100644 --- a/.github/workflows/hourly-commercialization-loop.yml +++ b/.github/workflows/hourly-commercialization-loop.yml @@ -129,18 +129,46 @@ jobs: set -euo pipefail venv="/tmp/rankweave-automation-venv-${GITHUB_RUN_ID}" sudo rm -rf "$venv" + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 + cargo +1.97.1 fetch --locked python -m venv "$venv" "$venv/bin/python" -m pip install --upgrade pip - "$venv/bin/python" -m pip install -e ".[dev]" hatchling + "$venv/bin/python" -m pip install -e ".[dev]" maturin==1.14.1 + native_cores=(src/rankweave/_rankweave_core*.so) + if [[ "${#native_cores[@]}" -ne 1 || ! -f "${native_cores[0]}" ]]; then + echo "trusted editable install did not produce exactly one native core" >&2 + exit 1 + fi + native_core_name="$(basename "${native_cores[0]}")" + trusted_native_core="${RUNNER_TEMP}/rankweave-trusted-native-core.so" + sudo install -o root -g root -m 0555 \ + "${native_cores[0]}" "$trusted_native_core" sudo chown -R root:root "$venv" sudo chmod -R a-w "$venv" sandbox_uid="$(id -u nobody)" sandbox_gid="$(id -g nobody)" + sandbox_rust="/tmp/rankweave-rust-${GITHUB_RUN_ID}" + cargo_home="${CARGO_HOME:-$HOME/.cargo}" + rust_sysroot="$(rustc +1.97.1 --print sysroot)" + sudo rm -rf "$sandbox_rust" + sudo mkdir -p "$sandbox_rust/cargo" "$sandbox_rust/toolchain" + sudo cp -a "$cargo_home/registry" "$sandbox_rust/cargo/registry" + sudo cp -a "$rust_sysroot/." "$sandbox_rust/toolchain/" + sudo chown -R "$sandbox_uid:$sandbox_gid" "$sandbox_rust/cargo" + sudo chown -R root:root "$sandbox_rust/toolchain" + sudo chmod -R a+rX,go-w "$sandbox_rust/toolchain" command -v setpriv >/dev/null - echo "AUTOMATION_VENV=$venv" >>"$GITHUB_ENV" - echo "AUTOMATION_BASE_SHA=$(git rev-parse HEAD)" >>"$GITHUB_ENV" - echo "SANDBOX_UID=$sandbox_uid" >>"$GITHUB_ENV" - echo "SANDBOX_GID=$sandbox_gid" >>"$GITHUB_ENV" + { + echo "AUTOMATION_VENV=$venv" + echo "AUTOMATION_NATIVE_CORE_NAME=$native_core_name" + echo "AUTOMATION_TRUSTED_NATIVE_CORE=$trusted_native_core" + echo "AUTOMATION_BASE_SHA=$(git rev-parse HEAD)" + echo "SANDBOX_UID=$sandbox_uid" + echo "SANDBOX_GID=$sandbox_gid" + echo "SANDBOX_CARGO_HOME=$sandbox_rust/cargo" + echo "SANDBOX_RUST_TOOLCHAIN=$sandbox_rust/toolchain" + } >>"$GITHUB_ENV" { echo "/opencode.json" echo "/.agent-red-output.txt" @@ -244,7 +272,8 @@ jobs: any file outside tests/ and docs/superpowers/specs/. The tests must express the buyer-visible contract, preserve RankWeave's - standard-library-only runtime, deterministic and immutable evidence, + no-third-party Python runtime dependency, one Rust calculation core, + deterministic and immutable evidence, fail-closed validation, Python 3.10+ compatibility, and modular standalone plus naruon-import use. The workflow will execute the test suite after this phase and requires a genuine test failure before implementation begins. @@ -279,6 +308,10 @@ jobs: set -euo pipefail rm -f opencode.json git clean -fdX + cp "$AUTOMATION_TRUSTED_NATIVE_CORE" \ + "src/rankweave/$AUTOMATION_NATIVE_CORE_NAME" + test "$(sha256sum "$AUTOMATION_TRUSTED_NATIVE_CORE" | cut -d " " -f1)" = \ + "$(sha256sum "src/rankweave/$AUTOMATION_NATIVE_CORE_NAME" | cut -d " " -f1)" "$AUTOMATION_VENV/bin/python" - <<'PY' from __future__ import annotations @@ -374,7 +407,7 @@ jobs: --noprofile --norc -c - 'cd "$WORKSPACE" && python -m pytest -q -p no:cacheprovider' + "cd \"$GITHUB_WORKSPACE\" && python -m pytest -q -p no:cacheprovider" ) set +e "${red_sandbox[@]}" "${red_environment[@]}" >"$red_output" 2>&1 @@ -462,7 +495,8 @@ jobs: environment variables, or paths outside the repository. Make the smallest coherent production change that turns the red tests - green. Preserve the standard-library-only runtime, store-agnostic modular + green. Preserve the no-third-party Python runtime dependency, one Rust + calculation core, store-agnostic modular architecture, deterministic behavior, immutable audit records, fail-closed input contracts, Python 3.10+ compatibility, full production docstrings, and standalone plus naruon-import use. Do not edit anything under .github/ @@ -474,6 +508,11 @@ jobs: statistical or standards claim without a primary source already recorded in the repository. Figma is not applicable because RankWeave has no UI. + This autonomous lane is Python-only. Do not edit Rust source, Cargo + manifests, build configuration, or Python build metadata. A Rust-core + increment requires a separately authored and reviewed maintainer pull + request. + Do not commit, push, open, approve, merge, publish, or release anything; the workflow performs deterministic validation and packages one protected pull request with maintainer-owned metadata. @@ -510,6 +549,10 @@ jobs: set -euo pipefail rm -f opencode.json .agent-red-output.txt git clean -fdX + cp "$AUTOMATION_TRUSTED_NATIVE_CORE" \ + "src/rankweave/$AUTOMATION_NATIVE_CORE_NAME" + test "$(sha256sum "$AUTOMATION_TRUSTED_NATIVE_CORE" | cut -d " " -f1)" = \ + "$(sha256sum "src/rankweave/$AUTOMATION_NATIVE_CORE_NAME" | cut -d " " -f1)" "$AUTOMATION_VENV/bin/python" - <<'PY' from __future__ import annotations @@ -568,20 +611,23 @@ jobs: ".gitmodules", "AGENTS.md", "CODEOWNERS", + "pyproject.toml", "SECURITY.md", } - forbidden_prefixes = (".github/", ".git/") + forbidden_prefixes = (".github/", ".git/", "crates/") allowed_exact = { "CHANGELOG.md", "README.md", - "pyproject.toml", } - allowed_prefixes = ("src/rankweave/", "tests/", "docs/") + allowed_prefixes = ( + "src/rankweave/", + "tests/", + "docs/", + ) allowed_suffixes = { ".json", ".md", ".py", - ".toml", ".txt", ".yaml", ".yml", @@ -644,7 +690,7 @@ jobs: ) metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) - if metadata["build-system"]["build-backend"] != "hatchling.build": + if metadata["build-system"]["build-backend"] != "maturin": raise SystemExit("build backend may not change") if metadata["project"].get("dependencies") != []: raise SystemExit("RankWeave runtime dependencies must remain empty") @@ -745,7 +791,7 @@ jobs: ) validation_environment=( env -i - "PATH=${AUTOMATION_VENV}/bin:/usr/bin:/bin" + "PATH=${AUTOMATION_VENV}/bin:${SANDBOX_RUST_TOOLCHAIN}/bin:/usr/bin:/bin" "HOME=$validation_home" "WORKSPACE=$GITHUB_WORKSPACE" "PYTHONPATH=$GITHUB_WORKSPACE/src" @@ -753,6 +799,11 @@ jobs: "SMOKE=$validation_smoke" "COVERAGE_FILE=$validation_coverage" "RUFF_CACHE_DIR=$ruff_cache" + "CARGO_HOME=$SANDBOX_CARGO_HOME" + "CARGO_TARGET_DIR=$validation_home/cargo-target" + "CARGO_NET_OFFLINE=true" + "RUSTC=$SANDBOX_RUST_TOOLCHAIN/bin/rustc" + "RUSTDOC=$SANDBOX_RUST_TOOLCHAIN/bin/rustdoc" PYTHONDONTWRITEBYTECODE=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_INDEX=1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a1e89bd..05c6497 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,16 +27,21 @@ jobs: permissions: contents: read outputs: - manifest_sha256: ${{ steps.distributions.outputs.manifest_sha256 }} + release_sha: ${{ steps.version.outputs.release_sha }} + release_version: ${{ steps.version.outputs.version }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.release_sha || github.sha }} + ref: ${{ github.event.repository.default_branch }} fetch-depth: 0 persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" + - name: Install pinned Rust toolchain + run: | + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: version: "0.12.1" @@ -80,15 +85,9 @@ jobs: exit 1 fi - checked_out_sha="$(git rev-parse HEAD)" - if [[ "$checked_out_sha" != "$release_sha" ]]; then - echo "checked-out commit ${checked_out_sha} does not match " \ - "released commit ${release_sha}" >&2 - exit 1 - fi - git fetch --force --tags origin "$DEFAULT_BRANCH" + git fetch --force --tags origin "$DEFAULT_BRANCH" "$release_sha" if ! git merge-base --is-ancestor \ - "$checked_out_sha" "origin/$DEFAULT_BRANCH"; then + "$release_sha" "origin/$DEFAULT_BRANCH"; then echo "released commit must be reachable from the default branch" >&2 exit 1 fi @@ -97,6 +96,13 @@ jobs: echo "release tag does not resolve to the released commit" >&2 exit 1 fi + git checkout --detach "$release_sha" + checked_out_sha="$(git rev-parse HEAD)" + if [[ "$checked_out_sha" != "$release_sha" ]]; then + echo "checked-out commit ${checked_out_sha} does not match " \ + "released commit ${release_sha}" >&2 + exit 1 + fi version="$(python - <<'PY' from pathlib import Path @@ -150,7 +156,10 @@ jobs: raise SystemExit("stable GitHub Release must not be a prerelease") PY - printf 'version=%s\n' "$version" >>"$GITHUB_OUTPUT" + { + printf 'release_sha=%s\n' "$release_sha" + printf 'version=%s\n' "$version" + } >>"$GITHUB_OUTPUT" - run: uv sync --frozen --extra dev --python 3.13 - name: Verify public version identity env: @@ -172,83 +181,147 @@ jobs: - run: uv run --frozen --extra dev --python 3.13 python -m ruff check . - run: uv run --frozen --extra dev --python 3.13 python -m coverage run -m pytest -q - run: uv run --frozen --extra dev --python 3.13 python -m coverage report - - run: uv build --wheel --sdist --out-dir dist - - name: Verify release archives + - run: uv build --sdist --out-dir dist + - name: Verify source distribution env: PACKAGE_VERSION: ${{ steps.version.outputs.version }} - run: | - uv run --frozen --extra dev --python 3.13 python - <<'PY' - from pathlib import Path - from tarfile import open as open_tarfile - from zipfile import ZipFile - import os - - version = os.environ["PACKAGE_VERSION"] - dist_path = Path("dist") - wheels = tuple(dist_path.glob("rankweave-*.whl")) - source_distributions = tuple( - dist_path.glob("rankweave-*.tar.gz") - ) - if len(wheels) != 1 or len(source_distributions) != 1: - raise SystemExit( - "release must contain exactly one wheel and one " - "source distribution" - ) - - expected_prefix = f"rankweave-{version}" - expected_wheel = f"{expected_prefix}-py3-none-any.whl" - expected_sdist = f"{expected_prefix}.tar.gz" - if wheels[0].name != expected_wheel: - raise SystemExit( - f"unexpected wheel name: {wheels[0].name!r}" - ) - if source_distributions[0].name != expected_sdist: - raise SystemExit( - "unexpected source distribution name: " - f"{source_distributions[0].name!r}" - ) + run: >- + uv run --frozen --extra dev --python 3.13 python + scripts/verify_release_archives.py --dist-dir dist + --version "$PACKAGE_VERSION" --require-sdist + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rankweave-build-sdist + path: dist/*.tar.gz + if-no-files-found: error + include-hidden-files: false + retention-days: 7 - with ZipFile(wheels[0]) as wheel_file: - wheel_members = set(wheel_file.namelist()) - required_wheel_members = { - "rankweave/__init__.py", - "rankweave/__main__.py", - "rankweave/artifact_verification.py", - "rankweave/cli.py", - "rankweave/cross_validation.py", - "rankweave/report_schemas.py", - "rankweave/schemas/artifact-verification-v1.schema.json", - "rankweave/schemas/trec-comparison-v1.schema.json", - "rankweave/schemas/trec-comparison-v2.schema.json", - "rankweave/schemas/trec-family-comparison-v1.schema.json", - "rankweave/schemas/trec-family-comparison-v2.schema.json", - "rankweave/temporal_backtesting.py", - "rankweave/py.typed", - } - missing_wheel_members = required_wheel_members - wheel_members - if missing_wheel_members: - raise SystemExit( - f"wheel is missing: {sorted(missing_wheel_members)!r}" - ) + wheels: + needs: build + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + artifact: linux + compatibility: manylinux2014 + wheel_tag: manylinux + - runner: macos-14 + artifact: macos + compatibility: "" + wheel_tag: macosx + - runner: windows-latest + artifact: windows + compatibility: "" + wheel_tag: win_amd64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + - name: Revalidate and checkout released commit + shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + RELEASE_SHA: ${{ needs.build.outputs.release_sha }} + run: | + git fetch --force origin "$DEFAULT_BRANCH" "$RELEASE_SHA" + git merge-base --is-ancestor "$RELEASE_SHA" "origin/$DEFAULT_BRANCH" + git checkout --detach "$RELEASE_SHA" + test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + - name: Install pinned Rust toolchain + shell: bash + run: | + rustup toolchain install 1.97.1 --profile minimal + rustup default 1.97.1 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.12.1" + enable-cache: false + - name: Build native wheel + shell: bash + env: + COMPATIBILITY: ${{ matrix.compatibility }} + run: | + build_args=(--release --locked --out dist --interpreter python) + if [[ -n "$COMPATIBILITY" ]]; then + build_args+=(--compatibility "$COMPATIBILITY") + fi + uvx --from maturin==1.14.1 maturin build "${build_args[@]}" + - name: Verify native wheel + shell: bash + env: + PACKAGE_VERSION: ${{ needs.build.outputs.release_version }} + WHEEL_TAG: ${{ matrix.wheel_tag }} + run: >- + python scripts/verify_release_archives.py --dist-dir dist + --version "$PACKAGE_VERSION" --wheel-tag "$WHEEL_TAG" + - name: Smoke-test built native wheel + shell: bash + run: | + python -m venv wheel-smoke + if [[ "$RUNNER_OS" == "Windows" ]]; then + smoke_python="wheel-smoke/Scripts/python.exe" + smoke_cli="wheel-smoke/Scripts/rankweave.exe" + else + smoke_python="wheel-smoke/bin/python" + smoke_cli="wheel-smoke/bin/rankweave" + fi + "$smoke_python" -m pip install --no-index --find-links dist rankweave + "$smoke_python" -c 'from rankweave import SemanticUnitExactIndex; assert SemanticUnitExactIndex' + "$smoke_cli" --help + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: rankweave-build-wheel-${{ matrix.artifact }} + path: dist/*.whl + if-no-files-found: error + include-hidden-files: false + retention-days: 7 - with open_tarfile(source_distributions[0], "r:gz") as archive: - source_members = set(archive.getnames()) - source_root = f"{expected_prefix}/" - required_source_members = { - source_root + "pyproject.toml", - source_root + "README.md", - source_root + "CHANGELOG.md", - source_root + "LICENSE", - source_root + "src/rankweave/__init__.py", - source_root + "tests/test_version.py", - } - missing_source_members = required_source_members - source_members - if missing_source_members: - raise SystemExit( - "source distribution is missing: " - f"{sorted(missing_source_members)!r}" - ) - PY + assemble: + needs: [build, wheels] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + manifest_sha256: ${{ steps.distributions.outputs.manifest_sha256 }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + - name: Revalidate and checkout released commit + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + RELEASE_SHA: ${{ needs.build.outputs.release_sha }} + run: | + git fetch --force origin "$DEFAULT_BRANCH" "$RELEASE_SHA" + git merge-base --is-ancestor "$RELEASE_SHA" "origin/$DEFAULT_BRANCH" + git checkout --detach "$RELEASE_SHA" + test "$(git rev-parse HEAD)" = "$RELEASE_SHA" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: rankweave-build-* + path: dist/ + merge-multiple: true + - name: Verify complete cross-platform release + env: + PACKAGE_VERSION: ${{ needs.build.outputs.release_version }} + run: >- + python scripts/verify_release_archives.py --dist-dir dist + --version "$PACKAGE_VERSION" --wheel-tag manylinux --wheel-tag macosx + --wheel-tag win_amd64 --require-sdist - name: Record immutable distribution checksums id: distributions run: | @@ -256,7 +329,7 @@ jobs: mkdir -p release-handoff ( cd dist - sha256sum *.whl *.tar.gz + sha256sum ./*.whl ./*.tar.gz ) > release-handoff/SHA256SUMS manifest_sha256="$( sha256sum release-handoff/SHA256SUMS | cut -d ' ' -f1 @@ -273,7 +346,7 @@ jobs: retention-days: 7 provenance: - needs: build + needs: assemble runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -288,7 +361,7 @@ jobs: path: handoff/ - name: Verify immutable distribution handoff env: - EXPECTED_MANIFEST_SHA256: ${{ needs.build.outputs.manifest_sha256 }} + EXPECTED_MANIFEST_SHA256: ${{ needs.assemble.outputs.manifest_sha256 }} run: | set -euo pipefail printf '%s %s\n' \ @@ -306,7 +379,7 @@ jobs: handoff/dist/*.tar.gz publish: - needs: [build, provenance] + needs: [assemble, provenance] runs-on: ubuntu-latest timeout-minutes: 10 environment: @@ -321,7 +394,7 @@ jobs: path: handoff/ - name: Verify immutable distribution handoff env: - EXPECTED_MANIFEST_SHA256: ${{ needs.build.outputs.manifest_sha256 }} + EXPECTED_MANIFEST_SHA256: ${{ needs.assemble.outputs.manifest_sha256 }} run: | set -euo pipefail printf '%s %s\n' \ diff --git a/.gitignore b/.gitignore index 53d91cd..36b82ca 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ dist/ venv/ .coverage htmlcov/ +.codegraph/ +target/ +src/rankweave/*.so diff --git a/AGENTS.md b/AGENTS.md index 103242e..869af1c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,8 @@ Operating guide for automated agents working in this repo. ## What this is -`rankweave` is a **pure-Python, stdlib-only** library and command-line tool for +`rankweave` is a Python library and command-line tool backed by one Rust +calculation core for language-agnostic hybrid-retrieval fusion, effectiveness evaluation, paired and family-wise statistical comparison, offline policy tuning, strict TREC interchange, and direct TREC benchmark comparison. It was extracted from @@ -14,9 +15,10 @@ Context Search under the lab's ONE SOURCE MULTI USE convention ## Hard rules -- **No dependencies.** The runtime imports only the Python standard library. - Do not add a runtime dependency; if you think you need one, the feature - probably belongs in the consumer, not here. +- **No third-party Python runtime dependencies.** Python adapters import only + the standard library and the packaged `rankweave._rankweave_core` extension. + Calculation belongs in `rankweave-core`; do not add a Python fallback or a + second arithmetic implementation. - **Store-agnostic.** RankWeave never talks to a database, embedding provider, search index, or benchmark download service. It fuses scores, evaluates and compares rankings, selects offline policies, parses interchange artifacts, diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 28cee97..f863863 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,7 +2,8 @@ ## Purpose -RankWeave is a dependency-free Python library and command-line component for +RankWeave is a Python library and command-line component backed by one Rust +calculation core and no third-party Python runtime dependency. It provides retrieval-score fusion, ranking evaluation, paired and candidate-family comparison, policy tuning, strict TREC interchange, and auditable report transport. It operates as a standalone package and as a bounded module inside @@ -11,8 +12,10 @@ naruon or another service-oriented system. ## Architectural boundaries 1. **Pure calculation core** — fusion, evaluation, randomization, Holm - correction, and tuning accept in-memory values and perform no network, - database, provider, or filesystem access. + correction, tuning, and exact semantic indexing accept in-memory values and + perform no network, database, provider, or filesystem access. Immutable + semantic snapshots precompute digest-bound scale and norm metadata, then + atomically replace only after complete validation (ADR 0008). 2. **Interchange adapters** — TREC parsers and formatters convert strict text artifacts to immutable domain records. 3. **Transport adapters** — the CLI performs bounded local reads and delegates @@ -45,6 +48,10 @@ scientific validity. ## Module map - `score_fusion.py` — scalar fusion primitives. +- `semantic_vector_ranking.py` — typed adapter to Rust-owned semantic-unit + cosine ranking; authorization and embedding generation remain upstream. +- `semantic_index.py` — typed adapter to immutable exact Rust index snapshots; + the caller owns persistence, model selection, and authorized candidate IDs. - `ranked_list_fusion.py` — complete-list fusion and contribution evidence. - `evaluation.py` — precision, recall, reciprocal rank, and graded nDCG. - `comparison.py` — exact and deterministic Monte Carlo paired randomization. @@ -130,9 +137,9 @@ separate. `publish.yml` independently accepts an external stable release event or the explicit tag/SHA dispatch. Its read-only build job verifies the existing GitHub Release, tag-to-commit identity, default-branch reachability, package version, -complete quality gate, and wheel/source contents. It records a SHA-256 manifest -and uploads both distributions plus that manifest as one immutable Actions -artifact. +complete quality gate, and platform-wheel/source contents. It records a SHA-256 +manifest and uploads the Linux, macOS, and Windows wheels, source distribution, +and that manifest as one immutable Actions artifact. Separate provenance and publication jobs verify the handoff before use. The provenance job creates GitHub build-provenance attestations. The protected diff --git a/CHANGELOG.md b/CHANGELOG.md index affdffe..069d4e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,45 @@ All notable changes to rankweave are documented here. The format follows [Keep a ## [Unreleased] ### Added +- Added immutable exact semantic-unit index snapshots with atomic validated + replacement, digest-bound model/dimension/vector evidence, caller-supplied + row or packed authorization, and deterministic multithreaded Rust scoring + (ADR 0008). +- Added a Rust-owned, versioned semantic-unit cosine ranking API with strict + vector validation, deterministic item/unit ties, winning-unit evidence, and + an ordered-input SHA-256 digest. Provider selection, authorization, weights, + and relevance thresholds remain caller-owned. +- Introduced the first `rankweave-core` Rust vertical slice and thin PyO3 + adapter for theoretical min-max normalization and unweighted Reciprocal Rank + Fusion, with exact public-Python parity and complete core coverage. +- ADR 0005 defines the versioned public-API compatibility policy: names in + `rankweave.__all__` frozen as of `0.18.0` are not removed or renamed within + a minor version, enforced by `tests/test_public_api_compatibility.py`. +### Changed +- Two-channel convex fusion now delegates its deterministic scalar arithmetic + to the Rust calculation core while retaining the public Python validation, + missing-evidence, and exception contracts. +- Bumped the pinned `uv` version from `0.11.29` to `0.12.1` in `pyproject.toml` + and every `astral-sh/setup-uv` workflow step (`ci.yml`, `create-release.yml`, + `publish.yml`) to match the version the central org coverage-evidence + pipeline trusts and hash-verifies. The prior pin was structurally + incompatible with that pipeline's fixed trusted `uv`: `uv export --frozen` + refuses to run whenever the invoking `uv`'s version does not match a + project's own `[tool.uv] required-version`, so every coverage-evidence run + against this repository failed regardless of the PR's own diff + (`ContextualWisdomLab/.github#1234`). `uv.lock` regenerated byte-identical + under `0.12.1` (RankWeave has zero runtime dependencies); full suite + re-verified at 661 passed, 100% statement/branch coverage, ruff clean. - Classic reciprocal-rank fusion results now expose the exact per-channel Cormack contribution beside each owned input rank, so consumers do not need to duplicate the fusion arithmetic. ### Fixed +- Supplied the network-isolated autonomous validation sandbox with a pinned, + pre-fetched Rust 1.97.1 toolchain and Cargo registry, so Maturin can build the + proposed wheel offline without exposing runner credentials or enabling a + second calculation path. - Restricted both autonomous OpenCode phases to explicit repository read paths and removed agent-authored pull-request metadata, preventing workspace-external reads or generated text from becoming a pull-request title or body. @@ -25,18 +58,6 @@ All notable changes to rankweave are documented here. The format follows [Keep a always-current `rankweave-hourly-review-repair.yml` caller added to `ContextualWisdomLab/.github`, matching the pattern already used by every other product repository in the organization. -### Changed -- Bumped the pinned `uv` version from `0.11.29` to `0.12.1` in `pyproject.toml` - and every `astral-sh/setup-uv` workflow step (`ci.yml`, `create-release.yml`, - `publish.yml`) to match the version the central org coverage-evidence - pipeline trusts and hash-verifies. The prior pin was structurally - incompatible with that pipeline's fixed trusted `uv`: `uv export --frozen` - refuses to run whenever the invoking `uv`'s version does not match a - project's own `[tool.uv] required-version`, so every coverage-evidence run - against this repository failed regardless of the PR's own diff - (`ContextualWisdomLab/.github#1234`). `uv.lock` regenerated byte-identical - under `0.12.1` (RankWeave has zero runtime dependencies); full suite - re-verified at 661 passed, 100% statement/branch coverage, ruff clean. ## [0.18.0] - 2026-08-05 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..190f0ee --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,308 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" +dependencies = [ + "libc", + "num-bigint", + "num-traits", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rankweave-core" +version = "0.18.0" +dependencies = [ + "num-bigint", + "num-traits", + "rayon", + "sha2", +] + +[[package]] +name = "rankweave-python" +version = "0.18.0" +dependencies = [ + "num-bigint", + "pyo3", + "rankweave-core", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..eea6076 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +members = ["crates/rankweave-core", "crates/rankweave-python"] +resolver = "2" + +[workspace.package] +edition = "2024" +license = "Apache-2.0" +rust-version = "1.97.1" +version = "0.18.0" diff --git a/README.md b/README.md index 2e7aea7..cdc93e1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # RankWeave -**Dependency-free, store-agnostic retrieval fusion, evaluation, statistical +**Python-runtime-dependency-free, store-agnostic retrieval fusion, evaluation, statistical comparison, tuning, TREC benchmarking, and auditable CLI workflows for Python 3.10+.** @@ -8,8 +8,8 @@ RankWeave combines lexical, dense, learned-sparse, graph, and other retrieval channels into deterministic rankings. It evaluates rankings, compares paired systems, controls family-wise error across candidate experiments, tunes fixed weighted-RRF policies, reads standard TREC artifacts, and exposes both pairwise -and candidate-family comparison contracts to shell and CI users. The runtime -uses only the Python standard library. +and candidate-family comparison contracts to shell and CI users. Python +adapters use only the standard library and the packaged Rust calculation core. RankWeave originated in the Context Search engine of [ContextualWisdomLab/naruon](https://github.com/ContextualWisdomLab/naruon) and @@ -32,7 +32,8 @@ remains suitable both as a standalone package and as a small MSA module. for every run and qrels input without exposing local paths. - **Fail-closed contracts:** malformed values, duplicate identifiers, missing queries, and invalid artifacts raise stable validation errors. -- **Portable core:** Apache-2.0, typed, Python 3.10+, and stdlib-only runtime. +- **Portable core:** Apache-2.0, typed, Python 3.10+, and one packaged Rust + calculation core with no third-party Python runtime dependency. ## Installation @@ -113,6 +114,42 @@ results = weighted_reciprocal_rank_fuse( Complete-list results expose immutable per-channel contribution evidence, including explicit missing channels. +## Rank provider-produced semantic units + +RankWeave compares vectors that your authorized retrieval boundary already +obtained. It does not select or call an embedding model. + +```python +from rankweave import SemanticUnitCandidate, rank_semantic_units + +report = rank_semantic_units( + [1.0, 0.0], + [ + SemanticUnitCandidate("post-a", "paragraph-1", [1.0, 0.0]), + SemanticUnitCandidate("post-a", "paragraph-2", [0.0, 1.0]), + SemanticUnitCandidate("post-b", "paragraph-1", [0.8, 0.2]), + ], +) + +assert report.results[0].item_id == "post-a" +assert report.results[0].winning_unit_id == "paragraph-1" +``` + +The report binds the ordered input with SHA-256, identifies the schema and +algorithm versions, and retains the winning semantic unit for each item. Invalid +dimensions, non-finite values, zero vectors, and duplicate item/unit pairs fail +with stable error codes. See [ADR 0007](docs/adr/0007-semantic-unit-cosine-ranking.md). + +For repeated queries over one governed snapshot, `SemanticUnitExactIndex` +validates canonical packed vectors once, records model/dimension/vector and +snapshot digests, and scores only caller-supplied authorized candidate IDs on +the deterministic multithreaded Rust CPU path. Snapshot replacement is atomic; +v1 exposes no partial mutation or approximate retrieval. The caller still owns +persistence, model selection, ABAC, and result post-authorization. See +[ADR 0008](docs/adr/0008-persistent-exact-semantic-index.md). +Large authorization sets may use the canonical length-prefixed packed identity +transport; it is exact and digest-equivalent to the ordinary identity rows. + ## Evaluate ranking quality ```python diff --git a/crates/rankweave-core/Cargo.toml b/crates/rankweave-core/Cargo.toml new file mode 100644 index 0000000..8147f00 --- /dev/null +++ b/crates/rankweave-core/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rankweave-core" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +num-bigint = "0.4.6" +num-traits = "0.2.19" +sha2 = "0.10.9" +rayon = "1.11.0" diff --git a/crates/rankweave-core/examples/profile_accelerate_batch.rs b/crates/rankweave-core/examples/profile_accelerate_batch.rs new file mode 100644 index 0000000..d953866 --- /dev/null +++ b/crates/rankweave-core/examples/profile_accelerate_batch.rs @@ -0,0 +1,271 @@ +//! Measure Accelerate f64 matrix-matrix parity at the consumer workload shape. + +#[cfg(target_os = "macos")] +mod macos { + use std::time::Instant; + + const CANDIDATES: usize = 6_578; + const DIMENSION: usize = 3_072; + const QUERIES: usize = 4; + const CBLAS_ROW_MAJOR: i32 = 101; + const CBLAS_NO_TRANS: i32 = 111; + + fn gamma(term_count: usize) -> f64 { + let unit_roundoff = 2.0_f64.powi(-53); + let product = term_count as f64 * unit_roundoff; + product / (1.0 - product) + } + + fn norm(values: impl Iterator) -> f64 { + values.map(|value| value * value).sum::().sqrt() + } + + fn ambiguity_set(intervals: &[(f64, f64)], limit: usize) -> Vec { + let mut lower_bounds = intervals + .iter() + .enumerate() + .map(|(index, interval)| (interval.0, index)) + .collect::>(); + lower_bounds.sort_by(|left, right| { + right + .0 + .total_cmp(&left.0) + .then_with(|| left.1.cmp(&right.1)) + }); + let kth_lower = lower_bounds[limit - 1].0; + intervals + .iter() + .enumerate() + .filter_map(|(index, interval)| (interval.1 >= kth_lower).then_some(index)) + .collect() + } + + #[link(name = "Accelerate", kind = "framework")] + unsafe extern "C" { + fn cblas_dgemm( + order: i32, + transpose_a: i32, + transpose_b: i32, + rows_a: i32, + columns_b: i32, + shared_dimension: i32, + alpha: f64, + a: *const f64, + leading_a: i32, + b: *const f64, + leading_b: i32, + beta: f64, + c: *mut f64, + leading_c: i32, + ); + } + + fn value(row: usize, column: usize) -> f64 { + let residue = (row.wrapping_mul(131) + column.wrapping_mul(17)) % 1_009; + (residue as f64 - 504.0) / 509.0 + } + + fn multiply(matrix: &[f64], queries_by_coordinate: &[f64], output: &mut [f64]) { + // SAFETY: all pointers reference contiguous f64 buffers whose exact + // row-major extents and leading dimensions are supplied below. + unsafe { + cblas_dgemm( + CBLAS_ROW_MAJOR, + CBLAS_NO_TRANS, + CBLAS_NO_TRANS, + CANDIDATES as i32, + QUERIES as i32, + DIMENSION as i32, + 1.0, + matrix.as_ptr(), + DIMENSION as i32, + queries_by_coordinate.as_ptr(), + QUERIES as i32, + 0.0, + output.as_mut_ptr(), + QUERIES as i32, + ); + } + } + + fn top_four(scores: &[f64], query: usize) -> [usize; 4] { + let mut candidates = (0..CANDIDATES).collect::>(); + candidates.sort_by(|left, right| { + scores[right * QUERIES + query] + .total_cmp(&scores[left * QUERIES + query]) + .then_with(|| left.cmp(right)) + }); + candidates[..4].try_into().expect("four ranked candidates") + } + + fn screened_top_four( + matrix: &[f64], + queries_by_coordinate: &[f64], + approximate_dots: &[f64], + candidate_norms: &[f64], + query_norms: &[f64], + query: usize, + ) -> ([usize; 4], usize) { + // Higham's dot-product model bounds each BLAS result and the + // coordinate-ordered scalar reference by gamma_n * |x|^T|y|. + // Cauchy-Schwarz gives |x|^T|y| <= ||x|| ||y||, so the scalar + // cosine lies within 2*gamma_n of the BLAS cosine. Equality remains + // ambiguous; only a strict upper-bound exclusion is safe. + let error = 2.0 * gamma(DIMENSION); + let intervals = (0..CANDIDATES) + .map(|candidate| { + let approximate = approximate_dots[candidate * QUERIES + query] + / (candidate_norms[candidate] * query_norms[query]); + ( + (approximate - error).max(-1.0), + (approximate + error).min(1.0), + ) + }) + .collect::>(); + let ambiguous = ambiguity_set(&intervals, 4); + let mut exact = ambiguous + .iter() + .map(|candidate| { + let dot = (0..DIMENSION).fold(0.0, |sum, coordinate| { + sum + matrix[candidate * DIMENSION + coordinate] + * queries_by_coordinate[coordinate * QUERIES + query] + }); + ( + dot / (candidate_norms[*candidate] * query_norms[query]), + *candidate, + ) + }) + .collect::>(); + exact.sort_by(|left, right| { + right + .0 + .total_cmp(&left.0) + .then_with(|| left.1.cmp(&right.1)) + }); + ( + exact[..4] + .iter() + .map(|entry| entry.1) + .collect::>() + .try_into() + .expect("four exact ambiguous candidates"), + ambiguous.len(), + ) + } + + pub fn run() { + let matrix = (0..CANDIDATES) + .flat_map(|row| (0..DIMENSION).map(move |column| value(row, column))) + .collect::>(); + let queries_by_coordinate = (0..DIMENSION) + .flat_map(|coordinate| { + (0..QUERIES).map(move |query| value(CANDIDATES + query, coordinate)) + }) + .collect::>(); + let mut accelerate = vec![0.0; CANDIDATES * QUERIES]; + multiply(&matrix, &queries_by_coordinate, &mut accelerate); + let candidate_norms = (0..CANDIDATES) + .map(|candidate| { + norm( + matrix[candidate * DIMENSION..(candidate + 1) * DIMENSION] + .iter() + .copied(), + ) + }) + .collect::>(); + let query_norms = (0..QUERIES) + .map(|query| { + norm( + (0..DIMENSION) + .map(|coordinate| queries_by_coordinate[coordinate * QUERIES + query]), + ) + }) + .collect::>(); + let mut scalar = vec![0.0; CANDIDATES * QUERIES]; + for candidate in 0..CANDIDATES { + for coordinate in 0..DIMENSION { + let candidate_value = matrix[candidate * DIMENSION + coordinate]; + for query in 0..QUERIES { + scalar[candidate * QUERIES + query] += + candidate_value * queries_by_coordinate[coordinate * QUERIES + query]; + } + } + } + let mismatched = scalar + .iter() + .zip(&accelerate) + .filter(|(left, right)| left.to_bits() != right.to_bits()) + .count(); + let maximum_absolute_difference = scalar + .iter() + .zip(&accelerate) + .map(|(left, right)| (left - right).abs()) + .fold(0.0, f64::max); + let top_four_mismatches = (0..QUERIES) + .filter(|query| top_four(&scalar, *query) != top_four(&accelerate, *query)) + .count(); + let screened = (0..QUERIES) + .map(|query| { + screened_top_four( + &matrix, + &queries_by_coordinate, + &accelerate, + &candidate_norms, + &query_norms, + query, + ) + }) + .collect::>(); + let screened_top_four_mismatches = screened + .iter() + .enumerate() + .filter(|(query, result)| top_four(&scalar, *query) != result.0) + .count(); + let maximum_ambiguity = screened + .iter() + .map(|result| result.1) + .max() + .expect("queries are nonempty"); + let mut elapsed = Vec::new(); + for _ in 0..30 { + let started = Instant::now(); + multiply(&matrix, &queries_by_coordinate, &mut accelerate); + for query in 0..QUERIES { + let _ = screened_top_four( + &matrix, + &queries_by_coordinate, + &accelerate, + &candidate_norms, + &query_norms, + query, + ); + } + elapsed.push(started.elapsed().as_secs_f64() * 1_000.0); + } + elapsed.sort_by(f64::total_cmp); + let mean = elapsed.iter().sum::() / elapsed.len() as f64; + println!( + "shape={CANDIDATES}x{DIMENSION}x{QUERIES} min_ms={:.3} mean_ms={mean:.3} p95_ms={:.3} max_ms={:.3} bit_mismatches={mismatched} approximate_top4_mismatches={top_four_mismatches} screened_top4_mismatches={screened_top_four_mismatches} maximum_ambiguity={maximum_ambiguity} max_abs_diff={maximum_absolute_difference:e}", + elapsed[0], elapsed[28], elapsed[29] + ); + } + + #[test] + fn equal_intervals_force_complete_scalar_fallback() { + let intervals = vec![(0.5, 0.5); 8]; + assert_eq!(ambiguity_set(&intervals, 4), (0..8).collect::>()); + } + + #[test] + fn near_tie_interval_is_never_excluded() { + let intervals = vec![(0.9, 0.91), (0.89, 0.905), (0.1, 0.2)]; + assert_eq!(ambiguity_set(&intervals, 1), vec![0, 1]); + } +} + +fn main() { + #[cfg(target_os = "macos")] + macos::run(); + #[cfg(not(target_os = "macos"))] + eprintln!("Accelerate profile is available only on macOS"); +} diff --git a/crates/rankweave-core/examples/profile_semantic_index.rs b/crates/rankweave-core/examples/profile_semantic_index.rs new file mode 100644 index 0000000..16d0094 --- /dev/null +++ b/crates/rankweave-core/examples/profile_semantic_index.rs @@ -0,0 +1,71 @@ +//! Profile exact snapshot build and authorized ranking with synthetic vectors. + +use std::env; +use std::time::Instant; + +use rankweave_core::semantic_index::SemanticUnitIndex; + +fn positive_usize(value: Option, label: &str) -> usize { + let parsed = value + .unwrap_or_else(|| panic!("missing {label}")) + .parse::() + .unwrap_or_else(|_| panic!("invalid {label}")); + assert!(parsed > 0, "{label} must be positive"); + parsed +} + +fn main() { + let mut arguments = env::args().skip(1); + let candidate_count = positive_usize(arguments.next(), "candidate count"); + let vector_dimension = positive_usize(arguments.next(), "vector dimension"); + let item_count = positive_usize(arguments.next(), "item count"); + assert!( + item_count <= candidate_count, + "item count exceeds candidates" + ); + assert!(arguments.next().is_none(), "unexpected argument"); + + let mut one_vector = Vec::with_capacity(vector_dimension * size_of::()); + one_vector.extend_from_slice(&1.0_f64.to_be_bytes()); + one_vector.resize(vector_dimension * size_of::(), 0); + let packed_vectors = one_vector.repeat(candidate_count); + let candidate_ids = (0..candidate_count) + .map(|index| { + ( + format!("item-{:08}", index % item_count), + format!("unit-{index:08}"), + ) + }) + .collect::>(); + let query_vector = std::iter::once(1.0) + .chain(std::iter::repeat_n(0.0, vector_dimension - 1)) + .collect::>(); + + let build_started = Instant::now(); + let index = SemanticUnitIndex::build( + "synthetic-snapshot-v1", + "synthetic-model-v1", + vector_dimension, + candidate_ids.clone(), + &packed_vectors, + ) + .expect("synthetic snapshot must build"); + let build_elapsed = build_started.elapsed(); + + let rank_started = Instant::now(); + let report = index + .rank_authorized("synthetic-model-v1", &query_vector, &candidate_ids) + .expect("synthetic authorization must rank"); + let rank_elapsed = rank_started.elapsed(); + + println!( + "candidates={candidate_count} dimension={vector_dimension} items={item_count} bytes={} build_ms={:.3} rank_ms={:.3} workers={} results={} input_digest={} output_digest={}", + packed_vectors.len(), + build_elapsed.as_secs_f64() * 1_000.0, + rank_elapsed.as_secs_f64() * 1_000.0, + report.worker_count, + report.results.len(), + report.ordered_input_digest, + report.output_digest, + ); +} diff --git a/crates/rankweave-core/src/lib.rs b/crates/rankweave-core/src/lib.rs new file mode 100644 index 0000000..c179390 --- /dev/null +++ b/crates/rankweave-core/src/lib.rs @@ -0,0 +1,649 @@ +//! Deterministic calculation primitives for RankWeave. + +use std::collections::{HashMap, HashSet}; +use std::fmt; + +use num_bigint::BigUint; +use num_traits::ToPrimitive; +use sha2::{Digest, Sha256}; + +pub mod semantic_index; + +/// Version of the semantic-unit ranking result envelope. +pub const SEMANTIC_UNIT_RANKING_SCHEMA_VERSION: &str = "rankweave.semantic-unit-ranking.v1"; + +/// Version of the semantic-unit cosine calculation contract. +pub const SEMANTIC_UNIT_COSINE_ALGORITHM_VERSION: &str = "rankweave.semantic-unit-cosine.v1"; + +/// One caller-owned semantic unit and its embedding vector. +#[derive(Clone, Debug, PartialEq)] +pub struct SemanticUnitCandidate { + /// Opaque identifier for the containing item. + pub item_id: String, + /// Opaque identifier for the semantic unit within the item. + pub unit_id: String, + /// Provider-produced vector; RankWeave does not select its model. + pub vector: Vec, +} + +/// The highest-scoring semantic unit retained for one item. +#[derive(Clone, Debug, PartialEq)] +pub struct SemanticUnitRank { + /// Opaque caller-owned item identifier. + pub item_id: String, + /// Opaque identifier of the unit that produced `score`. + pub winning_unit_id: String, + /// Raw cosine clamped to `[0, 1]`, without remapping or a cutoff. + pub score: f64, +} + +/// Versioned, reproducible semantic-unit ranking evidence. +#[derive(Clone, Debug, PartialEq)] +pub struct SemanticUnitRankingReport { + /// Stable result-envelope identifier. + pub schema_version: &'static str, + /// Stable calculation-contract identifier. + pub algorithm_version: &'static str, + /// SHA-256 over the canonical ordered request bytes. + pub ordered_input_digest: String, + /// Exact dimension shared by every accepted vector. + pub vector_dimension: usize, + /// Results sorted by descending score, then item identifier. + pub results: Vec, +} + +/// Explicit fail-closed semantic-unit ranking failures. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SemanticUnitRankingError { + /// The query vector has no dimensions. + EmptyQueryVector, + /// No candidate semantic unit was supplied. + EmptyCandidates, + /// A vector contains NaN or infinity. + NonFiniteVector { vector_label: String }, + /// A candidate dimension differs from the query dimension. + DimensionMismatch { + vector_label: String, + expected: usize, + actual: usize, + }, + /// Cosine is undefined for an all-zero vector. + ZeroNormVector { vector_label: String }, + /// An item/unit identity pair occurs more than once. + DuplicateCandidate { item_id: String, unit_id: String }, + /// Packed vectors do not contain exactly one binary64 value per coordinate. + PackedVectorByteLength { expected: usize, actual: usize }, +} + +impl SemanticUnitRankingError { + /// Stable machine-readable failure code. + #[must_use] + pub const fn code(&self) -> &'static str { + match self { + Self::EmptyQueryVector => "empty_query_vector", + Self::EmptyCandidates => "empty_candidates", + Self::NonFiniteVector { .. } => "non_finite_vector", + Self::DimensionMismatch { .. } => "dimension_mismatch", + Self::ZeroNormVector { .. } => "zero_norm_vector", + Self::DuplicateCandidate { .. } => "duplicate_candidate", + Self::PackedVectorByteLength { .. } => "packed_vector_byte_length", + } + } +} + +impl fmt::Display for SemanticUnitRankingError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyQueryVector => formatter.write_str("query vector must not be empty"), + Self::EmptyCandidates => formatter.write_str("candidates must not be empty"), + Self::NonFiniteVector { vector_label } => { + write!(formatter, "{vector_label} must contain only finite values") + } + Self::DimensionMismatch { + vector_label, + expected, + actual, + } => write!( + formatter, + "{vector_label} dimension must be {expected}, got {actual}" + ), + Self::ZeroNormVector { vector_label } => { + write!(formatter, "{vector_label} must have a non-zero norm") + } + Self::DuplicateCandidate { item_id, unit_id } => write!( + formatter, + "candidate ({item_id:?}, {unit_id:?}) must be unique" + ), + Self::PackedVectorByteLength { expected, actual } => write!( + formatter, + "packed candidate vectors must contain {expected} bytes, got {actual}" + ), + } + } +} + +impl std::error::Error for SemanticUnitRankingError {} + +fn validate_vector( + vector: &[f64], + expected_dimension: usize, + vector_label: String, +) -> Result<(), SemanticUnitRankingError> { + if vector.len() != expected_dimension { + return Err(SemanticUnitRankingError::DimensionMismatch { + vector_label, + expected: expected_dimension, + actual: vector.len(), + }); + } + if vector.iter().any(|value| !value.is_finite()) { + return Err(SemanticUnitRankingError::NonFiniteVector { vector_label }); + } + if vector.iter().all(|value| *value == 0.0) { + return Err(SemanticUnitRankingError::ZeroNormVector { vector_label }); + } + Ok(()) +} + +fn cosine_similarity(left: &[f64], right: &[f64]) -> f64 { + let left_scale = left.iter().map(|value| value.abs()).fold(0.0, f64::max); + let right_scale = right.iter().map(|value| value.abs()).fold(0.0, f64::max); + let (dot, left_squared, right_squared) = left.iter().zip(right).fold( + (0.0, 0.0, 0.0), + |(dot, left_squared, right_squared), (left_value, right_value)| { + let scaled_left = left_value / left_scale; + let scaled_right = right_value / right_scale; + ( + dot + scaled_left * scaled_right, + left_squared + scaled_left * scaled_left, + right_squared + scaled_right * scaled_right, + ) + }, + ); + (dot / (left_squared.sqrt() * right_squared.sqrt())).clamp(0.0, 1.0) +} + +fn update_length_prefixed(hasher: &mut Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +fn ordered_input_digest(query_vector: &[f64], candidates: &[SemanticUnitCandidate]) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"rankweave.semantic-unit-ranking.request.v1\0"); + hasher.update((query_vector.len() as u64).to_be_bytes()); + for value in query_vector { + hasher.update(value.to_bits().to_be_bytes()); + } + hasher.update((candidates.len() as u64).to_be_bytes()); + for candidate in candidates { + update_length_prefixed(&mut hasher, candidate.item_id.as_bytes()); + update_length_prefixed(&mut hasher, candidate.unit_id.as_bytes()); + hasher.update((candidate.vector.len() as u64).to_be_bytes()); + for value in &candidate.vector { + hasher.update(value.to_bits().to_be_bytes()); + } + } + format!("sha256:{:x}", hasher.finalize()) +} + +fn ordered_packed_input_digest( + query_vector: &[f64], + candidate_ids: &[(String, String)], + packed_vectors: &[u8], +) -> String { + let mut hasher = Sha256::new(); + hasher.update(b"rankweave.semantic-unit-ranking.request.v1\0"); + hasher.update((query_vector.len() as u64).to_be_bytes()); + for value in query_vector { + hasher.update(value.to_bits().to_be_bytes()); + } + hasher.update((candidate_ids.len() as u64).to_be_bytes()); + let vector_byte_count = size_of_val(query_vector); + for (index, (item_id, unit_id)) in candidate_ids.iter().enumerate() { + update_length_prefixed(&mut hasher, item_id.as_bytes()); + update_length_prefixed(&mut hasher, unit_id.as_bytes()); + hasher.update((query_vector.len() as u64).to_be_bytes()); + let start = index * vector_byte_count; + hasher.update(&packed_vectors[start..start + vector_byte_count]); + } + format!("sha256:{:x}", hasher.finalize()) +} + +fn consider_semantic_unit( + query_vector: &[f64], + identities: &mut HashSet<(String, String)>, + best_by_item: &mut HashMap, + item_id: &str, + unit_id: &str, + vector: &[f64], +) -> Result<(), SemanticUnitRankingError> { + if !identities.insert((item_id.to_owned(), unit_id.to_owned())) { + return Err(SemanticUnitRankingError::DuplicateCandidate { + item_id: item_id.to_owned(), + unit_id: unit_id.to_owned(), + }); + } + let vector_label = format!("candidate vector for item {item_id:?}, unit {unit_id:?}"); + validate_vector(vector, query_vector.len(), vector_label)?; + let proposed = SemanticUnitRank { + item_id: item_id.to_owned(), + winning_unit_id: unit_id.to_owned(), + score: cosine_similarity(query_vector, vector), + }; + best_by_item + .entry(item_id.to_owned()) + .and_modify(|current| { + if proposed.score > current.score + || (proposed.score == current.score + && proposed.winning_unit_id < current.winning_unit_id) + { + *current = proposed.clone(); + } + }) + .or_insert(proposed); + Ok(()) +} + +fn finish_semantic_unit_ranking( + query_vector: &[f64], + best_by_item: HashMap, + ordered_input_digest: String, +) -> SemanticUnitRankingReport { + let mut results: Vec<_> = best_by_item.into_values().collect(); + results.sort_by(|left, right| { + right + .score + .total_cmp(&left.score) + .then_with(|| left.item_id.cmp(&right.item_id)) + }); + SemanticUnitRankingReport { + schema_version: SEMANTIC_UNIT_RANKING_SCHEMA_VERSION, + algorithm_version: SEMANTIC_UNIT_COSINE_ALGORITHM_VERSION, + ordered_input_digest, + vector_dimension: query_vector.len(), + results, + } +} + +/// Rank items by their best caller-supplied semantic-unit cosine evidence. +/// +/// Candidate order is part of the digest, while result order is descending +/// score then ascending item identifier. Equal-scoring units for one item use +/// ascending unit identifier. No model, weight, threshold, or authorization +/// decision is made here. +pub fn rank_semantic_units( + query_vector: &[f64], + candidates: &[SemanticUnitCandidate], +) -> Result { + if query_vector.is_empty() { + return Err(SemanticUnitRankingError::EmptyQueryVector); + } + if candidates.is_empty() { + return Err(SemanticUnitRankingError::EmptyCandidates); + } + validate_vector(query_vector, query_vector.len(), "query vector".to_owned())?; + + let mut identities = HashSet::new(); + let mut best_by_item: HashMap = HashMap::new(); + for candidate in candidates { + consider_semantic_unit( + query_vector, + &mut identities, + &mut best_by_item, + &candidate.item_id, + &candidate.unit_id, + &candidate.vector, + )?; + } + Ok(finish_semantic_unit_ranking( + query_vector, + best_by_item, + ordered_input_digest(query_vector, candidates), + )) +} + +/// Rank canonical big-endian binary64 vectors without Python scalar expansion. +/// +/// Candidate vectors are concatenated in candidate identifier order. This +/// transport produces the same schema, algorithm, digest, and results as the +/// ordinary semantic-unit API for the equivalent decoded vectors. +pub fn rank_semantic_units_packed( + query_vector: &[f64], + candidate_ids: &[(String, String)], + packed_vectors: &[u8], +) -> Result { + if query_vector.is_empty() { + return Err(SemanticUnitRankingError::EmptyQueryVector); + } + if candidate_ids.is_empty() { + return Err(SemanticUnitRankingError::EmptyCandidates); + } + validate_vector(query_vector, query_vector.len(), "query vector".to_owned())?; + let expected = candidate_ids + .len() + .checked_mul(query_vector.len()) + .and_then(|count| count.checked_mul(size_of::())) + .unwrap_or(usize::MAX); + if packed_vectors.len() != expected { + return Err(SemanticUnitRankingError::PackedVectorByteLength { + expected, + actual: packed_vectors.len(), + }); + } + + let vector_byte_count = size_of_val(query_vector); + let mut identities = HashSet::new(); + let mut best_by_item = HashMap::new(); + let mut vector = Vec::with_capacity(query_vector.len()); + for (index, (item_id, unit_id)) in candidate_ids.iter().enumerate() { + vector.clear(); + let start = index * vector_byte_count; + for bytes in packed_vectors[start..start + vector_byte_count].chunks_exact(8) { + vector.push(f64::from_be_bytes( + bytes.try_into().expect("eight-byte chunk"), + )); + } + consider_semantic_unit( + query_vector, + &mut identities, + &mut best_by_item, + item_id, + unit_id, + &vector, + )?; + } + Ok(finish_semantic_unit_ranking( + query_vector, + best_by_item, + ordered_packed_input_digest(query_vector, candidate_ids, packed_vectors), + )) +} + +/// Scale a finite score with finite theoretical bounds and clamp it to `[0, 1]`. +#[must_use] +pub fn theoretical_min_max_normalize(score: f64, lower: f64, upper: f64) -> f64 { + if score <= lower { + return 0.0; + } + if score >= upper { + return 1.0; + } + let width = upper - lower; + let normalized = if width.is_finite() { + (score - lower) / width + } else { + (score / 2.0 - lower / 2.0) / (upper / 2.0 - lower / 2.0) + }; + normalized.clamp(0.0, 1.0) +} + +/// Combine two normalized scores using the caller-supplied semantic weight. +/// +/// Missing candidate evidence is represented by `None` and contributes the +/// documented theoretical minimum, zero. Validation remains at the public +/// Python boundary; this function owns only the deterministic arithmetic. +#[must_use] +pub fn convex_combination_score( + semantic_score: Option, + lexical_score: Option, + semantic_weight_alpha: f64, +) -> f64 { + let semantic_component = semantic_score.unwrap_or(0.0); + let lexical_component = lexical_score.unwrap_or(0.0); + semantic_weight_alpha * semantic_component + (1.0 - semantic_weight_alpha) * lexical_component +} + +/// Sum Reciprocal Rank Fusion contributions in caller-provided channel order. +#[must_use] +pub fn reciprocal_rank_fusion_score(ranks: &[BigUint], rank_constant_eta: &BigUint) -> f64 { + ranks.iter().fold(0.0, |score, rank| { + let denominator = (rank_constant_eta + rank).to_f64().unwrap_or(f64::INFINITY); + score + 1.0 / denominator + }) +} + +#[cfg(test)] +mod tests { + use super::{ + SemanticUnitCandidate, SemanticUnitRankingError, convex_combination_score, + rank_semantic_units, rank_semantic_units_packed, reciprocal_rank_fusion_score, + theoretical_min_max_normalize, + }; + use num_bigint::BigUint; + + #[test] + fn normalization_uses_theoretical_bounds_and_clamps() { + assert_eq!(theoretical_min_max_normalize(0.5, 0.0, 2.0), 0.25); + assert_eq!(theoretical_min_max_normalize(3.0, 0.0, 2.0), 1.0); + assert_eq!(theoretical_min_max_normalize(0.0, -f64::MAX, f64::MAX), 0.5); + assert_eq!( + theoretical_min_max_normalize(-f64::MAX, -f64::MAX, f64::MAX), + 0.0 + ); + assert_eq!( + theoretical_min_max_normalize(f64::MAX, -f64::MAX, f64::MAX), + 1.0 + ); + } + + #[test] + fn convex_fusion_preserves_formula_and_missing_evidence_semantics() { + assert_eq!( + convex_combination_score(Some(0.8), Some(0.5), 0.7), + 0.7 * 0.8 + (1.0 - 0.7) * 0.5 + ); + assert_eq!( + convex_combination_score(None, Some(0.5), 0.7), + (1.0 - 0.7) * 0.5 + ); + assert_eq!(convex_combination_score(Some(0.8), None, 0.7), 0.7 * 0.8); + assert_eq!(convex_combination_score(None, None, 0.7), 0.0); + } + + #[test] + fn rrf_preserves_input_order_for_the_reduction() { + let ranks = [BigUint::from(1_u8), BigUint::from(3_u8)]; + assert_eq!( + reciprocal_rank_fusion_score(&ranks, &BigUint::from(60_u8)), + 1.0 / 61.0 + 1.0 / 63.0 + ); + } + + fn candidate(item_id: &str, unit_id: &str, vector: &[f64]) -> SemanticUnitCandidate { + SemanticUnitCandidate { + item_id: item_id.to_owned(), + unit_id: unit_id.to_owned(), + vector: vector.to_vec(), + } + } + + #[test] + fn semantic_units_rank_by_best_unit_then_item_id() { + let candidates = vec![ + candidate("item-b", "unit-z", &[1.0, 0.0]), + candidate("item-a", "unit-z", &[1.0, 0.0]), + candidate("item-c", "unit-b", &[-1.0, 0.0]), + candidate("item-c", "unit-a", &[0.0, 1.0]), + candidate("item-c", "unit-c", &[-1.0, 0.0]), + ]; + let report = rank_semantic_units(&[1.0, 0.0], &candidates).unwrap(); + + assert_eq!(report.schema_version, "rankweave.semantic-unit-ranking.v1"); + assert_eq!( + report.algorithm_version, + "rankweave.semantic-unit-cosine.v1" + ); + assert!(report.ordered_input_digest.starts_with("sha256:")); + assert_eq!(report.vector_dimension, 2); + assert_eq!(report.results[0].item_id, "item-a"); + assert_eq!(report.results[1].item_id, "item-b"); + assert_eq!(report.results[2].winning_unit_id, "unit-a"); + assert_eq!(report.results[2].score, 0.0); + } + + #[test] + fn semantic_unit_digest_binds_order_and_exact_float_bits() { + let first = vec![ + candidate("item-a", "unit-a", &[1.0, 0.0]), + candidate("item-b", "unit-b", &[0.0, 1.0]), + ]; + let reversed = vec![first[1].clone(), first[0].clone()]; + let first_report = rank_semantic_units(&[1.0, 0.0], &first).unwrap(); + let repeated_report = rank_semantic_units(&[1.0, 0.0], &first).unwrap(); + let reversed_report = rank_semantic_units(&[1.0, 0.0], &reversed).unwrap(); + + assert_eq!( + first_report.ordered_input_digest, + repeated_report.ordered_input_digest + ); + assert_ne!( + first_report.ordered_input_digest, + reversed_report.ordered_input_digest + ); + } + + #[test] + fn semantic_unit_cosine_avoids_finite_vector_overflow() { + let report = rank_semantic_units( + &[f64::MAX, f64::MAX], + &[candidate("item", "unit", &[f64::MAX, f64::MAX])], + ) + .unwrap(); + assert!(report.results[0].score.is_finite()); + assert!(report.results[0].score > 0.999_999_999_999); + } + + #[test] + fn packed_semantic_units_preserve_exact_report_and_digest() { + let candidates = vec![ + candidate("item-b", "unit-z", &[1.0, 0.0]), + candidate("item-a", "unit-z", &[0.0, 1.0]), + ]; + let candidate_ids = candidates + .iter() + .map(|candidate| (candidate.item_id.clone(), candidate.unit_id.clone())) + .collect::>(); + let packed_vectors = candidates + .iter() + .flat_map(|candidate| candidate.vector.iter()) + .flat_map(|value| value.to_be_bytes()) + .collect::>(); + + assert_eq!( + rank_semantic_units_packed(&[1.0, 0.0], &candidate_ids, &packed_vectors).unwrap(), + rank_semantic_units(&[1.0, 0.0], &candidates).unwrap() + ); + } + + #[test] + fn packed_semantic_unit_validation_failures_are_explicit() { + let one_candidate = vec![("item".to_owned(), "unit".to_owned())]; + let cases = [ + ( + rank_semantic_units_packed(&[], &one_candidate, &[]), + "empty_query_vector", + ), + ( + rank_semantic_units_packed(&[1.0], &[], &[]), + "empty_candidates", + ), + ( + rank_semantic_units_packed(&[1.0], &one_candidate, b"short"), + "packed_vector_byte_length", + ), + ( + rank_semantic_units_packed(&[f64::NAN], &one_candidate, &1.0_f64.to_be_bytes()), + "non_finite_vector", + ), + ( + rank_semantic_units_packed(&[0.0], &one_candidate, &1.0_f64.to_be_bytes()), + "zero_norm_vector", + ), + ( + rank_semantic_units_packed(&[1.0], &one_candidate, &f64::NAN.to_be_bytes()), + "non_finite_vector", + ), + ( + rank_semantic_units_packed(&[1.0], &one_candidate, &0.0_f64.to_be_bytes()), + "zero_norm_vector", + ), + ( + rank_semantic_units_packed( + &[1.0], + &[ + ("item".to_owned(), "unit".to_owned()), + ("item".to_owned(), "unit".to_owned()), + ], + &[1.0_f64.to_be_bytes(), 1.0_f64.to_be_bytes()].concat(), + ), + "duplicate_candidate", + ), + ]; + for (result, expected_code) in cases { + assert_eq!(result.unwrap_err().code(), expected_code); + } + } + + #[test] + fn semantic_unit_validation_failures_are_explicit() { + let cases = [ + ( + rank_semantic_units(&[], &[candidate("item", "unit", &[1.0])]), + "empty_query_vector", + ), + (rank_semantic_units(&[1.0], &[]), "empty_candidates"), + ( + rank_semantic_units(&[f64::NAN], &[candidate("item", "unit", &[1.0])]), + "non_finite_vector", + ), + ( + rank_semantic_units(&[1.0], &[candidate("item", "unit", &[1.0, 2.0])]), + "dimension_mismatch", + ), + ( + rank_semantic_units(&[0.0], &[candidate("item", "unit", &[1.0])]), + "zero_norm_vector", + ), + ( + rank_semantic_units(&[1.0], &[candidate("item", "unit", &[0.0])]), + "zero_norm_vector", + ), + ( + rank_semantic_units( + &[1.0], + &[ + candidate("item", "unit", &[1.0]), + candidate("item", "unit", &[1.0]), + ], + ), + "duplicate_candidate", + ), + ]; + + for (result, expected_code) in cases { + let error = result.unwrap_err(); + assert_eq!(error.code(), expected_code); + assert!(!error.to_string().is_empty()); + assert_eq!(error, error.clone()); + assert!(!format!("{error:?}").is_empty()); + } + assert_eq!( + SemanticUnitRankingError::DimensionMismatch { + vector_label: "candidate".to_owned(), + expected: 2, + actual: 1, + } + .to_string(), + "candidate dimension must be 2, got 1" + ); + assert_eq!( + SemanticUnitRankingError::PackedVectorByteLength { + expected: 16, + actual: 5, + } + .to_string(), + "packed candidate vectors must contain 16 bytes, got 5" + ); + } +} diff --git a/crates/rankweave-core/src/semantic_index.rs b/crates/rankweave-core/src/semantic_index.rs new file mode 100644 index 0000000..2cbb2dd --- /dev/null +++ b/crates/rankweave-core/src/semantic_index.rs @@ -0,0 +1,2091 @@ +//! Persistent exact semantic-unit index with deterministic parallel scoring. + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::sync::{Arc, RwLock}; + +use rayon::prelude::*; +use sha2::{Digest, Sha256}; + +use crate::{SEMANTIC_UNIT_COSINE_ALGORITHM_VERSION, SemanticUnitRank}; + +/// Version of the immutable exact-index snapshot contract. +pub const SEMANTIC_INDEX_SNAPSHOT_SCHEMA_VERSION: &str = + "rankweave.semantic-unit-index-snapshot.v1"; + +/// Deterministic portable CPU execution profile. +pub const SEMANTIC_INDEX_CPU_EXECUTION_PROFILE: &str = "rankweave.semantic-unit-index.cpu-rayon.v1"; + +/// Exact top-k profile after a conservative scalar fallback. +pub const SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE: &str = + "rankweave.semantic-unit-index.top-k.cpu-rayon.v1"; + +/// Exact top-k profile screened by an interval-bounded Apple Accelerate call. +#[cfg(target_os = "macos")] +pub const SEMANTIC_INDEX_TOP_K_ACCELERATE_EXECUTION_PROFILE: &str = + "rankweave.semantic-unit-index.top-k.accelerate-interval.v1"; + +/// Immutable evidence describing one validated exact index snapshot. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SemanticIndexSnapshotEvidence { + /// Versioned snapshot-envelope identifier. + pub schema_version: &'static str, + /// Caller-owned immutable snapshot version. + pub snapshot_version: String, + /// SHA-256 binding the opaque model identity. + pub model_digest: String, + /// SHA-256 binding the vector dimension. + pub dimension_digest: String, + /// SHA-256 binding candidate identities and exact binary64 values. + pub vectors_digest: String, + /// SHA-256 binding all preceding snapshot evidence. + pub snapshot_digest: String, + /// Exact vector dimension. + pub vector_dimension: usize, + /// Number of indexed semantic units. + pub candidate_count: usize, +} + +/// Versioned exact-ranking result from one immutable index snapshot. +#[derive(Clone, Debug, PartialEq)] +pub struct SemanticIndexRankingReport { + /// Snapshot evidence used without mutation for this query. + pub snapshot: SemanticIndexSnapshotEvidence, + /// Existing exact cosine algorithm version. + pub algorithm_version: &'static str, + /// Portable deterministic CPU profile. + pub execution_profile: &'static str, + /// Number of Rayon workers visible to the owner runtime. + pub worker_count: usize, + /// SHA-256 over the snapshot, query, model, and ordered authorization set. + pub ordered_input_digest: String, + /// SHA-256 over the exact ordered result rows. + pub output_digest: String, + /// Exact item ranking after per-item maximum pooling. + pub results: Vec, +} + +/// Explicit fail-closed snapshot and authorization failures. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum SemanticIndexError { + /// Snapshot version is absent. + EmptySnapshotVersion, + /// Model identity is absent. + EmptyModelIdentity, + /// Vector dimension is zero. + EmptyVectorDimension, + /// Snapshot contains no candidates. + EmptyCandidates, + /// Packed vector bytes do not match candidate count and dimension. + PackedVectorByteLength { expected: usize, actual: usize }, + /// A vector contains NaN or infinity. + NonFiniteVector { vector_label: String }, + /// A vector has zero norm. + ZeroNormVector { vector_label: String }, + /// A candidate identity occurs more than once. + DuplicateCandidate { item_id: String, unit_id: String }, + /// Query model does not match the snapshot model. + ModelMismatch, + /// A batch contains no query vectors. + EmptyQueryBatch, + /// An exact top-k request asks for zero results. + EmptyTopK, + /// Query vector dimension does not match the snapshot. + DimensionMismatch { expected: usize, actual: usize }, + /// Query contains no authorized candidate identities. + EmptyAuthorization, + /// An authorized identity occurs more than once. + DuplicateAuthorization { item_id: String, unit_id: String }, + /// An authorized candidate does not exist in the immutable snapshot. + UnknownAuthorizedCandidate { item_id: String, unit_id: String }, + /// Packed authorization identities are truncated or have trailing bytes. + MalformedPackedAuthorization, + /// A packed authorization identity is not valid UTF-8. + NonUtf8PackedAuthorization, + /// The immutable snapshot lock was poisoned. + SnapshotLockPoisoned, +} + +impl SemanticIndexError { + /// Stable machine-readable failure code. + #[must_use] + pub const fn code(&self) -> &'static str { + match self { + Self::EmptySnapshotVersion => "empty_snapshot_version", + Self::EmptyModelIdentity => "empty_model_identity", + Self::EmptyVectorDimension => "empty_vector_dimension", + Self::EmptyCandidates => "empty_candidates", + Self::PackedVectorByteLength { .. } => "packed_vector_byte_length", + Self::NonFiniteVector { .. } => "non_finite_vector", + Self::ZeroNormVector { .. } => "zero_norm_vector", + Self::DuplicateCandidate { .. } => "duplicate_candidate", + Self::ModelMismatch => "model_mismatch", + Self::EmptyQueryBatch => "empty_query_batch", + Self::EmptyTopK => "empty_top_k", + Self::DimensionMismatch { .. } => "dimension_mismatch", + Self::EmptyAuthorization => "empty_authorization", + Self::DuplicateAuthorization { .. } => "duplicate_authorization", + Self::UnknownAuthorizedCandidate { .. } => "unknown_authorized_candidate", + Self::MalformedPackedAuthorization => "malformed_packed_authorization", + Self::NonUtf8PackedAuthorization => "non_utf8_packed_authorization", + Self::SnapshotLockPoisoned => "snapshot_lock_poisoned", + } + } +} + +impl fmt::Display for SemanticIndexError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.code()) + } +} + +impl std::error::Error for SemanticIndexError {} + +/// One immutable, exact, prevalidated semantic-unit snapshot. +#[derive(Clone, Debug)] +pub struct SemanticUnitIndex { + evidence: SemanticIndexSnapshotEvidence, + candidate_ids: Vec<(String, String)>, + candidate_lookup: HashMap>, + normalized_vectors: Vec, + #[cfg(target_os = "macos")] + absolute_normalized_vectors: Vec, + vector_norms: Vec, +} + +fn digest_bytes(domain: &[u8], values: impl IntoIterator>) -> String { + let mut hasher = Sha256::new(); + hasher.update(domain); + for value in values { + let value = value.as_ref(); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); + } + format!("sha256:{:x}", hasher.finalize()) +} + +fn update_length_prefixed(hasher: &mut Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +impl SemanticUnitIndex { + /// Build an immutable exact index from canonical big-endian binary64 bytes. + pub fn build( + snapshot_version: &str, + model_identity: &str, + vector_dimension: usize, + candidate_ids: Vec<(String, String)>, + packed_vectors: &[u8], + ) -> Result { + if snapshot_version.is_empty() { + return Err(SemanticIndexError::EmptySnapshotVersion); + } + if model_identity.is_empty() { + return Err(SemanticIndexError::EmptyModelIdentity); + } + if vector_dimension == 0 { + return Err(SemanticIndexError::EmptyVectorDimension); + } + if candidate_ids.is_empty() { + return Err(SemanticIndexError::EmptyCandidates); + } + let expected = candidate_ids + .len() + .checked_mul(vector_dimension) + .and_then(|count| count.checked_mul(size_of::())) + .unwrap_or(usize::MAX); + if packed_vectors.len() != expected { + return Err(SemanticIndexError::PackedVectorByteLength { + expected, + actual: packed_vectors.len(), + }); + } + + let mut identities = HashSet::new(); + let mut candidate_lookup: HashMap> = + HashMap::with_capacity(candidate_ids.len()); + let mut normalized_vectors = Vec::with_capacity(candidate_ids.len() * vector_dimension); + #[cfg(target_os = "macos")] + let mut absolute_normalized_vectors = + Vec::with_capacity(candidate_ids.len() * vector_dimension); + let mut vector_norms = Vec::with_capacity(candidate_ids.len()); + let vector_byte_count = vector_dimension * size_of::(); + for (index, (item_id, unit_id)) in candidate_ids.iter().enumerate() { + if !identities.insert((item_id.clone(), unit_id.clone())) { + return Err(SemanticIndexError::DuplicateCandidate { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + } + candidate_lookup + .entry(item_id.clone()) + .or_default() + .insert(unit_id.clone(), index); + let start = index * vector_byte_count; + let vector = packed_vectors[start..start + vector_byte_count] + .chunks_exact(8) + .map(|bytes| f64::from_be_bytes(bytes.try_into().expect("eight-byte chunk"))) + .collect::>(); + let vector_label = format!("candidate vector for item {item_id:?}, unit {unit_id:?}"); + if vector.iter().any(|value| !value.is_finite()) { + return Err(SemanticIndexError::NonFiniteVector { vector_label }); + } + let scale = vector.iter().map(|value| value.abs()).fold(0.0, f64::max); + if scale == 0.0 { + return Err(SemanticIndexError::ZeroNormVector { vector_label }); + } + let offset = normalized_vectors.len(); + normalized_vectors.extend(vector.iter().map(|value| value / scale)); + #[cfg(target_os = "macos")] + absolute_normalized_vectors + .extend(normalized_vectors[offset..].iter().map(|value| value.abs())); + let norm = normalized_vectors[offset..] + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + vector_norms.push(norm); + } + + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + let dimension_bytes = (vector_dimension as u64).to_be_bytes(); + let dimension_digest = digest_bytes( + b"rankweave.semantic-unit-index.dimension.v1\0", + [dimension_bytes], + ); + let mut vector_hasher = Sha256::new(); + vector_hasher.update(b"rankweave.semantic-unit-index.vectors.v1\0"); + vector_hasher.update((candidate_ids.len() as u64).to_be_bytes()); + for (index, (item_id, unit_id)) in candidate_ids.iter().enumerate() { + update_length_prefixed(&mut vector_hasher, item_id.as_bytes()); + update_length_prefixed(&mut vector_hasher, unit_id.as_bytes()); + vector_hasher.update((vector_dimension as u64).to_be_bytes()); + let start = index * vector_byte_count; + vector_hasher.update(&packed_vectors[start..start + vector_byte_count]); + } + let vectors_digest = format!("sha256:{:x}", vector_hasher.finalize()); + let snapshot_digest = digest_bytes( + b"rankweave.semantic-unit-index.snapshot.v1\0", + [ + snapshot_version.as_bytes(), + model_digest.as_bytes(), + dimension_digest.as_bytes(), + vectors_digest.as_bytes(), + ], + ); + let evidence = SemanticIndexSnapshotEvidence { + schema_version: SEMANTIC_INDEX_SNAPSHOT_SCHEMA_VERSION, + snapshot_version: snapshot_version.to_owned(), + model_digest, + dimension_digest, + vectors_digest, + snapshot_digest, + vector_dimension, + candidate_count: candidate_ids.len(), + }; + Ok(Self { + evidence, + candidate_ids, + candidate_lookup, + normalized_vectors, + #[cfg(target_os = "macos")] + absolute_normalized_vectors, + vector_norms, + }) + } + + /// Return immutable snapshot integrity evidence. + #[must_use] + pub fn evidence(&self) -> &SemanticIndexSnapshotEvidence { + &self.evidence + } + + /// Rank only the caller-authorized candidate identities with exact cosine. + pub fn rank_authorized( + &self, + model_identity: &str, + query_vector: &[f64], + authorized_candidate_ids: &[(String, String)], + ) -> Result { + let authorized_refs = authorized_candidate_ids + .iter() + .map(|(item_id, unit_id)| (item_id.as_str(), unit_id.as_str())) + .collect::>(); + self.rank_authorized_refs(model_identity, query_vector, &authorized_refs) + } + + fn rank_authorized_refs( + &self, + model_identity: &str, + query_vector: &[f64], + authorized_candidate_ids: &[(&str, &str)], + ) -> Result { + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + if model_digest != self.evidence.model_digest { + return Err(SemanticIndexError::ModelMismatch); + } + if authorized_candidate_ids.is_empty() { + return Err(SemanticIndexError::EmptyAuthorization); + } + let mut authorization_seen = HashSet::new(); + let mut authorized_indices = Vec::with_capacity(authorized_candidate_ids.len()); + for (item_id, unit_id) in authorized_candidate_ids { + if !authorization_seen.insert((item_id, unit_id)) { + return Err(SemanticIndexError::DuplicateAuthorization { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + } + let Some(index) = self + .candidate_lookup + .get(*item_id) + .and_then(|units| units.get(*unit_id)) + else { + return Err(SemanticIndexError::UnknownAuthorizedCandidate { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + }; + authorized_indices.push(*index); + } + let query = self.prepare_query(query_vector)?; + let dimension = self.evidence.vector_dimension; + let scored = authorized_indices + .par_iter() + .map(|index| { + let start = index * dimension; + let dot = query + .normalized + .iter() + .zip(&self.normalized_vectors[start..start + dimension]) + .fold(0.0, |sum, (left, right)| sum + left * right); + let score = (dot / (query.norm * self.vector_norms[*index])).clamp(0.0, 1.0); + (*index, score) + }) + .collect::>(); + let mut best_by_item = HashMap::new(); + for (index, score) in scored { + let (item_id, unit_id) = &self.candidate_ids[index]; + retain_best_unit(&mut best_by_item, item_id, unit_id, score); + } + Ok(self.finish_query_report( + &model_digest, + query_vector, + authorized_candidate_ids, + best_by_item, + )) + } + + fn rank_authorized_batch_refs( + &self, + model_identity: &str, + query_vectors: &[&[f64]], + authorized_candidate_ids: &[(&str, &str)], + ) -> Result, SemanticIndexError> { + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + if model_digest != self.evidence.model_digest { + return Err(SemanticIndexError::ModelMismatch); + } + if query_vectors.is_empty() { + return Err(SemanticIndexError::EmptyQueryBatch); + } + if authorized_candidate_ids.is_empty() { + return Err(SemanticIndexError::EmptyAuthorization); + } + let mut authorization_seen = HashSet::new(); + let mut authorized_indices = Vec::with_capacity(authorized_candidate_ids.len()); + for (item_id, unit_id) in authorized_candidate_ids { + if !authorization_seen.insert((item_id, unit_id)) { + return Err(SemanticIndexError::DuplicateAuthorization { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + } + let Some(index) = self + .candidate_lookup + .get(*item_id) + .and_then(|units| units.get(*unit_id)) + else { + return Err(SemanticIndexError::UnknownAuthorizedCandidate { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + }; + authorized_indices.push(*index); + } + + let prepared_queries = query_vectors + .iter() + .map(|query_vector| self.prepare_query(query_vector)) + .collect::, _>>()?; + let mut unique_query_indices = Vec::new(); + let mut original_to_unique = Vec::with_capacity(query_vectors.len()); + for (query_index, query_vector) in query_vectors.iter().enumerate() { + let existing = unique_query_indices.iter().position(|unique_index| { + vectors_have_identical_bits(query_vectors[*unique_index], query_vector) + }); + original_to_unique.push(existing.unwrap_or_else(|| { + unique_query_indices.push(query_index); + unique_query_indices.len() - 1 + })); + } + let unique_queries = unique_query_indices + .iter() + .map(|index| &prepared_queries[*index]) + .collect::>(); + let dimension = self.evidence.vector_dimension; + let query_count = unique_queries.len(); + let mut normalized_queries_by_coordinate = Vec::new(); + for coordinate in 0..dimension { + normalized_queries_by_coordinate.extend( + unique_queries + .iter() + .map(|query| query.normalized[coordinate]), + ); + } + let best_by_query = authorized_indices + .par_iter() + .fold( + || (vec![0.0; query_count], empty_best_maps(query_count)), + |(mut dots, mut best_by_query), index| { + dots.fill(0.0); + let start = index * dimension; + for coordinate in 0..dimension { + let candidate_value = self.normalized_vectors[start + coordinate]; + let query_start = coordinate * query_count; + for (dot, query_value) in dots.iter_mut().zip( + &normalized_queries_by_coordinate + [query_start..query_start + query_count], + ) { + *dot += query_value * candidate_value; + } + } + let (item_id, unit_id) = &self.candidate_ids[*index]; + for (query_index, query) in unique_queries.iter().enumerate() { + let score = (dots[query_index] / (query.norm * self.vector_norms[*index])) + .clamp(0.0, 1.0); + retain_best_unit(&mut best_by_query[query_index], item_id, unit_id, score); + } + (dots, best_by_query) + }, + ) + .map(|(_, best_by_query)| best_by_query) + .reduce( + || empty_best_maps(query_count), + |mut left, right| { + for (left_query, right_query) in left.iter_mut().zip(right) { + for result in right_query.into_values() { + retain_best_unit( + left_query, + &result.item_id, + &result.winning_unit_id, + result.score, + ); + } + } + left + }, + ); + + let unique_reports = unique_query_indices + .iter() + .zip(best_by_query) + .map(|(query_index, best_by_item)| { + self.finish_query_report( + &model_digest, + query_vectors[*query_index], + authorized_candidate_ids, + best_by_item, + ) + }) + .collect::>(); + Ok(original_to_unique + .into_iter() + .map(|unique_index| unique_reports[unique_index].clone()) + .collect()) + } + + #[cfg(target_os = "macos")] + fn rank_authorized_top_k_accelerate_refs( + &self, + model_identity: &str, + query_vectors: &[&[f64]], + authorized_candidate_ids: &[(&str, &str)], + top_k: usize, + ) -> Result, SemanticIndexError> { + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + if model_digest != self.evidence.model_digest { + return Err(SemanticIndexError::ModelMismatch); + } + if query_vectors.is_empty() { + return Err(SemanticIndexError::EmptyQueryBatch); + } + if authorized_candidate_ids.is_empty() { + return Err(SemanticIndexError::EmptyAuthorization); + } + let mut authorization_seen = HashSet::new(); + let mut authorized_indices = Vec::with_capacity(authorized_candidate_ids.len()); + for (item_id, unit_id) in authorized_candidate_ids { + if !authorization_seen.insert((item_id, unit_id)) { + return Err(SemanticIndexError::DuplicateAuthorization { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + } + let Some(index) = self + .candidate_lookup + .get(*item_id) + .and_then(|units| units.get(*unit_id)) + else { + return Err(SemanticIndexError::UnknownAuthorizedCandidate { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + }; + authorized_indices.push(*index); + } + let Some(roundoff) = DotRoundoffBound::new(self.evidence.vector_dimension) else { + return self.scalar_top_k_batch_refs( + model_identity, + query_vectors, + authorized_candidate_ids, + top_k, + ); + }; + let prepared_queries = query_vectors + .iter() + .map(|query| self.prepare_query(query)) + .collect::, _>>()?; + let Some((approximate_dots, approximate_absolute_dots)) = accelerate_matrix_multiply_pair( + &self.normalized_vectors, + &self.absolute_normalized_vectors, + &prepared_queries, + self.evidence.candidate_count, + self.evidence.vector_dimension, + ) else { + return self.scalar_top_k_batch_refs( + model_identity, + query_vectors, + authorized_candidate_ids, + top_k, + ); + }; + let query_count = prepared_queries.len(); + let mut reports = Vec::with_capacity(query_count); + for (query_index, query) in prepared_queries.iter().enumerate() { + let mut item_intervals: HashMap<&str, (f64, f64)> = HashMap::new(); + for index in &authorized_indices { + let (item_id, _) = &self.candidate_ids[*index]; + let offset = *index * query_count + query_index; + let interval = roundoff + .scalar_score_interval( + approximate_dots[offset], + approximate_absolute_dots[offset], + query.norm * self.vector_norms[*index], + ) + .unwrap_or((0.0, 1.0)); + item_intervals + .entry(item_id) + .and_modify(|current| { + current.0 = current.0.max(interval.0); + current.1 = current.1.max(interval.1); + }) + .or_insert(interval); + } + let ambiguity = ambiguous_items(&item_intervals, top_k); + if ambiguity.len() == item_intervals.len() { + let full = self + .rank_authorized_refs( + model_identity, + query_vectors[query_index], + authorized_candidate_ids, + ) + .expect("screening inputs were validated before scalar recomputation"); + reports.push(self.finish_top_k_report( + &model_digest, + query_vectors[query_index], + authorized_candidate_ids, + full.results, + top_k, + SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE, + )); + continue; + } + let dimension = self.evidence.vector_dimension; + let scored = authorized_indices + .par_iter() + .filter(|index| ambiguity.contains(self.candidate_ids[**index].0.as_str())) + .map(|index| { + let start = index * dimension; + let dot = query + .normalized + .iter() + .zip(&self.normalized_vectors[start..start + dimension]) + .fold(0.0, |sum, (left, right)| sum + left * right); + let score = (dot / (query.norm * self.vector_norms[*index])).clamp(0.0, 1.0); + (*index, score) + }) + .collect::>(); + let mut best_by_item = HashMap::new(); + for (index, score) in scored { + let (item_id, unit_id) = &self.candidate_ids[index]; + retain_best_unit(&mut best_by_item, item_id, unit_id, score); + } + reports.push(self.finish_top_k_report( + &model_digest, + query_vectors[query_index], + authorized_candidate_ids, + best_by_item.into_values().collect(), + top_k, + SEMANTIC_INDEX_TOP_K_ACCELERATE_EXECUTION_PROFILE, + )); + } + Ok(reports) + } + + fn prepare_query(&self, query_vector: &[f64]) -> Result { + if query_vector.len() != self.evidence.vector_dimension { + return Err(SemanticIndexError::DimensionMismatch { + expected: self.evidence.vector_dimension, + actual: query_vector.len(), + }); + } + if query_vector.iter().any(|value| !value.is_finite()) { + return Err(SemanticIndexError::NonFiniteVector { + vector_label: "query vector".to_owned(), + }); + } + let query_scale = query_vector + .iter() + .map(|value| value.abs()) + .fold(0.0, f64::max); + if query_scale == 0.0 { + return Err(SemanticIndexError::ZeroNormVector { + vector_label: "query vector".to_owned(), + }); + } + let normalized = query_vector + .iter() + .map(|value| value / query_scale) + .collect::>(); + let norm = normalized + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + Ok(PreparedQuery { normalized, norm }) + } + + fn finish_query_report( + &self, + model_digest: &str, + query_vector: &[f64], + authorized_candidate_ids: &[(&str, &str)], + best_by_item: HashMap, + ) -> SemanticIndexRankingReport { + let mut results = best_by_item.into_values().collect::>(); + results.sort_by(|left, right| { + right + .score + .total_cmp(&left.score) + .then_with(|| left.item_id.cmp(&right.item_id)) + }); + + let mut input_hasher = Sha256::new(); + input_hasher.update(b"rankweave.semantic-unit-index.query.v1\0"); + update_length_prefixed(&mut input_hasher, self.evidence.snapshot_digest.as_bytes()); + update_length_prefixed(&mut input_hasher, model_digest.as_bytes()); + input_hasher.update((query_vector.len() as u64).to_be_bytes()); + for value in query_vector { + input_hasher.update(value.to_bits().to_be_bytes()); + } + input_hasher.update((authorized_candidate_ids.len() as u64).to_be_bytes()); + for (item_id, unit_id) in authorized_candidate_ids { + update_length_prefixed(&mut input_hasher, item_id.as_bytes()); + update_length_prefixed(&mut input_hasher, unit_id.as_bytes()); + } + let ordered_input_digest = format!("sha256:{:x}", input_hasher.finalize()); + + let mut output_hasher = Sha256::new(); + output_hasher.update(b"rankweave.semantic-unit-index.result.v1\0"); + output_hasher.update((results.len() as u64).to_be_bytes()); + for result in &results { + update_length_prefixed(&mut output_hasher, result.item_id.as_bytes()); + update_length_prefixed(&mut output_hasher, result.winning_unit_id.as_bytes()); + output_hasher.update(result.score.to_bits().to_be_bytes()); + } + let output_digest = format!("sha256:{:x}", output_hasher.finalize()); + SemanticIndexRankingReport { + snapshot: self.evidence.clone(), + algorithm_version: SEMANTIC_UNIT_COSINE_ALGORITHM_VERSION, + execution_profile: SEMANTIC_INDEX_CPU_EXECUTION_PROFILE, + worker_count: rayon::current_num_threads(), + ordered_input_digest, + output_digest, + results, + } + } + + fn finish_top_k_report( + &self, + model_digest: &str, + query_vector: &[f64], + authorized_candidate_ids: &[(&str, &str)], + mut results: Vec, + top_k: usize, + execution_profile: &'static str, + ) -> SemanticIndexRankingReport { + results.sort_by(|left, right| { + right + .score + .total_cmp(&left.score) + .then_with(|| left.item_id.cmp(&right.item_id)) + }); + results.truncate(top_k); + + let mut input_hasher = Sha256::new(); + input_hasher.update(b"rankweave.semantic-unit-index.top-k-query.v1\0"); + update_length_prefixed(&mut input_hasher, self.evidence.snapshot_digest.as_bytes()); + update_length_prefixed(&mut input_hasher, model_digest.as_bytes()); + input_hasher.update((top_k as u64).to_be_bytes()); + input_hasher.update((query_vector.len() as u64).to_be_bytes()); + for value in query_vector { + input_hasher.update(value.to_bits().to_be_bytes()); + } + input_hasher.update((authorized_candidate_ids.len() as u64).to_be_bytes()); + for (item_id, unit_id) in authorized_candidate_ids { + update_length_prefixed(&mut input_hasher, item_id.as_bytes()); + update_length_prefixed(&mut input_hasher, unit_id.as_bytes()); + } + let ordered_input_digest = format!("sha256:{:x}", input_hasher.finalize()); + + let mut output_hasher = Sha256::new(); + output_hasher.update(b"rankweave.semantic-unit-index.top-k-result.v1\0"); + output_hasher.update((top_k as u64).to_be_bytes()); + output_hasher.update((results.len() as u64).to_be_bytes()); + for result in &results { + update_length_prefixed(&mut output_hasher, result.item_id.as_bytes()); + update_length_prefixed(&mut output_hasher, result.winning_unit_id.as_bytes()); + output_hasher.update(result.score.to_bits().to_be_bytes()); + } + let output_digest = format!("sha256:{:x}", output_hasher.finalize()); + SemanticIndexRankingReport { + snapshot: self.evidence.clone(), + algorithm_version: SEMANTIC_UNIT_COSINE_ALGORITHM_VERSION, + execution_profile, + worker_count: rayon::current_num_threads(), + ordered_input_digest, + output_digest, + results, + } + } + + fn scalar_top_k_batch_refs( + &self, + model_identity: &str, + query_vectors: &[&[f64]], + authorized_candidate_ids: &[(&str, &str)], + top_k: usize, + ) -> Result, SemanticIndexError> { + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + self.rank_authorized_batch_refs(model_identity, query_vectors, authorized_candidate_ids) + .map(|reports| { + reports + .into_iter() + .zip(query_vectors) + .map(|(report, query)| { + self.finish_top_k_report( + &model_digest, + query, + authorized_candidate_ids, + report.results, + top_k, + SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE, + ) + }) + .collect() + }) + } + + /// Rank ordered queries against one identical canonical packed authorization. + pub fn rank_authorized_batch_packed( + &self, + model_identity: &str, + query_vectors: &[Vec], + packed_authorization: &[u8], + ) -> Result, SemanticIndexError> { + let authorized = parse_packed_authorization(packed_authorization)?; + let query_refs = query_vectors.iter().map(Vec::as_slice).collect::>(); + self.rank_authorized_batch_refs(model_identity, &query_refs, &authorized) + } + + /// Return exact top-k reports for ordered queries and one packed authorization. + /// + /// Apple Accelerate may screen only candidates whose binary64 forward-error + /// intervals prove that they cannot cross the item-level kth boundary. Every + /// ambiguous item's units are recomputed in coordinate order. Other platforms + /// and operand sets outside the no-underflow proof fall back to the exact scalar + /// batch while retaining the separately versioned top-k digests. + pub fn rank_authorized_top_k_batch_packed( + &self, + model_identity: &str, + query_vectors: &[Vec], + packed_authorization: &[u8], + top_k: usize, + ) -> Result, SemanticIndexError> { + if top_k == 0 { + return Err(SemanticIndexError::EmptyTopK); + } + let authorized = parse_packed_authorization(packed_authorization)?; + let query_refs = query_vectors.iter().map(Vec::as_slice).collect::>(); + #[cfg(target_os = "macos")] + let result = self.rank_authorized_top_k_accelerate_refs( + model_identity, + &query_refs, + &authorized, + top_k, + ); + #[cfg(not(target_os = "macos"))] + let result = self.scalar_top_k_batch_refs(model_identity, &query_refs, &authorized, top_k); + result + } + + /* + The single-query packed API remains the stable compatibility surface; + parsing delegates to the same borrowed authorization representation as + the batch operation. + */ + fn rank_authorized_packed_inner( + &self, + model_identity: &str, + query_vector: &[f64], + packed_authorization: &[u8], + ) -> Result { + let authorized = parse_packed_authorization(packed_authorization)?; + self.rank_authorized_refs(model_identity, query_vector, &authorized) + } + + /// Rank a canonical packed ordered authorization set without Python rows. + pub fn rank_authorized_packed( + &self, + model_identity: &str, + query_vector: &[f64], + packed_authorization: &[u8], + ) -> Result { + self.rank_authorized_packed_inner(model_identity, query_vector, packed_authorization) + } + + /// Exercise exact scoring for one real packed authorization scope. + /// + /// The query is the first authorized candidate vector already owned by + /// this immutable snapshot. This keeps vector arithmetic in Rust while + /// forcing authorization parsing, the complete exact matrix traversal, + /// stable per-item reduction, and report digest construction before a + /// caller advertises readiness. Readiness callers discard the report. + pub fn preflight_authorized_packed( + &self, + model_identity: &str, + packed_authorization: &[u8], + ) -> Result { + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + if model_digest != self.evidence.model_digest { + return Err(SemanticIndexError::ModelMismatch); + } + let authorized = parse_packed_authorization(packed_authorization)?; + let Some((item_id, unit_id)) = authorized.first() else { + return Err(SemanticIndexError::EmptyAuthorization); + }; + let Some(index) = self + .candidate_lookup + .get(*item_id) + .and_then(|units| units.get(*unit_id)) + else { + return Err(SemanticIndexError::UnknownAuthorizedCandidate { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + }; + let start = index * self.evidence.vector_dimension; + let query = self.normalized_vectors[start..start + self.evidence.vector_dimension].to_vec(); + self.rank_authorized_refs(model_identity, &query, &authorized) + } + + /// Exercise the exact top-k profile for one real packed authorization scope. + pub fn preflight_authorized_top_k_packed( + &self, + model_identity: &str, + packed_authorization: &[u8], + top_k: usize, + ) -> Result { + if top_k == 0 { + return Err(SemanticIndexError::EmptyTopK); + } + let model_digest = digest_bytes( + b"rankweave.semantic-unit-index.model.v1\0", + [model_identity.as_bytes()], + ); + if model_digest != self.evidence.model_digest { + return Err(SemanticIndexError::ModelMismatch); + } + let authorized = parse_packed_authorization(packed_authorization)?; + let Some((item_id, unit_id)) = authorized.first() else { + return Err(SemanticIndexError::EmptyAuthorization); + }; + let Some(index) = self + .candidate_lookup + .get(*item_id) + .and_then(|units| units.get(*unit_id)) + else { + return Err(SemanticIndexError::UnknownAuthorizedCandidate { + item_id: (*item_id).to_owned(), + unit_id: (*unit_id).to_owned(), + }); + }; + let start = index * self.evidence.vector_dimension; + let query = self.normalized_vectors[start..start + self.evidence.vector_dimension].to_vec(); + self.rank_authorized_top_k_batch_packed( + model_identity, + &[query], + packed_authorization, + top_k, + ) + .map(|mut reports| reports.remove(0)) + } +} + +struct PreparedQuery { + normalized: Vec, + norm: f64, +} + +#[cfg(any(target_os = "macos", test))] +#[derive(Clone, Copy, Debug)] +struct DotRoundoffBound { + gamma: f64, + underflow_allowance: f64, +} + +#[cfg(any(target_os = "macos", test))] +impl DotRoundoffBound { + fn new(term_count: usize) -> Option { + let rounded_terms = term_count as f64 * f64::EPSILON / 2.0; + if !rounded_terms.is_finite() || rounded_terms >= 1.0 { + return None; + } + let gamma = rounded_terms / (1.0 - rounded_terms); + // The usual relative model excludes underflow. Conservatively charge + // one minimum-normal absolute error for every multiply and add, then + // amplify the accumulated errors by the same standard denominator. + let underflow_operations = 2.0 * term_count as f64; + let underflow_allowance = underflow_operations * f64::MIN_POSITIVE / (1.0 - rounded_terms); + Some(Self { + gamma, + underflow_allowance, + }) + } + + fn scalar_score_interval( + self, + approximate_dot: f64, + approximate_absolute_dot: f64, + norm_product: f64, + ) -> Option<(f64, f64)> { + if !approximate_dot.is_finite() + || !approximate_absolute_dot.is_finite() + || approximate_absolute_dot < 0.0 + || !norm_product.is_finite() + || norm_product <= 0.0 + { + return None; + } + // The second GEMM estimates |x|^T|y|. Its own forward bound gives an + // upper bound on the real absolute dot, which then bounds both the + // BLAS dot and the required coordinate-ordered scalar dot. + let absolute_dot_upper = + (approximate_absolute_dot + self.underflow_allowance) / (1.0 - self.gamma); + let dot_difference = 2.0 * (self.gamma * absolute_dot_upper + self.underflow_allowance); + if !absolute_dot_upper.is_finite() || !dot_difference.is_finite() { + return None; + } + // Division is monotone for the positive norm product. Widen each + // endpoint by one representable value to contain its rounded division. + let lower = ((approximate_dot - dot_difference) / norm_product) + .next_down() + .clamp(0.0, 1.0); + let upper = ((approximate_dot + dot_difference) / norm_product) + .next_up() + .clamp(0.0, 1.0); + Some((lower, upper)) + } +} + +#[cfg(any(target_os = "macos", test))] +fn ambiguous_items<'a>( + item_intervals: &HashMap<&'a str, (f64, f64)>, + top_k: usize, +) -> HashSet<&'a str> { + if top_k >= item_intervals.len() { + return item_intervals.keys().copied().collect(); + } + let mut lower_bounds = item_intervals + .values() + .map(|interval| interval.0) + .collect::>(); + lower_bounds.sort_by(|left, right| right.total_cmp(left)); + let kth_lower = lower_bounds[top_k - 1]; + item_intervals + .iter() + .filter_map(|(item_id, interval)| (interval.1 >= kth_lower).then_some(*item_id)) + .collect() +} + +#[cfg(target_os = "macos")] +#[link(name = "Accelerate", kind = "framework")] +unsafe extern "C" { + fn cblas_dgemm( + order: i32, + transpose_a: i32, + transpose_b: i32, + rows: i32, + columns: i32, + shared: i32, + alpha: f64, + left: *const f64, + left_stride: i32, + right: *const f64, + right_stride: i32, + beta: f64, + output: *mut f64, + output_stride: i32, + ); +} + +#[cfg(target_os = "macos")] +fn accelerate_matrix_multiply_pair( + matrix: &[f64], + absolute_matrix: &[f64], + queries: &[PreparedQuery], + candidate_count: usize, + dimension: usize, +) -> Option<(Vec, Vec)> { + let (rows, columns, shared) = accelerate_dimensions(candidate_count, queries.len(), dimension)?; + let mut queries_by_coordinate = Vec::with_capacity(dimension * queries.len()); + for coordinate in 0..dimension { + queries_by_coordinate.extend(queries.iter().map(|query| query.normalized[coordinate])); + } + let absolute_queries_by_coordinate = queries_by_coordinate + .iter() + .map(|value| value.abs()) + .collect::>(); + let mut output = vec![0.0; candidate_count * queries.len()]; + let mut absolute_output = vec![0.0; candidate_count * queries.len()]; + // SAFETY: every pointer addresses a contiguous allocation sized for the + // row-major dimensions and leading strides passed to Accelerate. The call + // is synchronous, and the immutable inputs outlive it. + unsafe { + cblas_dgemm( + 101, + 111, + 111, + rows, + columns, + shared, + 1.0, + matrix.as_ptr(), + shared, + queries_by_coordinate.as_ptr(), + columns, + 0.0, + output.as_mut_ptr(), + columns, + ); + cblas_dgemm( + 101, + 111, + 111, + rows, + columns, + shared, + 1.0, + absolute_matrix.as_ptr(), + shared, + absolute_queries_by_coordinate.as_ptr(), + columns, + 0.0, + absolute_output.as_mut_ptr(), + columns, + ); + } + Some((output, absolute_output)) +} + +#[cfg(target_os = "macos")] +fn accelerate_dimensions(rows: usize, columns: usize, shared: usize) -> Option<(i32, i32, i32)> { + Some(( + i32::try_from(rows).ok()?, + i32::try_from(columns).ok()?, + i32::try_from(shared).ok()?, + )) +} + +fn vectors_have_identical_bits(left: &[f64], right: &[f64]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| left.to_bits() == right.to_bits()) +} + +fn empty_best_maps(query_count: usize) -> Vec> { + (0..query_count).map(|_| HashMap::new()).collect() +} + +fn retain_best_unit( + best_by_item: &mut HashMap, + item_id: &str, + unit_id: &str, + score: f64, +) { + if let Some(current) = best_by_item.get_mut(item_id) { + if score > current.score + || (score == current.score && unit_id < current.winning_unit_id.as_str()) + { + current.winning_unit_id.clear(); + current.winning_unit_id.push_str(unit_id); + current.score = score; + } + return; + } + best_by_item.insert( + item_id.to_owned(), + SemanticUnitRank { + item_id: item_id.to_owned(), + winning_unit_id: unit_id.to_owned(), + score, + }, + ); +} + +fn parse_packed_authorization( + packed_authorization: &[u8], +) -> Result, SemanticIndexError> { + let mut cursor = 0_usize; + let count = read_packed_u64(packed_authorization, &mut cursor)?; + if count > ((packed_authorization.len() - cursor) / 16) as u64 { + return Err(SemanticIndexError::MalformedPackedAuthorization); + } + let count = count as usize; + let mut authorized = Vec::with_capacity(count); + for _ in 0..count { + let item = read_packed_text(packed_authorization, &mut cursor)?; + let unit = read_packed_text(packed_authorization, &mut cursor)?; + authorized.push((item, unit)); + } + if cursor != packed_authorization.len() { + return Err(SemanticIndexError::MalformedPackedAuthorization); + } + Ok(authorized) +} + +fn read_packed_u64(bytes: &[u8], cursor: &mut usize) -> Result { + let end = *cursor + 8; + let value = bytes + .get(*cursor..end) + .ok_or(SemanticIndexError::MalformedPackedAuthorization)?; + *cursor = end; + Ok(u64::from_be_bytes( + value.try_into().expect("eight-byte packed integer"), + )) +} + +fn read_packed_text<'a>( + bytes: &'a [u8], + cursor: &mut usize, +) -> Result<&'a str, SemanticIndexError> { + let length = read_packed_u64(bytes, cursor)?; + if length > (bytes.len() - *cursor) as u64 { + return Err(SemanticIndexError::MalformedPackedAuthorization); + } + let length = length as usize; + let end = *cursor + length; + let value = &bytes[*cursor..end]; + *cursor = end; + std::str::from_utf8(value).map_err(|_| SemanticIndexError::NonUtf8PackedAuthorization) +} + +/// Atomically replace an immutable exact index only after successful validation. +pub struct SemanticUnitIndexHandle { + current: RwLock>, +} + +impl SemanticUnitIndexHandle { + /// Create a handle from one fully validated snapshot. + #[must_use] + pub fn new(index: SemanticUnitIndex) -> Self { + Self { + current: RwLock::new(Arc::new(index)), + } + } + + /// Replace the current snapshot atomically after the caller builds it fully. + pub fn replace(&self, index: SemanticUnitIndex) -> Result<(), SemanticIndexError> { + let mut current = self + .current + .write() + .map_err(|_| SemanticIndexError::SnapshotLockPoisoned)?; + *current = Arc::new(index); + Ok(()) + } + + /// Acquire one immutable snapshot for a complete query. + pub fn snapshot(&self) -> Result, SemanticIndexError> { + self.current + .read() + .map(|current| Arc::clone(¤t)) + .map_err(|_| SemanticIndexError::SnapshotLockPoisoned) + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + use std::thread; + + use super::{ + DotRoundoffBound, SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE, SemanticIndexError, + SemanticUnitIndex, SemanticUnitIndexHandle, ambiguous_items, + }; + #[cfg(target_os = "macos")] + use super::{SEMANTIC_INDEX_TOP_K_ACCELERATE_EXECUTION_PROFILE, accelerate_dimensions}; + use crate::{SemanticUnitCandidate, rank_semantic_units}; + use rayon::ThreadPoolBuilder; + + fn packed(vectors: &[&[f64]]) -> Vec { + vectors + .iter() + .flat_map(|vector| vector.iter()) + .flat_map(|value| value.to_be_bytes()) + .collect() + } + + fn packed_authorization(identities: &[(String, String)]) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&(identities.len() as u64).to_be_bytes()); + for (item_id, unit_id) in identities { + bytes.extend_from_slice(&(item_id.len() as u64).to_be_bytes()); + bytes.extend_from_slice(item_id.as_bytes()); + bytes.extend_from_slice(&(unit_id.len() as u64).to_be_bytes()); + bytes.extend_from_slice(unit_id.as_bytes()); + } + bytes + } + + fn index(version: &str) -> SemanticUnitIndex { + SemanticUnitIndex::build( + version, + "model-v1", + 2, + vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ], + &packed(&[&[1.0, 0.0], &[1.0, 0.0], &[0.0, 1.0]]), + ) + .unwrap() + } + + #[test] + fn snapshot_evidence_and_authorized_results_are_exact() { + let index = index("snapshot-v1"); + let authorization = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let report = index + .rank_authorized("model-v1", &[1.0, 0.0], &authorization) + .unwrap(); + + assert_eq!(report.snapshot.snapshot_version, "snapshot-v1"); + assert_eq!(report.snapshot.vector_dimension, 2); + assert_eq!(report.snapshot.candidate_count, 3); + assert!(report.snapshot.snapshot_digest.starts_with("sha256:")); + assert!(report.ordered_input_digest.starts_with("sha256:")); + assert!(report.output_digest.starts_with("sha256:")); + assert_eq!(report.results.len(), 2); + assert_eq!(report.results[0].item_id, "item-a"); + assert_eq!(report.results[0].winning_unit_id, "unit-z"); + assert_eq!(report.results[1].item_id, "item-b"); + } + + #[test] + fn worker_count_does_not_change_exact_result_or_digests() { + let index = index("snapshot-v1"); + let authorization = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let run = |worker_count| { + ThreadPoolBuilder::new() + .num_threads(worker_count) + .build() + .unwrap() + .install(|| { + index + .rank_authorized("model-v1", &[1.0, 0.0], &authorization) + .unwrap() + }) + }; + let one_worker = run(1); + let four_workers = run(4); + + assert_eq!(one_worker.results, four_workers.results); + assert_eq!( + one_worker.ordered_input_digest, + four_workers.ordered_input_digest + ); + assert_eq!(one_worker.output_digest, four_workers.output_digest); + assert_eq!(one_worker.worker_count, 1); + assert_eq!(four_workers.worker_count, 4); + } + + #[test] + fn packed_authorization_preserves_exact_results_and_digests() { + let authorization = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let index = index("snapshot-v1"); + let rows = index + .rank_authorized("model-v1", &[1.0, 0.0], &authorization) + .unwrap(); + let packed = index + .rank_authorized_packed( + "model-v1", + &[1.0, 0.0], + &packed_authorization(&authorization), + ) + .unwrap(); + + assert_eq!(packed.results, rows.results); + assert_eq!(packed.ordered_input_digest, rows.ordered_input_digest); + assert_eq!(packed.output_digest, rows.output_digest); + } + + #[test] + fn packed_preflight_exercises_one_real_authorization_scope() { + let authorization = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let index = index("snapshot-v1"); + + let report = index + .preflight_authorized_packed("model-v1", &packed_authorization(&authorization)) + .unwrap(); + + assert_eq!(report.snapshot, *index.evidence()); + assert_eq!(report.results.len(), 2); + assert_eq!(report.results[0].item_id, "item-a"); + assert_eq!(report.results[1].item_id, "item-b"); + assert!(report.ordered_input_digest.starts_with("sha256:")); + assert!(report.output_digest.starts_with("sha256:")); + + let top_k = index + .preflight_authorized_top_k_packed("model-v1", &packed_authorization(&authorization), 1) + .unwrap(); + assert_eq!(top_k.results.len(), 1); + assert_eq!(top_k.results[0], report.results[0]); + } + + #[test] + fn packed_preflight_rejects_empty_and_unknown_scopes() { + let index = index("snapshot-v1"); + let cases = [ + index.preflight_authorized_packed( + "other-model", + &packed_authorization(&[("missing".to_owned(), "unit".to_owned())]), + ), + index.preflight_authorized_packed("model-v1", b"short"), + index.preflight_authorized_packed("model-v1", &packed_authorization(&[])), + index.preflight_authorized_packed( + "model-v1", + &packed_authorization(&[("missing".to_owned(), "unit".to_owned())]), + ), + ]; + + assert_eq!(cases[0].as_ref().unwrap_err().code(), "model_mismatch"); + assert_eq!( + cases[1].as_ref().unwrap_err().code(), + "malformed_packed_authorization" + ); + assert_eq!(cases[2].as_ref().unwrap_err().code(), "empty_authorization"); + assert_eq!( + cases[3].as_ref().unwrap_err().code(), + "unknown_authorized_candidate" + ); + + let top_k_cases = [ + index.preflight_authorized_top_k_packed("model-v1", b"short", 1), + index.preflight_authorized_top_k_packed("model-v1", &packed_authorization(&[]), 1), + index.preflight_authorized_top_k_packed( + "model-v1", + &packed_authorization(&[("missing".to_owned(), "unit".to_owned())]), + 1, + ), + index.preflight_authorized_top_k_packed( + "other-model", + &packed_authorization(&[("item-a".to_owned(), "unit-a".to_owned())]), + 1, + ), + index.preflight_authorized_top_k_packed( + "model-v1", + &packed_authorization(&[("item-a".to_owned(), "unit-a".to_owned())]), + 0, + ), + ]; + for (result, code) in top_k_cases.into_iter().zip([ + "malformed_packed_authorization", + "empty_authorization", + "unknown_authorized_candidate", + "model_mismatch", + "empty_top_k", + ]) { + assert_eq!(result.unwrap_err().code(), code); + } + } + + #[test] + fn packed_batch_matches_every_independent_query_and_digest() { + let authorization = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let packed_authorization = packed_authorization(&authorization); + let queries = vec![ + vec![1.0, 0.0], + vec![0.0, 1.0], + vec![1.0, 1.0], + vec![1.0, 0.0], + ]; + let index = index("snapshot-v1"); + + let batch = index + .rank_authorized_batch_packed("model-v1", &queries, &packed_authorization) + .unwrap(); + let independent = queries + .iter() + .map(|query| { + index + .rank_authorized_packed("model-v1", query, &packed_authorization) + .unwrap() + }) + .collect::>(); + + assert_eq!(batch, independent); + } + + #[test] + fn interval_screened_top_k_matches_coordinate_ordered_scalar() { + let ids = vec![ + ("item-a".to_owned(), "unit-low".to_owned()), + ("item-a".to_owned(), "unit-high".to_owned()), + ("item-b".to_owned(), "unit".to_owned()), + ("item-c".to_owned(), "unit".to_owned()), + ]; + let index = SemanticUnitIndex::build( + "snapshot-v1", + "model-v1", + 2, + ids.clone(), + &packed(&[&[0.0, 1.0], &[1.0, 0.0], &[0.9, 0.1], &[0.0, 1.0]]), + ) + .unwrap(); + let authorization = packed_authorization(&ids); + let queries = vec![vec![1.0, 0.0], vec![0.8, 0.2]]; + let top_k = index + .rank_authorized_top_k_batch_packed("model-v1", &queries, &authorization, 2) + .unwrap(); + let full = index + .rank_authorized_batch_packed("model-v1", &queries, &authorization) + .unwrap(); + + for (top_k_report, full_report) in top_k.iter().zip(full) { + assert_eq!(top_k_report.results, full_report.results[..2]); + assert_ne!( + top_k_report.ordered_input_digest, + full_report.ordered_input_digest + ); + assert_ne!(top_k_report.output_digest, full_report.output_digest); + } + #[cfg(target_os = "macos")] + assert!(top_k.iter().any(|report| { + report.execution_profile == SEMANTIC_INDEX_TOP_K_ACCELERATE_EXECUTION_PROFILE + })); + } + + #[test] + fn all_ambiguous_top_k_recomputes_complete_scalar_set() { + let ids = (0..8) + .map(|index| (format!("item-{index}"), "unit".to_owned())) + .collect::>(); + let vectors = (0..8).map(|_| &[1.0, 0.0][..]).collect::>(); + let index = + SemanticUnitIndex::build("snapshot-v1", "model-v1", 2, ids.clone(), &packed(&vectors)) + .unwrap(); + let top_k = index + .rank_authorized_top_k_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&ids), + 4, + ) + .unwrap(); + let full = index + .rank_authorized("model-v1", &[1.0, 0.0], &ids) + .unwrap(); + + assert_eq!(top_k[0].results, full.results[..4]); + assert_eq!( + top_k[0].execution_profile, + SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE + ); + } + + #[test] + fn mixed_sign_interval_screen_matches_scalar() { + let ids = vec![ + ("item-a".to_owned(), "unit".to_owned()), + ("item-b".to_owned(), "unit".to_owned()), + ("item-c".to_owned(), "unit".to_owned()), + ("item-d".to_owned(), "unit".to_owned()), + ]; + let index = SemanticUnitIndex::build( + "snapshot-v1", + "model-v1", + 3, + ids.clone(), + &packed(&[ + &[1.0, -0.5, 0.25], + &[0.9, -0.45, 0.2], + &[-0.5, 1.0, 0.25], + &[-1.0, -0.5, 0.1], + ]), + ) + .unwrap(); + let authorization = packed_authorization(&ids); + let queries = vec![vec![1.0, -0.25, 0.5], vec![-0.4, 1.0, 0.2]]; + let top_k = index + .rank_authorized_top_k_batch_packed("model-v1", &queries, &authorization, 2) + .unwrap(); + let full = index + .rank_authorized_batch_packed("model-v1", &queries, &authorization) + .unwrap(); + + for (screened, scalar) in top_k.iter().zip(full) { + assert_eq!(screened.results, scalar.results[..2]); + } + #[cfg(target_os = "macos")] + assert!(top_k.iter().any(|report| { + report.execution_profile == SEMANTIC_INDEX_TOP_K_ACCELERATE_EXECUTION_PROFILE + })); + } + + #[test] + fn interval_bound_contains_underflow_and_cancellation_cases() { + let bound = DotRoundoffBound::new(3).unwrap(); + let cancellation = bound.scalar_score_interval(0.0, 3.0, 1.0).unwrap(); + assert!(cancellation.0 <= 0.0); + assert!(cancellation.1 > 0.0); + + let underflow = bound + .scalar_score_interval(0.0, f64::MIN_POSITIVE, 1.0) + .unwrap(); + assert!(underflow.0 <= 0.0); + assert!(underflow.1 >= f64::MIN_POSITIVE); + + assert!(DotRoundoffBound::new(usize::MAX).is_none()); + for invalid in [ + bound.scalar_score_interval(f64::NAN, 1.0, 1.0), + bound.scalar_score_interval(1.0, f64::NAN, 1.0), + bound.scalar_score_interval(1.0, -1.0, 1.0), + bound.scalar_score_interval(1.0, 1.0, f64::INFINITY), + bound.scalar_score_interval(1.0, 1.0, 0.0), + bound.scalar_score_interval(f64::MAX, f64::MAX, f64::MIN_POSITIVE), + ] { + assert!(invalid.is_none()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn interval_top_k_validation_and_scalar_fallback_are_explicit() { + let exact_index = index("snapshot-v1"); + let known = ("item-a".to_owned(), "unit-a".to_owned()); + let packed_known = packed_authorization(std::slice::from_ref(&known)); + let cases = [ + exact_index.rank_authorized_top_k_batch_packed( + "other-model", + &[vec![1.0, 0.0]], + &packed_known, + 1, + ), + exact_index.rank_authorized_top_k_batch_packed("model-v1", &[], &packed_known, 1), + exact_index.rank_authorized_top_k_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&[]), + 1, + ), + exact_index.rank_authorized_top_k_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&[known.clone(), known.clone()]), + 1, + ), + exact_index.rank_authorized_top_k_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&[("missing".to_owned(), "unit".to_owned())]), + 1, + ), + exact_index.rank_authorized_top_k_batch_packed( + "model-v1", + &[vec![1.0]], + &packed_known, + 1, + ), + exact_index.rank_authorized_top_k_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + b"short", + 1, + ), + ]; + for (result, code) in cases.into_iter().zip([ + "model_mismatch", + "empty_query_batch", + "empty_authorization", + "duplicate_authorization", + "unknown_authorized_candidate", + "dimension_mismatch", + "malformed_packed_authorization", + ]) { + assert_eq!(result.unwrap_err().code(), code); + } + + let authorization = [("item-a", "unit-a")]; + let scalar = exact_index + .scalar_top_k_batch_refs("model-v1", &[&[0.0, 1.0]], &authorization, 1) + .unwrap(); + assert_eq!(scalar[0].results.len(), 1); + assert_eq!( + scalar[0].execution_profile, + SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE + ); + + let mut oversized_dimension = index("snapshot-v1"); + oversized_dimension.evidence.vector_dimension = usize::MAX; + assert_eq!( + oversized_dimension + .rank_authorized_top_k_accelerate_refs( + "model-v1", + &[&[0.0, 1.0]], + &authorization, + 1, + ) + .unwrap_err() + .code(), + "dimension_mismatch" + ); + + let mut oversized_candidate_count = index("snapshot-v1"); + oversized_candidate_count.evidence.candidate_count = i32::MAX as usize + 1; + let fallback = oversized_candidate_count + .rank_authorized_top_k_accelerate_refs("model-v1", &[&[0.0, 1.0]], &authorization, 1) + .unwrap(); + assert_eq!( + fallback[0].execution_profile, + SEMANTIC_INDEX_TOP_K_CPU_EXECUTION_PROFILE + ); + + let too_large = i32::MAX as usize + 1; + assert_eq!(accelerate_dimensions(1, 1, 1), Some((1, 1, 1))); + assert!(accelerate_dimensions(too_large, 1, 1).is_none()); + assert!(accelerate_dimensions(1, too_large, 1).is_none()); + assert!(accelerate_dimensions(1, 1, too_large).is_none()); + } + + #[test] + fn item_interval_screen_keeps_near_ties_and_pools_units_first() { + let intervals = HashMap::from([ + ("item-a", (0.90, 0.91)), + ("item-b", (0.89, 0.905)), + ("item-c", (0.10, 0.20)), + ]); + assert_eq!( + ambiguous_items(&intervals, 1), + HashSet::from(["item-a", "item-b"]) + ); + assert_eq!( + ambiguous_items(&intervals, 3), + intervals.keys().copied().collect() + ); + } + + #[test] + fn exact_top_k_rejects_zero_k() { + let index = index("snapshot-v1"); + assert_eq!( + index + .rank_authorized_top_k_batch_packed("model-v1", &[vec![1.0, 0.0]], b"", 0) + .unwrap_err() + .code(), + "empty_top_k" + ); + } + + #[test] + fn packed_batch_validation_failures_are_explicit() { + let index = index("snapshot-v1"); + let known = ("item-a".to_owned(), "unit-a".to_owned()); + let cases = [ + index.rank_authorized_batch_packed( + "other-model", + &[vec![1.0, 0.0]], + &packed_authorization(std::slice::from_ref(&known)), + ), + index.rank_authorized_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&[]), + ), + index.rank_authorized_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&[known.clone(), known.clone()]), + ), + index.rank_authorized_batch_packed( + "model-v1", + &[vec![1.0, 0.0]], + &packed_authorization(&[("missing".to_owned(), "unit".to_owned())]), + ), + index.rank_authorized_batch_packed( + "model-v1", + &[vec![1.0]], + &packed_authorization(std::slice::from_ref(&known)), + ), + ]; + let codes = [ + "model_mismatch", + "empty_authorization", + "duplicate_authorization", + "unknown_authorized_candidate", + "dimension_mismatch", + ]; + for (result, code) in cases.into_iter().zip(codes) { + assert_eq!(result.unwrap_err().code(), code); + } + } + + #[test] + fn malformed_packed_authorization_fails_closed() { + let index = index("snapshot-v1"); + assert_eq!( + index + .rank_authorized_batch_packed("model-v1", &[vec![1.0, 0.0]], b"short") + .unwrap_err() + .code(), + "malformed_packed_authorization" + ); + let cases = [ + (&b"short"[..], "malformed_packed_authorization"), + ( + &[0, 0, 0, 0, 0, 0, 0, 1][..], + "malformed_packed_authorization", + ), + ( + &[ + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 9, b'a', b'a', b'a', b'a', b'a', + b'a', b'a', b'a', + ][..], + "malformed_packed_authorization", + ), + ( + &[ + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, b'a', b'a', 0, 0, 0, 0, 0, 0, 0, + ][..], + "malformed_packed_authorization", + ), + ( + &[ + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, + ][..], + "non_utf8_packed_authorization", + ), + ]; + for (bytes, code) in cases { + assert_eq!( + index + .rank_authorized_packed("model-v1", &[1.0, 0.0], bytes) + .unwrap_err() + .code(), + code + ); + } + let mut trailing = packed_authorization(&[("item-a".to_owned(), "unit-a".to_owned())]); + trailing.push(0); + assert_eq!( + index + .rank_authorized_packed("model-v1", &[1.0, 0.0], &trailing) + .unwrap_err() + .code(), + "malformed_packed_authorization" + ); + } + + #[test] + fn authorization_never_returns_an_unlisted_candidate() { + let report = index("snapshot-v1") + .rank_authorized( + "model-v1", + &[1.0, 0.0], + &[("item-a".to_owned(), "unit-a".to_owned())], + ) + .unwrap(); + + assert_eq!(report.results.len(), 1); + assert_eq!(report.results[0].item_id, "item-a"); + assert_eq!(report.results[0].winning_unit_id, "unit-a"); + } + + #[test] + fn indexed_scores_equal_the_existing_exact_cosine_contract() { + let ids = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let vectors = [&[0.25, -0.5, 0.75][..], &[0.3, 0.2, 0.1], &[-0.4, 0.7, 0.2]]; + let query = [0.5, -0.25, 0.75]; + let indexed = + SemanticUnitIndex::build("snapshot-v1", "model-v1", 3, ids.clone(), &packed(&vectors)) + .unwrap() + .rank_authorized("model-v1", &query, &ids) + .unwrap(); + let scalar_candidates = ids + .iter() + .zip(vectors) + .map(|((item_id, unit_id), vector)| SemanticUnitCandidate { + item_id: item_id.clone(), + unit_id: unit_id.clone(), + vector: vector.to_vec(), + }) + .collect::>(); + let scalar = rank_semantic_units(&query, &scalar_candidates).unwrap(); + + assert_eq!(indexed.results, scalar.results); + } + + #[test] + fn exact_ties_choose_the_lexicographically_first_unit() { + let ids = vec![ + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let report = SemanticUnitIndex::build( + "snapshot-v1", + "model-v1", + 2, + ids.clone(), + &packed(&[&[1.0, 0.0], &[1.0, 0.0]]), + ) + .unwrap() + .rank_authorized("model-v1", &[1.0, 0.0], &ids) + .unwrap(); + + assert_eq!(report.results[0].winning_unit_id, "unit-a"); + } + + #[test] + fn cold_rebuild_preserves_snapshot_and_result_digests() { + let authorization = vec![ + ("item-b".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-z".to_owned()), + ("item-a".to_owned(), "unit-a".to_owned()), + ]; + let first = index("snapshot-v1"); + let restarted = index("snapshot-v1"); + let first_report = first + .rank_authorized("model-v1", &[1.0, 0.0], &authorization) + .unwrap(); + let restarted_report = restarted + .rank_authorized("model-v1", &[1.0, 0.0], &authorization) + .unwrap(); + + assert_eq!(first.evidence(), restarted.evidence()); + assert_eq!(first_report.results, restarted_report.results); + assert_eq!( + first_report.ordered_input_digest, + restarted_report.ordered_input_digest + ); + assert_eq!(first_report.output_digest, restarted_report.output_digest); + } + + #[test] + fn invalid_snapshot_and_query_inputs_fail_closed() { + let ids = vec![("item".to_owned(), "unit".to_owned())]; + let valid_bytes = packed(&[&[1.0]]); + let build_cases = [ + SemanticUnitIndex::build("", "model", 1, ids.clone(), &valid_bytes), + SemanticUnitIndex::build("snapshot", "", 1, ids.clone(), &valid_bytes), + SemanticUnitIndex::build("snapshot", "model", 0, ids.clone(), &[]), + SemanticUnitIndex::build("snapshot", "model", 1, vec![], &[]), + SemanticUnitIndex::build("snapshot", "model", 1, ids.clone(), b"short"), + SemanticUnitIndex::build("snapshot", "model", 1, ids.clone(), &packed(&[&[f64::NAN]])), + SemanticUnitIndex::build("snapshot", "model", 1, ids.clone(), &packed(&[&[0.0]])), + SemanticUnitIndex::build( + "snapshot", + "model", + 1, + vec![ids[0].clone(), ids[0].clone()], + &packed(&[&[1.0], &[1.0]]), + ), + ]; + let build_codes = [ + "empty_snapshot_version", + "empty_model_identity", + "empty_vector_dimension", + "empty_candidates", + "packed_vector_byte_length", + "non_finite_vector", + "zero_norm_vector", + "duplicate_candidate", + ]; + for (result, code) in build_cases.into_iter().zip(build_codes) { + assert_eq!(result.unwrap_err().code(), code); + } + + let index = + SemanticUnitIndex::build("snapshot", "model", 1, ids.clone(), &valid_bytes).unwrap(); + let rank_cases = [ + index.rank_authorized("other", &[1.0], &ids), + index.rank_authorized("model", &[1.0, 0.0], &ids), + index.rank_authorized("model", &[f64::NAN], &ids), + index.rank_authorized("model", &[0.0], &ids), + index.rank_authorized("model", &[1.0], &[]), + index.rank_authorized("model", &[1.0], &[ids[0].clone(), ids[0].clone()]), + index.rank_authorized( + "model", + &[1.0], + &[("missing".to_owned(), "unit".to_owned())], + ), + ]; + let rank_codes = [ + "model_mismatch", + "dimension_mismatch", + "non_finite_vector", + "zero_norm_vector", + "empty_authorization", + "duplicate_authorization", + "unknown_authorized_candidate", + ]; + for (result, code) in rank_cases.into_iter().zip(rank_codes) { + assert_eq!(result.unwrap_err().code(), code); + } + assert_eq!( + index + .rank_authorized_batch_packed("model", &[], &packed_authorization(&ids),) + .unwrap_err() + .code(), + "empty_query_batch" + ); + } + + #[test] + fn snapshot_replacement_is_atomic_and_failed_build_preserves_current() { + let handle = SemanticUnitIndexHandle::new(index("snapshot-v1")); + let in_flight_snapshot = handle.snapshot().unwrap(); + let invalid = SemanticUnitIndex::build( + "snapshot-v2", + "model-v1", + 2, + vec![("item".to_owned(), "unit".to_owned())], + b"short", + ); + assert!(invalid.is_err()); + assert_eq!( + handle.snapshot().unwrap().evidence().snapshot_version, + "snapshot-v1" + ); + + handle.replace(index("snapshot-v2")).unwrap(); + assert_eq!( + handle.snapshot().unwrap().evidence().snapshot_version, + "snapshot-v2" + ); + assert_eq!( + in_flight_snapshot.evidence().snapshot_version, + "snapshot-v1" + ); + } + + #[test] + fn poisoned_snapshot_lock_fails_closed() { + let handle = Arc::new(SemanticUnitIndexHandle::new(index("snapshot-v1"))); + let poisoned = Arc::clone(&handle); + assert!( + thread::spawn(move || { + let _guard = poisoned.current.write().unwrap(); + panic!("synthetic lock poison"); + }) + .join() + .is_err() + ); + + assert_eq!( + handle.snapshot().unwrap_err(), + SemanticIndexError::SnapshotLockPoisoned + ); + assert_eq!( + handle.replace(index("snapshot-v2")).unwrap_err(), + SemanticIndexError::SnapshotLockPoisoned + ); + } + + #[test] + fn error_display_is_the_stable_code() { + for error in [ + SemanticIndexError::ModelMismatch, + SemanticIndexError::EmptyQueryBatch, + SemanticIndexError::SnapshotLockPoisoned, + ] { + assert_eq!(error.to_string(), error.code()); + } + } +} diff --git a/crates/rankweave-python/Cargo.toml b/crates/rankweave-python/Cargo.toml new file mode 100644 index 0000000..d77ebf2 --- /dev/null +++ b/crates/rankweave-python/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "rankweave-python" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lib] +name = "_rankweave_core" +crate-type = ["cdylib"] +path = "src/lib.rs" + +[dependencies] +num-bigint = "0.4.6" +pyo3 = { version = "0.29.2", features = ["abi3-py310", "extension-module", "num-bigint"] } +rankweave-core = { path = "../rankweave-core" } diff --git a/crates/rankweave-python/src/lib.rs b/crates/rankweave-python/src/lib.rs new file mode 100644 index 0000000..18291ac --- /dev/null +++ b/crates/rankweave-python/src/lib.rs @@ -0,0 +1,306 @@ +//! Python bindings for the RankWeave calculation core. + +use num_bigint::BigUint; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyBytes; +use rankweave_core::semantic_index::{ + SemanticIndexRankingReport, SemanticIndexSnapshotEvidence, + SemanticUnitIndex as CoreSemanticUnitIndex, SemanticUnitIndexHandle, +}; + +#[pyfunction] +fn theoretical_min_max_normalize(score: f64, lower: f64, upper: f64) -> f64 { + rankweave_core::theoretical_min_max_normalize(score, lower, upper) +} + +#[pyfunction] +fn convex_combination_score( + semantic_score: Option, + lexical_score: Option, + semantic_weight_alpha: f64, +) -> f64 { + rankweave_core::convex_combination_score(semantic_score, lexical_score, semantic_weight_alpha) +} + +#[pyfunction] +fn reciprocal_rank_fusion_score(ranks: Vec, rank_constant_eta: BigUint) -> f64 { + rankweave_core::reciprocal_rank_fusion_score(&ranks, &rank_constant_eta) +} + +type SemanticUnitReportTuple = (String, String, String, usize, Vec<(String, String, f64)>); +type SemanticIndexEvidenceTuple = (String, String, String, String, String, String, usize, usize); +type SemanticIndexReportTuple = ( + SemanticIndexEvidenceTuple, + String, + String, + usize, + String, + String, + Vec<(String, String, f64)>, +); + +fn index_error(error: rankweave_core::semantic_index::SemanticIndexError) -> PyErr { + PyValueError::new_err(format!( + "{}: exact semantic index rejected input ({error:?})", + error.code() + )) +} + +fn evidence_tuple(evidence: &SemanticIndexSnapshotEvidence) -> SemanticIndexEvidenceTuple { + ( + evidence.schema_version.to_owned(), + evidence.snapshot_version.clone(), + evidence.model_digest.clone(), + evidence.dimension_digest.clone(), + evidence.vectors_digest.clone(), + evidence.snapshot_digest.clone(), + evidence.vector_dimension, + evidence.candidate_count, + ) +} + +fn index_report_tuple(report: SemanticIndexRankingReport) -> SemanticIndexReportTuple { + ( + evidence_tuple(&report.snapshot), + report.algorithm_version.to_owned(), + report.execution_profile.to_owned(), + report.worker_count, + report.ordered_input_digest, + report.output_digest, + report + .results + .into_iter() + .map(|result| (result.item_id, result.winning_unit_id, result.score)) + .collect(), + ) +} + +#[pyclass] +struct SemanticUnitIndex { + handle: SemanticUnitIndexHandle, +} + +#[pymethods] +impl SemanticUnitIndex { + #[new] + fn new( + snapshot_version: &str, + model_identity: &str, + vector_dimension: usize, + candidate_ids: Vec<(String, String)>, + packed_vectors: &Bound<'_, PyBytes>, + ) -> PyResult { + let index = CoreSemanticUnitIndex::build( + snapshot_version, + model_identity, + vector_dimension, + candidate_ids, + packed_vectors.as_bytes(), + ) + .map_err(index_error)?; + Ok(Self { + handle: SemanticUnitIndexHandle::new(index), + }) + } + + fn snapshot_evidence(&self) -> PyResult { + let snapshot = self.handle.snapshot().map_err(index_error)?; + Ok(evidence_tuple(snapshot.evidence())) + } + + fn replace_snapshot( + &self, + snapshot_version: &str, + model_identity: &str, + vector_dimension: usize, + candidate_ids: Vec<(String, String)>, + packed_vectors: &Bound<'_, PyBytes>, + ) -> PyResult<()> { + let replacement = CoreSemanticUnitIndex::build( + snapshot_version, + model_identity, + vector_dimension, + candidate_ids, + packed_vectors.as_bytes(), + ) + .map_err(index_error)?; + self.handle.replace(replacement).map_err(index_error) + } + + fn rank_authorized( + &self, + py: Python<'_>, + model_identity: String, + query_vector: Vec, + authorized_candidate_ids: Vec<(String, String)>, + ) -> PyResult { + let snapshot = self.handle.snapshot().map_err(index_error)?; + py.detach(move || { + snapshot + .rank_authorized(&model_identity, &query_vector, &authorized_candidate_ids) + .map(index_report_tuple) + .map_err(index_error) + }) + } + + fn rank_authorized_packed( + &self, + py: Python<'_>, + model_identity: String, + query_vector: Vec, + packed_authorization: &Bound<'_, PyBytes>, + ) -> PyResult { + let snapshot = self.handle.snapshot().map_err(index_error)?; + let packed_authorization = packed_authorization.as_bytes().to_vec(); + py.detach(move || { + snapshot + .rank_authorized_packed(&model_identity, &query_vector, &packed_authorization) + .map(index_report_tuple) + .map_err(index_error) + }) + } + + fn preflight_authorized_packed( + &self, + py: Python<'_>, + model_identity: String, + packed_authorization: &Bound<'_, PyBytes>, + ) -> PyResult { + let snapshot = self.handle.snapshot().map_err(index_error)?; + let packed_authorization = packed_authorization.as_bytes().to_vec(); + py.detach(move || { + snapshot + .preflight_authorized_packed(&model_identity, &packed_authorization) + .map(index_report_tuple) + .map_err(index_error) + }) + } + + fn preflight_authorized_top_k_packed( + &self, + py: Python<'_>, + model_identity: String, + packed_authorization: &Bound<'_, PyBytes>, + top_k: usize, + ) -> PyResult { + let snapshot = self.handle.snapshot().map_err(index_error)?; + let packed_authorization = packed_authorization.as_bytes().to_vec(); + py.detach(move || { + snapshot + .preflight_authorized_top_k_packed(&model_identity, &packed_authorization, top_k) + .map(index_report_tuple) + .map_err(index_error) + }) + } + + fn rank_authorized_batch_packed( + &self, + py: Python<'_>, + model_identity: String, + query_vectors: Vec>, + packed_authorization: &Bound<'_, PyBytes>, + ) -> PyResult> { + let snapshot = self.handle.snapshot().map_err(index_error)?; + let packed_authorization = packed_authorization.as_bytes().to_vec(); + py.detach(move || { + snapshot + .rank_authorized_batch_packed( + &model_identity, + &query_vectors, + &packed_authorization, + ) + .map(|reports| reports.into_iter().map(index_report_tuple).collect()) + .map_err(index_error) + }) + } + + fn rank_authorized_top_k_batch_packed( + &self, + py: Python<'_>, + model_identity: String, + query_vectors: Vec>, + packed_authorization: &Bound<'_, PyBytes>, + top_k: usize, + ) -> PyResult> { + let snapshot = self.handle.snapshot().map_err(index_error)?; + let packed_authorization = packed_authorization.as_bytes().to_vec(); + py.detach(move || { + snapshot + .rank_authorized_top_k_batch_packed( + &model_identity, + &query_vectors, + &packed_authorization, + top_k, + ) + .map(|reports| reports.into_iter().map(index_report_tuple).collect()) + .map_err(index_error) + }) + } +} + +#[pyfunction] +fn rank_semantic_units( + query_vector: Vec, + candidates: Vec<(String, String, Vec)>, +) -> PyResult { + let candidates: Vec<_> = candidates + .into_iter() + .map( + |(item_id, unit_id, vector)| rankweave_core::SemanticUnitCandidate { + item_id, + unit_id, + vector, + }, + ) + .collect(); + let report = rankweave_core::rank_semantic_units(&query_vector, &candidates) + .map_err(|error| PyValueError::new_err(format!("{}: {error}", error.code())))?; + Ok(( + report.schema_version.to_owned(), + report.algorithm_version.to_owned(), + report.ordered_input_digest, + report.vector_dimension, + report + .results + .into_iter() + .map(|result| (result.item_id, result.winning_unit_id, result.score)) + .collect(), + )) +} + +#[pyfunction] +fn rank_semantic_units_packed( + query_vector: Vec, + candidate_ids: Vec<(String, String)>, + packed_vectors: &Bound<'_, PyBytes>, +) -> PyResult { + let report = rankweave_core::rank_semantic_units_packed( + &query_vector, + &candidate_ids, + packed_vectors.as_bytes(), + ) + .map_err(|error| PyValueError::new_err(format!("{}: {error}", error.code())))?; + Ok(( + report.schema_version.to_owned(), + report.algorithm_version.to_owned(), + report.ordered_input_digest, + report.vector_dimension, + report + .results + .into_iter() + .map(|result| (result.item_id, result.winning_unit_id, result.score)) + .collect(), + )) +} + +#[pymodule] +fn _rankweave_core(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(theoretical_min_max_normalize, module)?)?; + module.add_function(wrap_pyfunction!(convex_combination_score, module)?)?; + module.add_function(wrap_pyfunction!(reciprocal_rank_fusion_score, module)?)?; + module.add_function(wrap_pyfunction!(rank_semantic_units, module)?)?; + module.add_function(wrap_pyfunction!(rank_semantic_units_packed, module)?)?; + module.add_class::()?; + Ok(()) +} diff --git a/docs/adr/0005-public-api-compatibility-policy.md b/docs/adr/0005-public-api-compatibility-policy.md new file mode 100644 index 0000000..9f06fa7 --- /dev/null +++ b/docs/adr/0005-public-api-compatibility-policy.md @@ -0,0 +1,94 @@ +# ADR 0005: Versioned public-API compatibility policy + +- **Status: Accepted** +- **Date:** 2026-08-22 +- **Scope:** the package-root API, installed CLI, and versioned JSON transports + +## Context + +Two known consumers pin RankWeave differently today, and the difference is a +symptom of a real gap, not a matter of taste. Naruon pins the published +`rankweave==0.1.0` package. LineageWeave pins a specific `main` git commit +(`docs/product-technical-gap-baseline.md` §3; LineageWeave ADR 0024) because +it needs post-0.1.0 APIs (`weighted_reciprocal_rank_fuse` with weighted +channels) that are not on PyPI yet. Neither consumer has a written contract +describing which upgrades are safe, because RankWeave has never published +one. A consumer currently has to read source history to guess. + +RankWeave's version is `0.18.0` — pre-1.0 by strict SemVer, where any `0.x` +release is conventionally allowed to break compatibility. That convention is +correct for a package with no real consumers. It is the wrong signal for a +package two other repositories in this organization already import through +`services.hybrid_retrieval`-shaped seams and treat as a stable dependency. + +## Decision + +1. **The Python package-root surface is exactly `rankweave.__all__`.** Anything reachable + only through `rankweave..` and not re-exported at the + package root is internal and may change without notice. The root + `__init__.py` docstring and `README.md`'s documented functions are the + two authoritative, human-readable views of this same set; keep them + synchronized when `__all__` changes. The installed `rankweave` console + entry point and its independently versioned pairwise/family JSON schemas + are additional public transport contracts; they are not Python symbols and + therefore are frozen by their entry-point and schema-version tests rather + than by `__all__`. +2. **Effective immediately as of this ADR (source-tree `0.18.0`), names in + `__all__` are not removed or renamed within a minor version.** A symbol + present in `__all__` at one published minor version (`0.N.0`) stays + present, importable, and behaviorally compatible through every patch + release of that minor version. Removing or renaming a symbol requires a + minor version bump at minimum, and the removed name must appear in + `CHANGELOG.md` under a `### Removed` heading naming its replacement, if + any. +3. **Enforcement is a test, not a promise.** `tests/test_public_api_compatibility.py` + freezes the exact `__all__` set as of this ADR and asserts every frozen + name is still exported and still resolvable + (`hasattr(rankweave, name)`). A PR that breaks this test is either adding + a genuine removal — which must update the frozen set, the `CHANGELOG.md` + `### Removed` entry, and this ADR's frozen-set reference together in one + reviewed change — or it is an accidental regression the test caught + before a consumer did. New additions to `__all__` do not need to touch + the frozen set; the test only asserts a lower bound. +4. **Behavioral compatibility, not just import compatibility.** A symbol + staying importable but silently changing its numeric defaults, gain + function, `eta`, or channel-weight semantics is still a breaking change + for LineageWeave ADR 0024, which pins those exact values. Changes to + defaults documented as research-grounded in `docs/research/README.md` + require the same minor-version-bump-plus-CHANGELOG discipline as a + removal, even when the function name is untouched. +5. **This does not commit to PyPI publication cadence.** Issue #35 (PyPI + Trusted Publisher misconfiguration) is a separate, orthogonal gap. This + ADR governs what a version *means* once published; it does not promise + when the next version *will be* published. + +## Consequences + +A consumer reading this ADR can safely pin `rankweave>=0.18,<0.19` (or the +equivalent commit range) and know that upgrading within that range never +removes a symbol it already imports. A consumer that needs a symbol added +after `0.18.0` still has to pin a specific commit or wait for the next minor +release, exactly as LineageWeave does today — this ADR does not retroactively +publish anything, it only makes the existing informal expectation +enforceable and visible. + +Future ADRs that intentionally remove or rename a public symbol must update +`tests/test_public_api_compatibility.py`'s frozen set in the same PR, and +must reference this ADR in their own consequences section. + +## Alternatives considered + +- **Full SemVer `1.0.0` commitment now:** rejected. A `1.0.0` bump implies a + stability claim beyond what this ADR makes (it says nothing about numeric + defaults changing across minor versions, only within them staying put). + Reaching `1.0.0` is a future decision, not a byproduct of writing this + policy. +- **No enforcement, policy text only:** rejected. AGENTS.md's own standing + rule is "write tests before behavior changes"; a compatibility policy with + no test is a claim nobody checks. +- **Deprecation-warning period before removal:** considered but out of + scope for this ADR. RankWeave has no runtime warning mechanism today + (stdlib-only, no logging framework mandated) and no removal has been + proposed yet to design one against. Add it as a follow-up ADR when a + concrete removal is proposed, grounded in an actual case instead of a + hypothetical one. diff --git a/docs/adr/0006-rust-calculation-core.md b/docs/adr/0006-rust-calculation-core.md new file mode 100644 index 0000000..402e7fd --- /dev/null +++ b/docs/adr/0006-rust-calculation-core.md @@ -0,0 +1,157 @@ +# ADR 0006: One Rust calculation core behind the public Python contract + +- **Status: Accepted** +- **Date:** 2026-08-26 +- **Scope:** RankWeave fusion, evaluation, comparison, and policy-assessment arithmetic + +## Context + +RankWeave is the ecosystem owner for retrieval fusion and ranking evidence. +LineageWeave ADR 0225 forbids a second consumer-side arithmetic engine, while +RankWeave currently implements the production calculations in Python. Issue +#45 requires one Rust implementation without changing the research-grounded +public semantics or moving source access, authorization, retrieval, or provider +work into this package. + +Parallel floating-point reduction cannot be introduced casually. Rayon states +that the reduction order of floating-point `sum` is unspecified, and NVIDIA +documents that parallel evaluation order and fused multiply-add can change a +floating-point result. An unspecified reduction order would break RankWeave's +deterministic result and tie contracts. + +PyO3 supports native Python extension modules and Python's stable ABI. Maturin +supports mixed Python/Rust projects, allowing the established Python API and +type surface to remain the public adapter while wheel artifacts contain the +sole calculation implementation. + +## Decision + +RankWeave will use one Cargo workspace with two responsibility boundaries: + +1. `rankweave-core` is a Python-independent Rust library containing validation, + fusion, evaluation, comparison, and policy-assessment arithmetic. +2. `_rankweave_core` is a thin PyO3 extension. Existing Python modules validate + transport types, translate stable public records, and call Rust; they do not + retain a second calculation path. +3. Maturin builds the existing mixed Python package. The first Rust release + targets the repository's minimum supported CPython stable ABI and retains + the existing `rankweave` import and console entrypoints. + +The engine accepts ordered caller evidence and an explicit policy envelope. +The envelope binds algorithm revision, policy revision, estimator identity, +estimator artifact digest, ordered active channels, and all numerical policy +values. A missing, mismatched, non-finite, or unproven policy fails closed. +RankWeave never estimates or renormalizes a policy inside a fusion request. + +Candidate-level absence and channel-level unavailability remain distinct: + +- when an explicitly active channel did not return one candidate, the existing + documented theoretical-minimum contribution and missing-channel evidence are + preserved; +- when the channel itself is unavailable, the caller must supply a separately + estimated policy for the remaining exact channel set. RankWeave rejects an + active-channel mismatch and never converts channel unavailability into a + candidate-level zero. + +The output envelope contains ordered results, per-channel score/rank/weight and +contribution evidence, missing-candidate markers, algorithm revision, policy +revision, estimator provenance, input digest, backend identity, and explicit +limitations. It contains no database identifier beyond opaque caller-owned +item/channel/query identifiers. + +## CPU and GPU execution + +The CPU backend uses Rust `f64`. Rayon may schedule independent candidates, +queries, or randomization draws, but it must not reduce the floating-point +terms of one score or metric in an unspecified order. Each scalar reduction +follows the documented caller order with the same operation sequence as the +reference vector; deterministic sorting applies the public tie contract after +parallel work completes. + +The optional GPU backend is an explicit caller choice, never an automatically +selected size threshold. It uses CUDA double precision, fixed input order per +scalar result, no fast-math mode, and no contraction that changes the public +operation sequence. The backend is available only after the complete packaged +conformance vectors produce bit-identical finite output, ordering, contribution +evidence, validation errors, and digests against the CPU backend on that build. +If conformance fails or a compatible device is absent, a GPU request returns +backend-unavailable; it never falls back silently or substitutes a tolerance. + +Benchmarks record exact source revision, backend revision, hardware, driver, +compiler flags, thread/device configuration, workload digest, distribution, +and result-conformance digest. RankWeave publishes no CPU/GPU throughput or +capacity claim without that artifact. + +## Migration and compatibility + +Migration is vertical by public operation, not a permanent dual engine: + +1. freeze current public vectors, errors, ordered evidence, and artifact + schemas as cross-language conformance fixtures; +2. implement and expose one operation in Rust; +3. switch its Python function to the extension and delete the corresponding + Python arithmetic in the same change; +4. prove Python API, CPU, and optional GPU conformance plus complete Rust and + Python coverage before moving the next operation; and +5. release immutable wheels before any consumer upgrades or deletes its own + compatibility seam. + +No environment flag may restore deleted Python arithmetic. An unsupported +platform receives an explicit installation or backend-unavailable failure; +source fallback is not a second production engine. + +## Security and operability properties + +- Rust core inputs are bounded before allocation and reject duplicate, + non-finite, and domain-invalid state. +- Python releases contain no provider, database, identity, or network client. +- Cargo, Python, and build-tool versions are immutable in lockfiles and CI. +- Wheels, SBOMs, attestations, schemas, type markers, and source revisions stay + bound to the same release. +- Panic does not cross the extension boundary; public failures retain stable, + documented Python exception classes. + +## Consequences + +- RankWeave no longer remains a pure-Python implementation, but the Python + public surface and store-agnostic product boundary remain intact. +- Wheel coverage expands by operating system, architecture, and supported + Python ABI; source-only installation requires the pinned Rust toolchain. +- Deterministic scalar arithmetic constrains where parallel reduction is legal. +- GPU acceleration remains optional and explicit because portability and exact + evidence are more important than an inferred dispatch policy. +- LineageWeave can delete local fusion arithmetic only after a released + RankWeave artifact and consumer-pin upgrade prove the full envelope. + +## Rejected alternatives + +- **Keep Python as the production arithmetic engine:** leaves the owning + repository inconsistent with the ecosystem calculation boundary. +- **Retain Python as a runtime fallback:** creates the duplicate engine this + decision removes and permits platform-dependent semantics. +- **Move fusion into TEPP or fast-mlsirm:** those products own measurement and + estimation; they may produce policy provenance but do not own retrieval + fusion. +- **Use Rayon floating-point `sum`:** its reduction order is unspecified. +- **Choose CPU or GPU from an item-count rule:** introduces an ungrounded + heuristic and makes backend identity workload-dependent. +- **Accept an arbitrary CPU/GPU tolerance:** weakens exact contribution and tie + evidence. A nonconforming GPU backend is unavailable instead. + +## References — APA 7th edition + +IEEE Computer Society. (2019). *IEEE standard for floating-point arithmetic* +(IEEE Std 754-2019). IEEE. https://doi.org/10.1109/IEEESTD.2019.8766229 + +NVIDIA Corporation. (2026). *CUDA C++ best practices guide* (Version 13.3). +https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/ + +PyO3 Project. (2026). *Building and distribution*. PyO3 user guide. +https://pyo3.rs/main/building-and-distribution.html + +PyO3 Project. (2026). *Features reference*. PyO3 user guide. +https://pyo3.rs/main/features + +Rayon Developers. (2026). *ParallelIterator*. Rayon 1.12.0 documentation. +https://docs.rs/rayon/1.12.0/rayon/iter/trait.ParallelIterator.html + diff --git a/docs/adr/0007-semantic-unit-cosine-ranking.md b/docs/adr/0007-semantic-unit-cosine-ranking.md new file mode 100644 index 0000000..8343bb6 --- /dev/null +++ b/docs/adr/0007-semantic-unit-cosine-ranking.md @@ -0,0 +1,105 @@ +# ADR 0007: Rust-owned semantic-unit cosine ranking + +- **Status: Accepted** +- **Date:** 2026-08-26 +- **Scope:** ranking caller-authorized embedding vectors by semantic unit + +## Context + +Consumers need to compare one query embedding with paragraph-, DOM-, or image +region-level embeddings without copying vector arithmetic into each product. +RankWeave owns retrieval-ranking calculation, while the consumer still owns +source access and authorization and contextual-orchestrator owns embedding +model discovery and execution. ADR 0006 requires migrated arithmetic to have +one Rust implementation behind the Python contract. + +Cosine is undefined for a zero vector. Ragged or non-finite vectors do not +describe one valid vector space. Neither case may be repaired by padding, +dropping coordinates, inventing a fallback score, or choosing another model. + +## Decision + +`rank_semantic_units` accepts one already-authorized query vector and an +ordered sequence of `(item_id, unit_id, vector)` candidates. The Rust core: + +1. rejects empty, non-finite, zero-norm, dimension-mismatched, and duplicate + item/unit inputs with stable error codes; +2. computes cosine in caller coordinate order, scaling each vector by its + maximum absolute component before the dot product and norms to avoid finite + square overflow; +3. clamps raw cosine to `[0, 1]` without remapping `[-1, 1]`, adding a weight, + or applying a relevance threshold; +4. retains the highest-scoring unit for each item, breaking an exact unit tie + by ascending `unit_id`; +5. orders items by descending score and then ascending `item_id`; and +6. returns the winning unit, score, vector dimension, schema and algorithm + versions, and a SHA-256 digest of a canonical length-prefixed encoding of + the exact ordered query and candidates. + +The digest binds UTF-8 identifiers and IEEE 754 binary64 bit patterns. It is +integrity evidence only, not authentication, provenance, or scientific +validity. The Python module is a typed record/transport adapter and contains no +second cosine implementation. + +An additive packed adapter accepts the same ordered candidate identities plus +one concatenation of their IEEE 754 binary64 coordinates in network byte order +(most-significant byte first). It rejects every byte length other than + + +`candidate count × query dimension × 8` + + +and decodes in the Rust core. For equivalent inputs it must return the exact +same v1 schema, algorithm, ordered-input digest, scores, winning units, and +ordering as `rank_semantic_units`. This representation removes Python scalar +expansion only; it is not a persistent retrieval index or a latency claim. + +## Responsibility boundary + +- The caller filters and authorizes candidates before the call and + post-authorizes returned opaque identifiers. +- contextual-orchestrator selects the embedding provider and model and returns + vectors plus model provenance. +- RankWeave validates and ranks supplied vectors. It does not call a provider, + select a model, query a store, infer a cutoff, or apply a business threshold. + +## Consequences + +- Consumers can delete local cosine and per-item max-pooling arithmetic after + they pin a released RankWeave artifact containing this contract. +- Negative cosine is represented by the documented channel infimum `0`, not a + manufactured positive score. +- Identifier lexical order is now public tie evidence; callers needing another + order must supply it as a separate downstream presentation policy. +- SHA-256 adds one small, lockfile-pinned Rust dependency; no Python runtime + dependency is added. +- Consumers with canonical binary64 storage can avoid constructing one Python + float object per stored coordinate. Exact ranking still examines every + supplied coordinate, so a separate accepted owner-index contract is required + before treating this transport as a bounded-latency retrieval path. + +## Rejected alternatives + +- **Keep cosine in each consumer:** duplicates the calculation and versioning + boundary that ADR 0006 removes. +- **Pad ragged vectors or treat zero norm as zero similarity:** fabricates a + valid comparison from invalid vector-space evidence. +- **Map cosine from `[-1, 1]` to `[0, 1]`:** changes raw embedding similarity + and creates a positive score for orthogonal or opposing evidence. +- **Select the model or authorization policy here:** crosses the provider and + consumer trust boundaries. +- **Use native-endian packed values:** makes the same request decode differently + across hosts and cannot preserve the canonical v1 digest. + +## References — APA 7th edition + +IEEE Computer Society. (2019). *IEEE standard for floating-point arithmetic* +(IEEE Std 754-2019). IEEE. https://doi.org/10.1109/IEEESTD.2019.8766229 + +National Institute of Standards and Technology. (2015). *Secure Hash Standard +(SHS)* (FIPS PUB 180-4). U.S. Department of Commerce. +https://doi.org/10.6028/NIST.FIPS.180-4 + +Salton, G., & Buckley, C. (1988). Term-weighting approaches in automatic text +retrieval. *Information Processing & Management, 24*(5), 513–523. +https://doi.org/10.1016/0306-4573(88)90021-0 diff --git a/docs/adr/0008-persistent-exact-semantic-index.md b/docs/adr/0008-persistent-exact-semantic-index.md new file mode 100644 index 0000000..b1cfa8c --- /dev/null +++ b/docs/adr/0008-persistent-exact-semantic-index.md @@ -0,0 +1,236 @@ +# ADR 0008: Persistent exact semantic-unit index snapshots + +- **Status: Accepted** +- **Date:** 2026-08-31 +- **Scope:** exact authorization-scoped ranking over reusable embedding snapshots + +## Context + +ADR 0007 owns exact semantic-unit cosine and its ordered-input integrity +evidence. Its scalar and packed request forms still validate, hash, transfer, +and score every coordinate on every query. A consumer with 6,578 vectors of +dimension 3,072 therefore submits 161,660,928 vector bytes for each request. +The packed form removes Python scalar expansion but is not an index. + +Consumers need a reusable owner-side calculation structure without moving +model selection, database access, or ABAC policy into RankWeave. Approximate +nearest-neighbor structures are not acceptable because their candidate loss +would change recall. Query-result caches are also not acceptable because they +do not bind a result to the exact authorized evidence snapshot. + +## Decision + +'SemanticUnitExactIndex' builds one immutable snapshot from an opaque snapshot +version, opaque model identity, vector dimension, ordered candidate identities, +and canonical big-endian IEEE 754 binary64 vectors. Build validates the complete +snapshot before it can become active and records separate SHA-256 digests for +the model identity, dimension, exact vectors plus identities, and the combined +snapshot. + +The Rust core precomputes each validated candidate vector's maximum-absolute +scale and Euclidean norm and stores its scaled coordinates in one contiguous +array. This is exact index metadata tied to the snapshot digest, not a score +cache. A query: + +1. supplies the same opaque model identity, one nonzero finite vector, and the + complete ordered set of caller-authorized candidate identities; +2. fails closed on a model or dimension mismatch, duplicate authorization, or + an authorized identity absent from the immutable snapshot; +3. computes every authorized dot product in Rust, with no threshold, + approximate pruning, candidate window, or dropped coordinate; +4. uses Rayon indexed parallel iteration, preserving each dot product's + coordinate order and collecting in authorization order so worker count does + not change scores, ranking, or digests; +5. applies ADR 0007's exact per-item maximum and deterministic tie rules; and +6. returns snapshot evidence, CPU execution profile and observed worker count, + ordered input digest, output digest, and exact result rows. + +The equivalent packed-authorization transport starts with an unsigned +big-endian 64-bit identity count, followed by one unsigned big-endian 64-bit +byte length and UTF-8 byte string for each item id and unit id in order. It +produces the same input digest, scores, output digest, and failure semantics as +the row transport. It removes per-identity Python/FFI rows but does not alter +the caller-supplied authorization set. + +An additive packed batch operation accepts two or more ordered query vectors +against one identical packed authorization buffer and immutable snapshot. It +validates and resolves that authorization once, then visits each authorized +candidate vector once while accumulating every query's dot product in that +query's original coordinate order. Candidate traversal remains Rayon-parallel; +the terms of any one dot product are never parallel-reduced. Each returned +query report is evidence-equivalent to an independent packed query: the same +ordered-input digest, output digest, scores, winning units, and ordering. The +batch has no cross-request result cache and is valid only when the consumer +proves that every request supplied the same authorization snapshot digest; +each consumer still post-authorizes its own returned rows. The explicit query +and authorization sequences bound the work surface without introducing a +workload-size dispatch heuristic. +Bit-identical query vectors inside one batch share one exact dot-product +calculation as deterministic common-subexpression elimination; the operation +still emits a separately ordered report and the independently defined input +and output digests for every supplied query. This lifetime ends with the batch. + +'replace_snapshot' first builds and validates a complete replacement outside +the active lock, then swaps one immutable reference atomically. A concurrent +query retains the old snapshot for its whole execution or acquires the new one; +it never observes a mixed snapshot. V1 intentionally has no incremental +mutation API. Consumers recover after restart by loading the same immutable +version and bytes from their governed persistent projection and comparing the +owner-computed digests before activation. + +The portable required execution profile is deterministic multithreaded CPU. +RankWeave advertises no GPU profile in v1. A future accelerator requires its own +accepted owner decision, real device execution evidence, and exact or +explicitly bounded parity against the CPU profile; a device label alone is not +evidence. + +A 2026-08-31 Apple Accelerate `dgemm` profile over a synthetic +6,578-by-3,072 matrix and four queries measured 3.291-3.584 ms, but 13,007 of +26,312 dot products differed bitwise from the coordinate-ordered owner result +(maximum absolute difference 3.41e-13). Stable top-k alone cannot repair the +existing complete-result digest, so this backend remains unavailable. A future +exact top-k accelerator may use Higham's IEEE-754 forward-error bound only to +prove that a candidate cannot cross the kth boundary, followed by +coordinate-ordered scalar recomputation of every ambiguous candidate. If the +bound excludes none, it must run the complete scalar path. Such screening +needs a separately versioned top-k output digest and adversarial near-tie and +all-ambiguous conformance tests before activation; an empirical tolerance is +not a substitute. + +The follow-up proof profile applies the standard binary64 unit roundoff +`u = 2^-53` and `gamma_n = nu / (1 - nu)`. Both BLAS and the required +coordinate-ordered scalar dot lie within `gamma_n |x|^T|y|` of the real dot. +The owner therefore performs one `dgemm(A, Q)` for approximate signed dots and +a second `dgemm(|A|, |Q|)` to bound the absolute dot for every matrix/query +pair. `|A|` is immutable derived snapshot metadata bound by the same source +vector digest; `|Q|` exists only for the batch. The absolute GEMM's own error +is inverted conservatively before the two dot-error bounds are added. The +implementation also charges one minimum-normal absolute error for each +multiply and add, divided by the standard error denominator, so cancellation, +subnormal products, and gradual underflow remain contained rather than relying +on the relative-error model outside its assumptions. Invalid or non-finite +bounds fall back to the complete scalar path. + +Only a candidate whose upper endpoint is strictly below the kth-largest lower +endpoint is excluded. Each score interval is widened outward by one binary64 +value around the positive-norm division. Equality remains ambiguous, and every +ambiguous candidate is recomputed in coordinate order before stable top-k. +The earlier one-GEMM 6,578-by-3,072-by-four profile measured 3.605-4.762 ms, +but it was only a profile: it used a Cauchy-Schwarz simplification and a +same-sign sufficient condition, so it did not cover the mixed-sign embedding +workload. Its corrective 30-sample nearest-rank p95 was 4.604 ms. It is retained +as historical motivation, not activation evidence. The accepted production +contract is the two-GEMM absolute-dot proof above, with mixed-sign near-tie, +all-equal, cancellation, and underflow conformance tests. + +The additive `rank_authorized_top_k_batch_packed` contract now implements that +proof on macOS. Before screening, unit intervals pool to item intervals by +taking the maximum lower and upper endpoints, matching the exact per-item +maximum definition. A candidate item is excluded only when its maximum upper +endpoint is strictly below the kth-largest item lower endpoint. Every unit of +every remaining item is then recomputed with the existing coordinate-ordered +scalar loop; equality, near ties, and an all-ambiguous set therefore retain the +same stable item and winning-unit result as the scalar contract. + +Unavailable Accelerate, an unrepresentable BLAS dimension, a non-finite bound, +or a screen that excludes no item falls back to the complete scalar path. The +underflow allowance is part of the mathematical interval, never a workload +tolerance. The top-k input and output digests use separate `v1` domains and +bind `k`, so they cannot be confused with the complete-ranking digest. +Non-macOS platforms retain the exact scalar top-k profile. + +`preflight_authorized_top_k_packed` uses the first identity from the caller's +real packed authorization scope as its query and executes the same exact top-k +profile. It exists only to close readiness before a consumer accepts traffic; +the returned report remains fully versioned and is discarded by readiness +callers rather than cached as a user result. + +The production two-GEMM implementation over the same synthetic +6,578-by-3,072 matrix and four distinct queries preserved the scalar top-four +prefix exactly. Thirty direct owner calls measured 11.939 ms minimum, +12.260 ms mean, 12.861 ms nearest-rank p95, and 12.972 ms maximum. This does +not establish a deterministic wall-clock maximum: a later 500-iteration +consumer-path trace observed four calls above 20 ms, including one owner call +at 61.509 ms during host contention. The profile is therefore an exact +calculation prerequisite, not evidence that a consumer's 20 ms end-to-end SLO +is met. + +A subsequent process-isolation profile removed Python and the consumer event +loop. A host-native Rust process held the same-shape 6,578-by-3,072 synthetic +snapshot behind one immutable handle, assigned macOS user-initiated QoS to the +caller and a dedicated Rayon pool, and checked stable output digests for four +distinct mixed-sign queries. An initial 100-call sweep of every outer worker +count from one through the host's ten physical cores measured 10.052-15.216 ms +and appeared to pass 20 ms. That apparent calibration did not survive ten +fresh one-worker processes: two of 1,000 later calls took 52.948 ms and 81.775 +ms. A startup worker-count calibration therefore cannot prove a deterministic +maximum and is not an accepted execution contract. + +The same harness in the declared four-vCPU Colima Linux runtime exercised the +portable coordinate-ordered scalar fallback. One through four outer workers +respectively measured maxima of 82.359, 56.221, 52.808, and 80.210 ms across +100 calls each. Neither the native macOS process nor the portable Linux profile +proves that every call completes within 20 ms. RankWeave consequently does not +publish a native owner service, hard-coded worker count, or consumer endpoint +from these measurements. + +## Responsibility boundary + +- The consumer persists the source projection, selects the model-specific + snapshot, computes ABAC eligibility, supplies the complete authorized + candidate identities, and post-authorizes every returned identity. +- RankWeave validates, indexes, and scores only supplied vectors and returns no + item absent from the caller authorization. +- RankWeave remains store- and provider-agnostic. It does not query a database, + interpret tenant attributes, select an embedding model, or persist customer + data. + +## Consequences + +- Snapshot build and restart recovery remain proportional to all snapshot + bytes, while a warm query transfers only its vector and authorized opaque + identities. +- Exact precomputed scales and norms remove repeated decode, validation, and + norm work. Every authorized coordinate still participates in scoring. +- Identically authorized concurrent queries can share one candidate-vector + traversal without sharing results or relaxing per-query evidence. +- Rayon adds a lockfile-pinned Rust dependency but no Python runtime + dependency. Worker count comes from the owner runtime and is reported; it is + not a ranking parameter. +- Consumers must keep the prior snapshot active or report unavailability when + replacement validation fails. They may not partially repair a snapshot. + +## Rejected alternatives + +- **HNSW, IVFFlat, or another approximate index:** can omit a true nearest + candidate and therefore changes exact recall. +- **Filter after approximate retrieval:** can lose authorized evidence before + ABAC filtering and cannot prove completeness. +- **Per-query packed full snapshot:** still transfers and scans the full source + representation on every query. +- **Incrementally mutate v1 in place:** permits mixed model, dimension, and + vector versions unless a more complex transaction/digest protocol is added. +- **Consumer-owned norms or cosine:** duplicates owner arithmetic and weakens + the digest boundary. +- **Fixed worker count:** is an ungrounded deployment knob. Indexed parallel + work is deterministic for every observed worker count. +- **Coalesce by model or question alone:** authorization equality is not + implied by either value and would risk returning a row outside one caller's + evidence snapshot. + +## References — APA 7th edition + +IEEE Computer Society. (2019). *IEEE standard for floating-point arithmetic* +(IEEE Std 754-2019). IEEE. https://doi.org/10.1109/IEEESTD.2019.8766229 + +National Institute of Standards and Technology. (2015). *Secure Hash Standard +(SHS)* (FIPS PUB 180-4). U.S. Department of Commerce. +https://doi.org/10.6028/NIST.FIPS.180-4 + +Higham, N. J. (2002). *Accuracy and stability of numerical algorithms* (2nd +ed.). Society for Industrial and Applied Mathematics. +https://doi.org/10.1137/1.9780898718027 + +Salton, G., & Buckley, C. (1988). Term-weighting approaches in automatic text +retrieval. *Information Processing & Management, 24*(5), 513–523. +https://doi.org/10.1016/0306-4573(88)90021-0 diff --git a/docs/operations/hourly-commercialization-loop.md b/docs/operations/hourly-commercialization-loop.md index 90ca486..01c10fa 100644 --- a/docs/operations/hourly-commercialization-loop.md +++ b/docs/operations/hourly-commercialization-loop.md @@ -71,10 +71,16 @@ the red gate. ### 3. Implementation and deterministic validation The verified red state is committed locally only so model fallback can return -to a known tree. The implementation phase may edit normal product, -documentation, version, and package files, but it still cannot execute Bash, -use the web, touch external directories, or edit `.github/`, `.git/`, or agent -control files. +to a known tree. The implementation phase may edit normal Python product and +documentation files, but it still cannot execute Bash, use the web, touch +external directories, or edit `.github/`, `.git/`, or agent control files. + +This autonomous lane is limited to Python production changes. Rust source, +Cargo manifests, `pyproject.toml`, and the complete `crates/` tree are outside +its diff boundary. That restriction keeps model-authored native code out of +later credentialed builders and makes the trusted base extension the correct +native dependency for final validation. A Rust-core increment requires a +separate maintainer-authored and reviewed pull request. A deterministic post-agent gate rejects: @@ -84,7 +90,8 @@ A deterministic post-agent gate rejects: - more than 25 changed files; - any file larger than 256 KiB; - more than 1 MiB of changed-file content; -- proposals without a production `src/rankweave/*.py` change. +- proposals without a production `src/rankweave/*.py` change; +- Rust, Cargo, or Python build-metadata changes. The accepted proposal then runs, with no provider or GitHub credential and no network access: diff --git a/docs/product-requirements.md b/docs/product-requirements.md new file mode 100644 index 0000000..b9582c1 --- /dev/null +++ b/docs/product-requirements.md @@ -0,0 +1,189 @@ +# RankWeave Product Requirements + +Status: Candidate product contract + +Product: `ContextualWisdomLab/RankWeave` + +Normative architecture: repository ADRs, including +[`ADR 0006`](adr/0006-rust-calculation-core.md), and `AGENTS.md` + +Supporting evidence: [`docs/research/README.md`](research/README.md) + +## 1. Product purpose + +RankWeave is the store-agnostic calculation and audit boundary for retrieval +fusion, ranking evaluation, paired and family-wise comparison, fixed-policy +selection, temporal backtesting, and strict TREC interchange. It runs as a +standalone Python package and CLI and as an imported module in products such as +Naruon and LineageWeave. + +The product turns caller-owned ranked or scored evidence into deterministic, +inspectable ranking evidence. It does not retrieve source records, authorize a +reader, select a model, call a provider, or infer business facts. + +## 2. Users and jobs + +| User | Job | Required evidence | +| --- | --- | --- | +| Retrieval engineer | Combine heterogeneous retrieval channels | Ordered results, exact channel contributions, input policy, and deterministic tie behavior | +| Evaluation researcher | Compare systems on complete judgment sets | Per-query metrics, effect estimate, raw inference evidence, and declared experimental boundary | +| Platform integrator | Embed fusion without copying an engine | Stable typed API, wheel contract, fail-closed validation, and immutable version pin | +| Release operator | Run auditable comparisons in CI | Bounded CLI input, versioned JSON, artifact digests when requested, and stable exit behavior | +| Auditor | Reconstruct what produced a ranking decision | Algorithm/policy identity, ordered inputs, limitations, and integrity-bound artifacts | + +## 3. Product principles + +1. **One arithmetic owner.** Consumers supply evidence and policy; they do not + copy RankWeave fusion or evaluation arithmetic. +2. **No invented signal.** Within an explicitly active channel policy, a + candidate absent from one channel contributes that scoring function's + documented theoretical minimum. An unavailable channel, query, judgment, + policy, or artifact is unavailable or invalid and must not be silently + converted into an active zero-valued channel. +3. **No arbitrary policy.** Channel weights, folds, cutoffs, test alternatives, + and candidate families are caller-provided, research-grounded inputs with + provenance. RankWeave does not guess them. +4. **Complete auditability.** Public results retain the evidence necessary to + reproduce ordering, contributions, metrics, and statistical comparison. +5. **Experiment separation.** Policy selection uses declared validation or + training evidence; held-out and temporal assessment remain distinct from + final all-data recommendations. +6. **Deterministic compatibility.** Identifier alignment, ordered tie breaks, + bounded parsing, and versioned schemas are public contracts. + +## 4. Functional requirements + +### 4.1 Fusion + +- Accept caller-owned scored or rank-only channel results and an explicit, + compatible policy. +- Reject non-finite values, invalid domains, duplicate identifiers, and + incompatible channel/policy sets before calculation. +- Return a deterministic complete ranking with exact per-channel contribution + evidence. Represent candidate-level absence explicitly and apply the + documented theoretical-minimum semantics; never use those semantics to hide + that an entire policy channel was unavailable. +- Preserve first-seen order where the documented public contract resolves an + exact score tie by input order. +- Introduce no LineageWeave-specific threshold, candidate window, database + query, or authorization rule. + +### 4.2 Semantic-unit vector ranking + +- Accept one caller-authorized query vector and ordered semantic-unit vectors. +- Reject invalid vector-space evidence rather than padding or inventing a + fallback score. +- Return deterministic per-item winning-unit evidence and versioned ordered + input integrity evidence. +- Do not select an embedding model, apply authorization, or infer a threshold. +- Build immutable exact snapshots with digest-bound model, dimension, identity, + vector, and precomputed scale/norm evidence; replace a snapshot only after + complete validation. +- Score every caller-authorized candidate on a deterministic multithreaded Rust + CPU path. Never use approximate pruning or return an identity absent from the + supplied authorization set. + +### 4.3 Evaluation and comparison + +- Require complete ranking/judgment query-set parity. +- Produce per-query and aggregate precision, recall, reciprocal-rank, and + graded nDCG evidence under the documented metric definitions. +- Align paired comparisons by query identifier and preserve every difference. +- Expose exact or deterministic Monte Carlo randomization evidence and keep + p-values separate from effect size and deployment value. +- Compare an explicit ordered candidate family against one baseline and retain + raw plus Holm-adjusted evidence in the original candidate order. + +### 4.4 Policy assessment + +- Evaluate only caller-declared, ordered policy families. +- Keep validation selection, explicit-fold out-of-fold assessment, temporal + assessment, and final all-data recommendation as distinct result objects. +- Accept caller-owned fold and availability-time boundaries; never generate a + hidden random split or claim that a supplied grouping is leakage-safe. + +### 4.5 TREC and CLI interoperability + +- Parse and format the documented TREC run and qrels profiles with bounded + memory and stable physical-line diagnostics. +- Emit exactly one versioned UTF-8 JSON document on success and no stdout on an + expected CLI error. +- Preserve default v1 documents; artifact digests and byte counts are explicit + v2 contracts and disclose no local paths. +- Package strict Draft 2020-12 schemas for every emitted report contract. + +## 5. Quality requirements + +- Production behavior is deterministic for identical ordered inputs. +- Package-root records, errors, and functions exported through + `rankweave.__all__`, plus the installed CLI entry point and its versioned + transport schemas, remain backward compatible under ADR 0005. +- Statement, branch, public-docstring, wheel-install, CLI, schema, and edge-case + checks remain complete for every release candidate. +- Inputs are bounded and fail closed at trust boundaries. Integrity digests are + not described as authentication, attestation, or scientific validity. +- Release authorization and PyPI publication use exact immutable source and + artifact evidence; a source-only commit is not a released consumer contract. + +## 6. Architecture and ecosystem boundary + +```mermaid +flowchart LR + Consumer[Authorized consumer] -->|ranked or scored evidence plus policy provenance| Adapter[RankWeave public API or CLI] + Adapter --> Engine[RankWeave calculation engine] + Engine --> Report[ranking, contributions, evaluation, and audit evidence] + Report --> Consumer + Consumer -. owns .-> Store[(source store)] + Consumer -. owns .-> Auth[authorization] + Consumer -. owns .-> Provider[retrieval or model providers] +``` + +- Naruon owns retrieval, source access, and its package-version upgrade. +- LineageWeave owns authorized lineage evidence and consumes released + RankWeave results; it does not own fusion arithmetic. +- TEPP and fast-mlsirm own psychometric measurement and estimation. RankWeave + may consume provenance-bearing policies but does not invent a theta or + reimplement those models. +- contextual-orchestrator owns LLM and model-routing decisions. + +The current release remains dependency-free Python. This development head +moves theoretical min-max normalization and unweighted RRF into the Rust core; +issue #45 tracks the remaining fusion and evaluation migration under ADR 0006. +Until a Rust-backed release is published and pinned, consumers must not claim +the engine is available to them. + +## 7. Explicit non-goals + +- Database, external search-service, HTTP, ORM, identity, or authorization + integration. RankWeave exact snapshots remain in-memory calculation objects + loaded from caller-owned persistence. +- Provider/model discovery, embedding generation, OCR, VISION, or LLM + orchestration. +- Hidden score normalization, generated weights, inferred folds, generated + candidate families, or automatic production deployment. +- Psychometric estimation, causal interpretation, or business-value scoring. +- A product UI; RankWeave supplies library, CLI, and machine-readable evidence + contracts. Consumer products own rendered interaction design. + +## 8. Release acceptance + +A release is acceptable only when one exact source head proves: + +1. all public vectors and edge cases pass under the documented semantics; +2. line, branch, and public-docstring coverage are complete; +3. Ruff, full tests, wheel build, isolated install, console/module entrypoints, + schema validation, and package-content checks pass; +4. research and ADR traceability matches every changed numerical contract; +5. CHANGELOG and synchronized version metadata describe the same artifact; +6. governed protected-branch review and immutable publication evidence are + complete; and +7. each consumer separately upgrades to the released version or immutable + source pin before claiming the capability. + +## 9. Current product gaps + +The evidence ledger and prioritized acceptance gaps live in +[`docs/product-technical-gap-baseline.md`](product-technical-gap-baseline.md). +In particular, issue #45 must preserve this contract while replacing duplicate +consumer arithmetic with one Rust-backed production engine; it must not use the +migration to add local policies or unsupported performance claims. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md new file mode 100644 index 0000000..b51c5c3 --- /dev/null +++ b/docs/product-technical-gap-baseline.md @@ -0,0 +1,268 @@ +# RankWeave product/technical gap baseline + +The product scope and release acceptance contract are defined in +[`docs/product-requirements.md`](product-requirements.md). This file is the +current evidence ledger; it does not replace the PRD. + +Initial status snapshot as of 2026-08-22, with a mitigation-status update on +2026-08-23 (§2). This document exists so a reviewer can answer one question +without re-deriving it from scratch: **what does a RankWeave user still +not get today, and what is already closed?** It is a living document — update +it whenever a §2 row's state changes, a §6 gap closes, or a new gap is found +(see Maintenance below). Do not let it drift from `gh pr list` / `gh issue +list` reality. + +## 1. Product identity and responsibility boundary + +RankWeave is a **leaf product**: a Python library and CLI with no third-party +Python runtime dependencies and one packaged Rust calculation core for hybrid-retrieval score fusion, +ranking evaluation, paired/family statistical comparison, offline policy +tuning (including caller-owned blocked-fold cross-validation and +availability-time backtesting), and strict TREC interchange +(ARCHITECTURE.md; AGENTS.md). It must run standalone and be swallowed whole as +a module by a host (Naruon today; LineageWeave as of this session) — the +"hub-and-leaf" composition documented in PR #40's README rewrite is the +supported integration shape, not an MSA violation. + +**What RankWeave is *not*:** a database, an embedding provider, a search +index, a benchmark-download service, or an HTTP service. It never calls a +network or a store. Adding any of those belongs in a consumer, not here +(AGENTS.md, "Hard rules"). + +## 2. Current PR/issue queue (evidence, not aspiration) + +Snapshot taken during this session's review→fix→checks→merge pass: + +| # | Title | State entering session | Action taken this session | +|---|---|---|---| +| PR #39 | docs: make README operator-first and honest about PyPI | BLOCKED, no review decision | **Closed**, superseded by #40 — same PyPI-honest install fix, broader restructuring | +| PR #40 | Rewrite RankWeave README for customers and operators | BLOCKED, CHANGES_REQUESTED, 2 checks FAILURE (transient GitHub 503 on 2026-08-17 18:17–18:18 UTC) | Fixed the install-order defect and a forward-reference gap (post-0.1.0 API examples above the caveat explaining they need the git install); zero unresolved review threads; all checks green; **blocked only on org-wide review-dispatch throughput (see below)** | +| PR #36 | fix(ci): restore executable hourly governance | BLOCKED, CHANGES_REQUESTED (stale, predating `dismiss_stale_reviews_on_push`) | Verified both issue #37 defects are correctly fixed in-branch; fixed a real doc-structure bug (orphaned heading) found by review; zero unresolved review threads; all checks green; **blocked only on org-wide review-dispatch throughput (see below)** | +| PR #41 | docs: add product/technical gap baseline (this document) | — (opened this session) | Fixed a wording-precision defect found by review; zero unresolved review threads; all checks green; **blocked only on org-wide review-dispatch throughput (see below)** | +| PR #42 | docs: add ADR 0005 versioned public-API compatibility policy | — (opened this session) | Freezes `rankweave.__all__` as of 0.18.0 (§6 gap 2, below), enforced by `tests/test_public_api_compatibility.py`; zero unresolved review threads; all checks green; **blocked only on org-wide review-dispatch throughput (see below)** | +| Issue #38 | Disable orphaned release/PR-repair/hourly-loop workflow identities | Open | **Closed.** Disabled 24 orphaned workflow identities via the Actions API (`PUT .../workflows/{id}/disable`); verified only `ci.yml`, `create-release.yml`, `hourly-commercialization-loop.yml`, `publish.yml` (plus GitHub's own Dependabot/CodeQL dynamic entries) remain active | +| Issue #37 | Fleet automation incident: NVIDIA-before-queue-gate ordering + `secrets: inherit` | Open, fix in PR #36 | Left open pending #36 merge — do not close on a claim, close when the fix actually lands on `main` | +| Issue #35 | PyPI Trusted Publisher misconfigured for v0.18.0 (`invalid-publisher`) | Open | **Blocked-external.** Requires a PyPI project-owner action at `pypi.org/manage/project/rankweave/settings/publishing/` that no repository automation or GitHub API call can perform. Re-verified still blocked; documented here rather than silently dropped, per the standing rule against silently ignoring an external-only blocker. | + +### Org-wide review-dispatch throughput bottleneck (discovered this session) + +All four RankWeave PRs above reached "zero unresolved threads, all checks +green" during this session but stayed unmergeable because +`ContextualWisdomLab/.github`'s `pr-review-merge-scheduler.yml` +`org-queue-sweep` job enforces **one OpenCode review dispatch per 15-minute +sweep, shared across the entire organization** — not per repository. Evidence +(run `32556682284`, job `96991889320`, 2026-08-22T06:36Z): all three of +RankWeave's ready PRs logged `wait: ... review dispatch limit reached` in the +same sweep that also processed ThreadWeave and EgressWeave. Filed as +[ContextualWisdomLab/.github#1219](https://github.com/ContextualWisdomLab/.github/issues/1219) +with full evidence rather than unilaterally raising a shared, cost-relevant +throttle without visibility into its intended ceiling. This is an +organization-scale gap, not RankWeave-specific — expect it to keep affecting +every repository's merge latency until resolved centrally. + +**Mitigation status (updated 2026-08-23):** four independent contributing +causes have since been identified and three merged in +`ContextualWisdomLab/.github`: rotation of the queue-sweep's repo walk order +(#1220, merged) so the shared 1-dispatch/15-minute budget no longer always +starves the same tail of repositories; an `actionlint` +`job.workflow_*`-context bug in a related scheduler workflow (#1221, merged); +a dead GitHub Models fallback chain in `strix.yml` that was blocking checks +with a hallucinated finding (#1226, merged). A fourth fix — rate-limit-aware +retry/defer for the shared GitHub App installation token that the sweep job +itself hits (#1245) — is open but not yet merged pending resolution of an +adversarial-review finding (a per-repo retry cost that can collide with the +sweep job's overall timeout under sustained contention). None of these have +yet flipped RankWeave's own PRs to mergeable as of this update; a separate, +unrelated cause (GitHub Models retirement affecting the review model pool, +`ContextualWisdomLab/.github#624`) may also still be contributing and remains +open. + +Re-run before trusting this table stale: `gh pr list --state open` and +`gh issue list --state open` against `ContextualWisdomLab/RankWeave`. + +## 3. LineageWeave reuse-boundary analysis + +The request that opened this session asked to find and prioritize +LineageWeave-related PRs. Verification (not assumption) was required first, +per the standing instruction to confirm the actual PR location by +product/responsibility boundary rather than by name: + +- The dependency direction is one-way: **LineageWeave depends on RankWeave**, + not the reverse. LineageWeave's `lineageweave/rankweave_client.py` fail-closes + (`RankWeaveNotAvailable`) if the package is missing. +- LineageWeave ADR 0225 now assigns fusion arithmetic and contribution evidence + to RankWeave and forbids a second LineageWeave engine. LineageWeave issue + #338 and PR #663 record the cross-repository consumer boundary. +- RankWeave issue #45 is the active owner work item for a Rust calculation core + behind the public Python contract. It requires provenance-bearing policies + and prohibits invented weights, thresholds, candidate windows, folds, and + missing-channel zeros. +- Existing documented numeric semantics remain research-traceable: Cormack et + al. (2009) ground RRF and Bruch et al. (2024) ground convex fusion. A paper's + support for unequal channel reliability does not establish a particular + consumer weight vector; exact weights require estimator provenance. + +**Conclusion:** the product boundary is explicit but not shipped end to end. +RankWeave must publish the Rust-backed owner contract, and each consumer must +upgrade its immutable pin before deleting duplicate arithmetic or claiming the +new engine. + +## 4. Capability inventory + +| Capability | Status | Evidence | +|---|---|---| +| Fuse scored/rank-only channels (TM2C2 convex, weighted RRF) | Shipped | `score_fusion.py`, `ranked_list_fusion.py`; Bruch et al. (2024), Cormack et al. (2009) | +| Evaluate rankings (precision/recall/RR/nDCG@k) | Shipped | `evaluation.py`; Järvelin & Kekäläinen (2002) | +| Paired statistical comparison with exact/Monte Carlo randomization | Shipped | `comparison.py`; Smucker et al. (2007) | +| Family-wise comparison with Holm correction | Shipped | `trec_family_comparison.py`; Holm (1979) | +| Caller-owned blocked-fold cross-validation (convex + weighted-RRF) | Shipped | `cross_validation.py`; ADR 0002 | +| Availability-time backtesting (no future-leakage) | Shipped | `temporal_backtesting.py`; ADR 0003 | +| Strict TREC interchange (qrels/runs) | Shipped | `trec.py`; NIST TREC guidance | +| Exact-byte artifact verification (opt-in v2 reports) | Shipped | `report_schemas.py`, `docs/artifact-verification.md` | +| Governed, provenance-attested PyPI release | **Broken today** | Issue #35 — Trusted Publisher misconfigured, `0.18.0` unpublished | +| Honest, PyPI-accurate customer README | **In flight** | PR #40 (this session) | +| Public API stability guarantee for external consumers (LineageWeave, Naruon) | Implemented, unreleased | ADR 0005 defines the versioned package-root and CLI transport contracts; `tests/test_public_api_compatibility.py` enforces them. Issue #35 still prevents publishing this source contract for consumers. | + +## 5. TRD-lite — technical contract summary + +- **Runtime:** Python 3.10+, no third-party Python runtime dependency, one + packaged Rust calculation core, Apache-2.0. +- **Public surface:** `FusionSettings`, `fuse_channel_scores`, + `weighted_convex_combination_score`, `weighted_convex_fuse`, + `weighted_reciprocal_rank_fuse`, `evaluate_rankings`, `compare_rankings`, + `compare_ranking_reports`, `cross_validate_weighted_convex_fusion`, + `cross_validate_weighted_reciprocal_rank_fusion`, + `rank_semantic_units`, + `tune_weighted_convex_fusion`, TREC parse/format/compare/compare-family, CLI + transports, and packaged JSON Schema (v1/v2) report contracts. +- **CLI transport:** `rankweave compare`, `rankweave compare-family`, + `rankweave verify-artifacts` — bounded (64 MiB/artifact), UTF-8 strict, + exit-code contract `0`/`1`/`2` (`docs/cli.md`). +- **Governance plane:** `create-release.yml` (verify, read-only) → + `contents: write` GitHub Release job → isolated `actions: write` dispatch of + `publish.yml` → OIDC PyPI Trusted Publishing. No stored registry credential + (`docs/releasing.md`, ADR 0004). + +## 6. Gap analysis, prioritized by user-visible leverage + +Severity: 🔴 blocks a user today · 🟡 user-visible friction · 🟢 hardening/roadmap. + +1. 🔴 **PyPI publication is broken** (issue #35). A user who reads the README + and runs `pip install rankweave` gets `0.1.0`, while the reviewed source + tree and GitHub Release are `0.18.0` — the published package is missing + every capability shipped since 0.1.0 (cross-validation, temporal + backtesting, artifact verification v2, weighted-RRF cross-validation) with + no version-number relationship between "0.1.0" and "0.18.0" beyond "older + and newer." This is the single highest-leverage gap: it is not a code gap, + it is a configuration action blocked outside this repository. **Next action:** + surface this to whoever holds the PyPI org owner role; nothing further is + automatable from here. +2. 🟡 **The versioned public-API policy is not released.** ADR 0005 and + `tests/test_public_api_compatibility.py` now define and enforce the + package-root and CLI transport compatibility contracts in source. Naruon + and LineageWeave still cannot consume that work as a published contract + while issue #35 blocks the next PyPI release. **Recommendation:** publish + the exact verified release after the external Trusted Publisher + configuration is repaired; do not represent a source-only commit as an + available consumer version. +3. 🟡 **Workflow-identity lifecycle has no self-cleaning step** (root cause + behind issue #38, now remediated once). Every future one-shot + PR-repair/finalizer workflow will re-accumulate orphaned identities unless + its own bounded-use teardown calls the disable endpoint. **Recommendation:** + add a `close-empty`-style final step to the pattern these one-shot + workflows already follow, or a periodic sweep job, so this doesn't recur + as a fresh fleet incident every few weeks. +4. 🟢 **Central `pr-review-fix-scheduler.yml` is reachable again.** During + this session's investigation of PR #36, `ContextualWisdomLab/.github`'s + `pr-review-fix-scheduler.yml` at a fresh commit was confirmed to exist and + be resolvable (the old pinned commit `21397126…` is still unreachable — + 201 commits behind, 6 ahead, genuinely diverged — so PR #36's fail-closed + local hold job remains correct and necessary as-is). Restoring the full + central repair call with a new reachable pin is a legitimate follow-up, + but it is new scope requiring its own test-first change per AGENTS.md, not + something to fold into #36's already-reviewed diff. **Recommendation:** + file a follow-up issue tracking the re-pin once the central scheduler's + current commit is confirmed stable, rather than merging it unreviewed + inside an unrelated PR. +5. 🟡 **The first production fusion primitives are Rust-backed; migration is + incomplete** (issue #45). + LineageWeave ADR 0225 names RankWeave as the sole fusion owner, while the + development head now routes theoretical normalization, two-channel convex + fusion, and unweighted RRF through `rankweave-core`; N-channel weighted, + evaluation, comparison, and tuning arithmetic still execute in Python. The + remaining migration keeps + the public Python surface as an adapter over one Rust core, preserves + exact documented semantics, and publishes provenance and limitations with + every calculation envelope. CPU multithreading and an optional GPU path + require exact-workload parity and benchmark evidence; no throughput claim + is available yet. The migration must not add an inferred policy, threshold, + candidate window, fold, or channel weight. +6. 🟢 **Multilevel/temporal modeling mandate — partially inapplicable, partially + already shipped.** RankWeave fuses and evaluates rankings; it does not fit + respondents to latent traits, so the atomistic-fallacy multilevel/ + multiple-membership concern (which governs person-level psychometric + estimation) does not have a natural target inside this repository — that + concern belongs to fast-mlsirm/TEPP, which do estimate latent parameters + from nested data. The **temporal** half of the mandate is already shipped + here: `temporal_backtesting.py` (ADR 0003) enforces availability-time + windows and forbids future-leaking assessment queries. No gap to close on + this axis beyond keeping ADR 0003's guarantees intact. + +## 7. UI/UX, Storybook, and accessibility scope note + +RankWeave has no frontend application — it is a library and CLI. The +`ui-ux-pro-max` / `anti-ui-slop` / Storybook / e2e-testing instruction from +the parent mandate is scoped here to the only customer-facing "surface" that +exists: README/CLI-output presentation quality and documentation structure +(now addressed for PyPI honesty in PR #40), not a component library. If a +RankWeave-adjacent frontend surface is ever added (unlikely given the leaf +product boundary in §1), this section should be replaced with a real +Storybook inventory, design-token audit, and the full accessibility/ +interaction/performance/typography/animation/forms/navigation/charts +checklist from the parent mandate. Until then, forcing that checklist onto a +library with no UI would be inventing scope the product does not have. + +## 8. References (APA 7th) + +Bruch, S., Nardini, F. M., Rulli, C., & Venturini, R. (2024). Efficient +and effective tree-based and neural learning to rank. *Foundations and Trends +in Information Retrieval*, 17(1), 1–123. + +Cormack, G. V., Clarke, C. L. A., & Buettcher, S. (2009). Reciprocal rank +fusion outperforms condorcet and individual rank learning methods. In +*Proceedings of the 32nd international ACM SIGIR conference on Research and +development in information retrieval* (pp. 758–759). ACM. +https://doi.org/10.1145/1571941.1572114 + +Holm, S. (1979). A simple sequentially rejective multiple test procedure. +*Scandinavian Journal of Statistics*, 6(2), 65–70. + +Järvelin, K., & Kekäläinen, J. (2002). Cumulated gain-based evaluation of IR +techniques. *ACM Transactions on Information Systems*, 20(4), 422–446. +https://doi.org/10.1145/582415.582418 + +Samuel, S., DeGenaro, D., Guallar-Blasco, J., Sanders, K., Eisape, O., +Spendlove, T., Reddy, A., Martin, A., Yates, A., Yang, E., Carpenter, C., +Etter, D., Kayi, E., Wiesner, M., Murray, K., & Kriz, R. (2025). MMMORRF: +Multimodal multilingual modularized reciprocal rank fusion. In *Proceedings of +the 48th International ACM SIGIR Conference on Research and Development in +Information Retrieval* (pp. 4004–4009). Association for Computing Machinery. +https://doi.org/10.1145/3726302.3730157 + +Smucker, M. D., Allan, J., & Carterette, B. (2007). A comparison of +statistical significance tests for information retrieval evaluation. In +*Proceedings of the sixteenth ACM conference on Conference on information and +knowledge management* (pp. 623–632). ACM. +https://doi.org/10.1145/1321440.1321528 + +Complete references, including standards documents (SLSA v1.2, FIPS 180-4, +RFC 8259, JSON Schema Draft 2020-12), remain in `docs/research/README.md` — +this section does not duplicate that index, only the sources cited directly +in §§3 and 4 above. + +## Maintenance + +Update this file in the same PR that changes the state of any row in §2, or +in the next PR after a gap in §6 is closed. A stale gap-baseline document is +worse than none — it makes a real product look worse than it is or hides a +real defect behind a claimed fix. diff --git a/docs/releasing.md b/docs/releasing.md index 171c611..12cc684 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -111,7 +111,7 @@ A failed re-publication is not silently skipped. PyPI versions are immutable; co Download the exact wheel and source distribution from PyPI before verifying them. An authorized repository operator may alternatively download the short-lived `rankweave-distributions` artifact from the successful publication workflow run during its retention window. Verify GitHub's build provenance against this repository: ```bash -gh attestation verify path/to/rankweave-0.18.0-py3-none-any.whl \ +gh attestation verify path/to/rankweave-0.18.0-cp310-abi3-PLATFORM.whl \ --repo ContextualWisdomLab/RankWeave gh attestation verify path/to/rankweave-0.18.0.tar.gz \ diff --git a/docs/research/README.md b/docs/research/README.md index d29e54a..d12ca07 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -333,6 +333,27 @@ requirement independently of locale-specific text-stream encodings. ## References (APA 7th edition) +### Rust calculation and numerical execution boundary + +IEEE Computer Society. (2019). *IEEE standard for floating-point arithmetic* +(IEEE Std 754-2019). IEEE. https://doi.org/10.1109/IEEESTD.2019.8766229 + +Higham, N. J. (2002). *Accuracy and stability of numerical algorithms* (2nd +ed.). Society for Industrial and Applied Mathematics. +https://doi.org/10.1137/1.9780898718027 + +NVIDIA Corporation. (2026). *CUDA C++ best practices guide* (Version 13.3). +https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/ + +PyO3 Project. (2026). *Building and distribution*. PyO3 user guide. +https://pyo3.rs/main/building-and-distribution.html + +PyO3 Project. (2026). *Features reference*. PyO3 user guide. +https://pyo3.rs/main/features + +Rayon Developers. (2026). *ParallelIterator*. Rayon 1.12.0 documentation. +https://docs.rs/rayon/1.12.0/rayon/iter/trait.ParallelIterator.html + Bray, T. (Ed.). (2017). *The JavaScript Object Notation (JSON) Data Interchange Format* (RFC 8259). RFC Editor. https://doi.org/10.17487/RFC8259 diff --git a/docs/superpowers/specs/2026-08-04-nim-commercialization-loop-design.md b/docs/superpowers/specs/2026-08-04-nim-commercialization-loop-design.md index 241e500..b0550df 100644 --- a/docs/superpowers/specs/2026-08-04-nim-commercialization-loop-design.md +++ b/docs/superpowers/specs/2026-08-04-nim-commercialization-loop-design.md @@ -39,8 +39,10 @@ and **GitHub mutation** into explicit trust zones. 8. Commit the verified red state locally so model fallback can reset to a known tree without pushing anything. 9. Run an **implementation phase** with the same no-execution and no-web - permissions. Edits are allowed in normal product and documentation files but - denied under `.github/`, `.git/`, and agent-control files. + permissions. Edits are allowed in Python product and documentation files but + denied under `.github/`, `.git/`, agent-control files, `crates/`, Cargo + manifests, and Python build metadata. Native changes require a separate + maintainer-authored and reviewed pull request. 10. Apply a deterministic diff gate: text files only, no symlinks or submodules, no protected paths, no rename/copy/conflict state, bounded file count, bounded individual/aggregate bytes, and at least one production Python diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..19305ad --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +rust = "1.97.1" diff --git a/pyproject.toml b/pyproject.toml index 655a6c3..81a38d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [build-system] -requires = ["hatchling==1.31.0"] -build-backend = "hatchling.build" +requires = ["maturin==1.14.1"] +build-backend = "maturin" [project] name = "rankweave" version = "0.18.0" -description = "Dependency-free retrieval fusion, evaluation, family-wise statistical comparison, policy tuning, TREC benchmarking, and auditable CLI workflows." +description = "Rust-backed retrieval fusion, evaluation, family-wise statistical comparison, policy tuning, TREC benchmarking, and auditable CLI workflows." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" @@ -41,7 +41,7 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Rust", "Topic :: Text Processing :: Indexing", "Topic :: Scientific/Engineering :: Information Analysis", "Typing :: Typed", @@ -68,8 +68,11 @@ dev = [ [tool.uv] required-version = "==0.12.1" -[tool.hatch.build.targets.wheel] -packages = ["src/rankweave"] +[tool.maturin] +manifest-path = "crates/rankweave-python/Cargo.toml" +module-name = "rankweave._rankweave_core" +python-source = "src" +include = ["CHANGELOG.md", "tests/test_version.py"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/scripts/verify_release_archives.py b/scripts/verify_release_archives.py new file mode 100644 index 0000000..0d2481d --- /dev/null +++ b/scripts/verify_release_archives.py @@ -0,0 +1,170 @@ +"""Verify that RankWeave release archives contain the governed native package.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from tarfile import open as open_tarfile +from zipfile import ZipFile + +REQUIRED_WHEEL_MEMBERS = frozenset( + { + "rankweave/__init__.py", + "rankweave/__main__.py", + "rankweave/_rankweave_core.pyi", + "rankweave/_validation.py", + "rankweave/artifact_verification.py", + "rankweave/cli.py", + "rankweave/comparison.py", + "rankweave/cross_validation.py", + "rankweave/evaluation.py", + "rankweave/py.typed", + "rankweave/query_normalization.py", + "rankweave/ranked_list_fusion.py", + "rankweave/report_schemas.py", + "rankweave/schemas/__init__.py", + "rankweave/schemas/artifact-verification-v1.schema.json", + "rankweave/schemas/trec-comparison-v1.schema.json", + "rankweave/schemas/trec-comparison-v2.schema.json", + "rankweave/schemas/trec-family-comparison-v1.schema.json", + "rankweave/schemas/trec-family-comparison-v2.schema.json", + "rankweave/score_fusion.py", + "rankweave/semantic_index.py", + "rankweave/semantic_vector_ranking.py", + "rankweave/temporal_backtesting.py", + "rankweave/trec.py", + "rankweave/trec_comparison.py", + "rankweave/trec_family_comparison.py", + "rankweave/tuning.py", + } +) + +REQUIRED_SOURCE_MEMBERS = frozenset( + { + "Cargo.lock", + "Cargo.toml", + "CHANGELOG.md", + "LICENSE", + "README.md", + "crates/rankweave-core/Cargo.toml", + "crates/rankweave-core/src/lib.rs", + "crates/rankweave-core/src/semantic_index.rs", + "crates/rankweave-python/Cargo.toml", + "crates/rankweave-python/src/lib.rs", + "pyproject.toml", + "src/rankweave/__init__.py", + "src/rankweave/_rankweave_core.pyi", + "src/rankweave/semantic_index.py", + "src/rankweave/semantic_vector_ranking.py", + "tests/test_version.py", + } +) + + +def verify_release_archives( + dist_dir: Path, + version: str, + *, + expected_wheel_tags: tuple[str, ...], + require_sdist: bool, +) -> None: + """Fail unless release archives and their governed members are complete.""" + wheels = tuple(sorted(dist_dir.glob("rankweave-*.whl"))) + expected_wheel_count = len(expected_wheel_tags) + if len(wheels) != expected_wheel_count: + raise ValueError( + f"release requires exactly {expected_wheel_count} wheel(s); " + f"found {len(wheels)}" + ) + + expected_prefix = f"rankweave-{version}-cp310-abi3-" + wheel_names = tuple(wheel.name for wheel in wheels) + for wheel_name in wheel_names: + if not wheel_name.startswith(expected_prefix): + raise ValueError(f"unexpected stable-ABI wheel name: {wheel_name!r}") + for required_tag in expected_wheel_tags: + matches = tuple(name for name in wheel_names if required_tag in name) + if len(matches) != 1: + raise ValueError( + f"release requires exactly one {required_tag!r} wheel; " + f"found {len(matches)}" + ) + + for wheel in wheels: + with ZipFile(wheel) as wheel_file: + wheel_members = set(wheel_file.namelist()) + missing_wheel_members = REQUIRED_WHEEL_MEMBERS - wheel_members + if missing_wheel_members: + raise ValueError( + f"wheel is missing: {sorted(missing_wheel_members)!r}" + ) + expected_extension = ".pyd" if "-win_" in wheel.name else ".so" + if not any( + member.startswith("rankweave/_rankweave_core.") + and member.endswith(expected_extension) + for member in wheel_members + ): + raise ValueError( + "wheel is missing the compiled RankWeave core for its platform" + ) + + source_distributions = tuple(sorted(dist_dir.glob("rankweave-*.tar.gz"))) + expected_sdist_count = int(require_sdist) + if len(source_distributions) != expected_sdist_count: + raise ValueError( + f"release requires exactly {expected_sdist_count} source " + f"distribution(s); found {len(source_distributions)}" + ) + if not require_sdist: + return + + source_distribution = source_distributions[0] + expected_sdist_name = f"rankweave-{version}.tar.gz" + if source_distribution.name != expected_sdist_name: + raise ValueError( + f"unexpected source distribution name: {source_distribution.name!r}" + ) + source_root = f"rankweave-{version}/" + with open_tarfile(source_distribution, "r:gz") as archive: + source_members = set(archive.getnames()) + required_source_members = { + source_root + member for member in REQUIRED_SOURCE_MEMBERS + } + missing_source_members = required_source_members - source_members + if missing_source_members: + raise ValueError( + f"source distribution is missing: {sorted(missing_source_members)!r}" + ) + + +def _parse_arguments() -> argparse.Namespace: + """Parse the release-archive verification command line.""" + parser = argparse.ArgumentParser() + parser.add_argument("--dist-dir", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument( + "--wheel-tag", + action="append", + default=[], + help="required unique substring in one stable-ABI wheel filename", + ) + parser.add_argument("--require-sdist", action="store_true") + return parser.parse_args() + + +def main() -> None: + """Run release-archive verification from the command line.""" + arguments = _parse_arguments() + try: + verify_release_archives( + arguments.dist_dir, + arguments.version, + expected_wheel_tags=tuple(arguments.wheel_tag), + require_sdist=arguments.require_sdist, + ) + except ValueError as error: + raise SystemExit(str(error)) from error + + +if __name__ == "__main__": + main() diff --git a/src/rankweave/__init__.py b/src/rankweave/__init__.py index 24a700a..402d1f0 100644 --- a/src/rankweave/__init__.py +++ b/src/rankweave/__init__.py @@ -1,11 +1,11 @@ """rankweave — retrieval fusion, evaluation, comparison, and tuning. -Pure-Python (stdlib-only) fusion of lexical, semantic, learned-sparse, -and other retrieval channels, complete-list fusion, ranked-effectiveness -evaluation, paired and family-wise statistical comparison, offline weight -policy tuning, strict TREC interchange, and Unicode NFC query normalization. -Store-agnostic: bring your own channels; rankweave combines and evaluates their -evidence. +Python adapters over one Rust calculation core for lexical, semantic, +learned-sparse, and other retrieval channels, complete-list fusion, +ranked-effectiveness evaluation, paired and family-wise statistical comparison, +offline weight policy tuning, strict TREC interchange, and Unicode NFC query +normalization. Store-agnostic: bring your own channels; rankweave combines and +evaluates their evidence. Two fusion strategies, research-grounded (see ``docs/research/``): @@ -107,6 +107,18 @@ weighted_convex_combination_score, weighted_reciprocal_rank_fusion_score, ) +from rankweave.semantic_index import ( + SemanticIndexRankingReport, + SemanticIndexSnapshotEvidence, + SemanticUnitExactIndex, +) +from rankweave.semantic_vector_ranking import ( + SemanticUnitCandidate, + SemanticUnitRank, + SemanticUnitRankingReport, + rank_semantic_units, + rank_semantic_units_packed, +) from rankweave.temporal_backtesting import ( WeightedConvexBacktestReport, WeightedConvexBacktestWindow, @@ -184,6 +196,12 @@ "RankingComparisonReport", "RankingEvaluationReport", "RankingMetrics", + "SemanticUnitCandidate", + "SemanticIndexRankingReport", + "SemanticIndexSnapshotEvidence", + "SemanticUnitExactIndex", + "SemanticUnitRank", + "SemanticUnitRankingReport", "ReportSchemaDescriptor", "SUPPORTED_COMPARISON_ALTERNATIVES", "SUPPORTED_COMPARISON_METRICS", @@ -232,6 +250,8 @@ "parse_trec_run", "reciprocal_rank_fuse", "reciprocal_rank_fusion_score", + "rank_semantic_units", + "rank_semantic_units_packed", "theoretical_min_max_normalize", "tune_weighted_convex_fusion", "tune_weighted_reciprocal_rank_fusion", diff --git a/src/rankweave/_rankweave_core.pyi b/src/rankweave/_rankweave_core.pyi new file mode 100644 index 0000000..d54ab21 --- /dev/null +++ b/src/rankweave/_rankweave_core.pyi @@ -0,0 +1,152 @@ +"""Static types for the packaged RankWeave Rust extension.""" + + +def theoretical_min_max_normalize( + score: float, + lower: float, + upper: float, +) -> float: ... + + +def convex_combination_score( + semantic_score: float | None, + lexical_score: float | None, + semantic_weight_alpha: float, +) -> float: ... + + +def reciprocal_rank_fusion_score( + ranks: list[int], + rank_constant_eta: int, +) -> float: ... + + +def rank_semantic_units( + query_vector: list[float], + candidates: list[tuple[str, str, list[float]]], +) -> tuple[str, str, str, int, list[tuple[str, str, float]]]: ... + + +def rank_semantic_units_packed( + query_vector: list[float], + candidate_ids: list[tuple[str, str]], + packed_vectors: bytes, +) -> tuple[str, str, str, int, list[tuple[str, str, float]]]: ... + + +class SemanticUnitIndex: + def __init__( + self, + snapshot_version: str, + model_identity: str, + vector_dimension: int, + candidate_ids: list[tuple[str, str]], + packed_vectors: bytes, + ) -> None: ... + + def snapshot_evidence( + self, + ) -> tuple[str, str, str, str, str, str, int, int]: ... + + def replace_snapshot( + self, + snapshot_version: str, + model_identity: str, + vector_dimension: int, + candidate_ids: list[tuple[str, str]], + packed_vectors: bytes, + ) -> None: ... + + def rank_authorized( + self, + model_identity: str, + query_vector: list[float], + authorized_candidate_ids: list[tuple[str, str]], + ) -> tuple[ + tuple[str, str, str, str, str, str, int, int], + str, + str, + int, + str, + str, + list[tuple[str, str, float]], + ]: ... + + def rank_authorized_packed( + self, + model_identity: str, + query_vector: list[float], + packed_authorization: bytes, + ) -> tuple[ + tuple[str, str, str, str, str, str, int, int], + str, + str, + int, + str, + str, + list[tuple[str, str, float]], + ]: ... + + def preflight_authorized_packed( + self, + model_identity: str, + packed_authorization: bytes, + ) -> tuple[ + tuple[str, str, str, str, str, str, int, int], + str, + str, + int, + str, + str, + list[tuple[str, str, float]], + ]: ... + + def preflight_authorized_top_k_packed( + self, + model_identity: str, + packed_authorization: bytes, + top_k: int, + ) -> tuple[ + tuple[str, str, str, str, str, str, int, int], + str, + str, + int, + str, + str, + list[tuple[str, str, float]], + ]: ... + + def rank_authorized_batch_packed( + self, + model_identity: str, + query_vectors: list[list[float]], + packed_authorization: bytes, + ) -> list[ + tuple[ + tuple[str, str, str, str, str, str, int, int], + str, + str, + int, + str, + str, + list[tuple[str, str, float]], + ] + ]: ... + + def rank_authorized_top_k_batch_packed( + self, + model_identity: str, + query_vectors: list[list[float]], + packed_authorization: bytes, + top_k: int, + ) -> list[ + tuple[ + tuple[str, str, str, str, str, str, int, int], + str, + str, + int, + str, + str, + list[tuple[str, str, float]], + ] + ]: ... diff --git a/src/rankweave/score_fusion.py b/src/rankweave/score_fusion.py index d75e193..4e5fcee 100644 --- a/src/rankweave/score_fusion.py +++ b/src/rankweave/score_fusion.py @@ -40,6 +40,7 @@ from collections.abc import Mapping from dataclasses import dataclass +from rankweave import _rankweave_core from rankweave._validation import ( _require_finite, _require_positive_integer, @@ -113,8 +114,9 @@ def theoretical_min_max_normalize( raise ValueError("bounds must be finite") from exc if upper_bound <= lower_bound: raise ValueError("bounds must satisfy upper > lower") - normalized = (score - lower_bound) / (upper_bound - lower_bound) - return min(1.0, max(0.0, normalized)) + return _rankweave_core.theoretical_min_max_normalize( + score, lower_bound, upper_bound + ) def convex_combination_score( @@ -134,11 +136,8 @@ def convex_combination_score( if lexical_score is not None: _require_unit_interval(lexical_score, "lexical_score") _require_unit_interval(semantic_weight_alpha, "semantic_weight_alpha") - semantic_component = semantic_score if semantic_score is not None else 0.0 - lexical_component = lexical_score if lexical_score is not None else 0.0 - return ( - semantic_weight_alpha * semantic_component - + (1.0 - semantic_weight_alpha) * lexical_component + return _rankweave_core.convex_combination_score( + semantic_score, lexical_score, semantic_weight_alpha ) @@ -183,13 +182,16 @@ def reciprocal_rank_fusion_score( validated_eta = _require_positive_integer( rank_constant_eta, "rank_constant_eta" ) - fused_score = 0.0 + validated_ranks = [] for channel_name, one_based_rank in channel_ranks.items(): - validated_rank = _require_positive_integer( - one_based_rank, f"rank for channel {channel_name!r}" + validated_ranks.append( + _require_positive_integer( + one_based_rank, f"rank for channel {channel_name!r}" + ) ) - fused_score += 1.0 / (validated_eta + validated_rank) - return fused_score + return _rankweave_core.reciprocal_rank_fusion_score( + validated_ranks, validated_eta + ) def weighted_reciprocal_rank_fusion_score( diff --git a/src/rankweave/semantic_index.py b/src/rankweave/semantic_index.py new file mode 100644 index 0000000..da8b0f0 --- /dev/null +++ b/src/rankweave/semantic_index.py @@ -0,0 +1,267 @@ +"""Typed adapter for immutable exact semantic-unit index snapshots.""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from rankweave import _rankweave_core +from rankweave.semantic_vector_ranking import SemanticUnitRank + + +@dataclass(frozen=True) +class SemanticIndexSnapshotEvidence: + """Integrity evidence for one immutable owner index snapshot.""" + + schema_version: str + snapshot_version: str + model_digest: str + dimension_digest: str + vectors_digest: str + snapshot_digest: str + vector_dimension: int + candidate_count: int + + +@dataclass(frozen=True) +class SemanticIndexRankingReport: + """Exact authorization-scoped ranking from one immutable snapshot.""" + + snapshot: SemanticIndexSnapshotEvidence + algorithm_version: str + execution_profile: str + worker_count: int + ordered_input_digest: str + output_digest: str + results: tuple[SemanticUnitRank, ...] + + +class SemanticUnitExactIndex: + """Own one atomically replaceable exact Rust index snapshot.""" + + def __init__( + self, + snapshot_version: str, + model_identity: str, + vector_dimension: int, + candidate_ids: Sequence[tuple[str, str]], + packed_vectors: bytes, + ) -> None: + """Build the initial immutable snapshot before exposing the index.""" + + self._native = _rankweave_core.SemanticUnitIndex( + snapshot_version, + model_identity, + vector_dimension, + list(candidate_ids), + packed_vectors, + ) + + @property + def snapshot_evidence(self) -> SemanticIndexSnapshotEvidence: + """Return integrity evidence for the currently active snapshot.""" + + return SemanticIndexSnapshotEvidence(*self._native.snapshot_evidence()) + + def replace_snapshot( + self, + snapshot_version: str, + model_identity: str, + vector_dimension: int, + candidate_ids: Sequence[tuple[str, str]], + packed_vectors: bytes, + ) -> None: + """Build fully, then atomically replace the active immutable snapshot.""" + + self._native.replace_snapshot( + snapshot_version, + model_identity, + vector_dimension, + list(candidate_ids), + packed_vectors, + ) + + def rank_authorized( + self, + model_identity: str, + query_vector: Sequence[float], + authorized_candidate_ids: Sequence[tuple[str, str]], + ) -> SemanticIndexRankingReport: + """Rank exactly and return no identity absent from caller authorization.""" + + ( + snapshot, + algorithm, + execution_profile, + worker_count, + input_digest, + output_digest, + rows, + ) = self._native.rank_authorized( + model_identity, + list(query_vector), + list(authorized_candidate_ids), + ) + return SemanticIndexRankingReport( + snapshot=SemanticIndexSnapshotEvidence(*snapshot), + algorithm_version=algorithm, + execution_profile=execution_profile, + worker_count=worker_count, + ordered_input_digest=input_digest, + output_digest=output_digest, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + + def rank_authorized_packed( + self, + model_identity: str, + query_vector: Sequence[float], + packed_authorization: bytes, + ) -> SemanticIndexRankingReport: + """Rank a canonical length-prefixed authorization byte buffer.""" + ( + snapshot, + algorithm, + execution_profile, + worker_count, + input_digest, + output_digest, + rows, + ) = self._native.rank_authorized_packed( + model_identity, + list(query_vector), + packed_authorization, + ) + return SemanticIndexRankingReport( + snapshot=SemanticIndexSnapshotEvidence(*snapshot), + algorithm_version=algorithm, + execution_profile=execution_profile, + worker_count=worker_count, + ordered_input_digest=input_digest, + output_digest=output_digest, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + + def preflight_authorized_packed( + self, + model_identity: str, + packed_authorization: bytes, + ) -> SemanticIndexRankingReport: + """Exercise exact owner scoring for one real authorization scope.""" + ( + snapshot, + algorithm, + execution_profile, + worker_count, + input_digest, + output_digest, + rows, + ) = self._native.preflight_authorized_packed( + model_identity, + packed_authorization, + ) + return SemanticIndexRankingReport( + snapshot=SemanticIndexSnapshotEvidence(*snapshot), + algorithm_version=algorithm, + execution_profile=execution_profile, + worker_count=worker_count, + ordered_input_digest=input_digest, + output_digest=output_digest, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + + def preflight_authorized_top_k_packed( + self, + model_identity: str, + packed_authorization: bytes, + top_k: int, + ) -> SemanticIndexRankingReport: + """Exercise the exact top-k profile for one real authorization scope.""" + ( + snapshot, + algorithm, + execution_profile, + worker_count, + input_digest, + output_digest, + rows, + ) = self._native.preflight_authorized_top_k_packed( + model_identity, packed_authorization, top_k + ) + return SemanticIndexRankingReport( + snapshot=SemanticIndexSnapshotEvidence(*snapshot), + algorithm_version=algorithm, + execution_profile=execution_profile, + worker_count=worker_count, + ordered_input_digest=input_digest, + output_digest=output_digest, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + + def rank_authorized_batch_packed( + self, + model_identity: str, + query_vectors: Sequence[Sequence[float]], + packed_authorization: bytes, + ) -> tuple[SemanticIndexRankingReport, ...]: + """Rank ordered queries against one identical packed authorization.""" + + reports = self._native.rank_authorized_batch_packed( + model_identity, + [list(query) for query in query_vectors], + packed_authorization, + ) + return tuple( + SemanticIndexRankingReport( + snapshot=SemanticIndexSnapshotEvidence(*snapshot), + algorithm_version=algorithm, + execution_profile=execution_profile, + worker_count=worker_count, + ordered_input_digest=input_digest, + output_digest=output_digest, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + for ( + snapshot, + algorithm, + execution_profile, + worker_count, + input_digest, + output_digest, + rows, + ) in reports + ) + + def rank_authorized_top_k_batch_packed( + self, + model_identity: str, + query_vectors: Sequence[Sequence[float]], + packed_authorization: bytes, + top_k: int, + ) -> tuple[SemanticIndexRankingReport, ...]: + """Return exact top-k reports with interval-safe owner acceleration.""" + + reports = self._native.rank_authorized_top_k_batch_packed( + model_identity, + [list(query) for query in query_vectors], + packed_authorization, + top_k, + ) + return tuple( + SemanticIndexRankingReport( + snapshot=SemanticIndexSnapshotEvidence(*snapshot), + algorithm_version=algorithm, + execution_profile=execution_profile, + worker_count=worker_count, + ordered_input_digest=input_digest, + output_digest=output_digest, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + for ( + snapshot, + algorithm, + execution_profile, + worker_count, + input_digest, + output_digest, + rows, + ) in reports + ) diff --git a/src/rankweave/semantic_vector_ranking.py b/src/rankweave/semantic_vector_ranking.py new file mode 100644 index 0000000..1b1232b --- /dev/null +++ b/src/rankweave/semantic_vector_ranking.py @@ -0,0 +1,80 @@ +"""Versioned semantic-unit cosine ranking backed by the Rust core.""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from rankweave import _rankweave_core + + +@dataclass(frozen=True) +class SemanticUnitCandidate: + """One caller-authorized semantic unit and its provider-produced vector.""" + + item_id: str + unit_id: str + vector: Sequence[float] + + +@dataclass(frozen=True) +class SemanticUnitRank: + """The highest-scoring semantic unit retained for one item.""" + + item_id: str + winning_unit_id: str + score: float + + +@dataclass(frozen=True) +class SemanticUnitRankingReport: + """Versioned ranking and exact ordered-input integrity evidence.""" + + schema_version: str + algorithm_version: str + ordered_input_digest: str + vector_dimension: int + results: tuple[SemanticUnitRank, ...] + + +def rank_semantic_units( + query_vector: Sequence[float], + candidates: Sequence[SemanticUnitCandidate], +) -> SemanticUnitRankingReport: + """Rank items by their best semantic-unit cosine without selecting a model.""" + + schema, algorithm, digest, dimension, rows = _rankweave_core.rank_semantic_units( + list(query_vector), + [ + (candidate.item_id, candidate.unit_id, list(candidate.vector)) + for candidate in candidates + ], + ) + return SemanticUnitRankingReport( + schema_version=schema, + algorithm_version=algorithm, + ordered_input_digest=digest, + vector_dimension=dimension, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) + + +def rank_semantic_units_packed( + query_vector: Sequence[float], + candidate_ids: Sequence[tuple[str, str]], + packed_vectors: bytes, +) -> SemanticUnitRankingReport: + """Rank canonical big-endian binary64 vectors without scalar expansion.""" + + schema, algorithm, digest, dimension, rows = ( + _rankweave_core.rank_semantic_units_packed( + list(query_vector), + list(candidate_ids), + packed_vectors, + ) + ) + return SemanticUnitRankingReport( + schema_version=schema, + algorithm_version=algorithm, + ordered_input_digest=digest, + vector_dimension=dimension, + results=tuple(SemanticUnitRank(*row) for row in rows), + ) diff --git a/tests/test_ci_supply_chain.py b/tests/test_ci_supply_chain.py index 516d16a..3ba0deb 100644 --- a/tests/test_ci_supply_chain.py +++ b/tests/test_ci_supply_chain.py @@ -28,7 +28,7 @@ def _references(workflow: str) -> tuple[tuple[str, str], ...]: def test_ci_pins_reviewed_node24_action_commits(): workflow = _workflow_text() - assert workflow.count(f"actions/checkout@{CHECKOUT_SHA}") == 2 + assert workflow.count(f"actions/checkout@{CHECKOUT_SHA}") == 3 assert workflow.count(f"actions/setup-python@{SETUP_PYTHON_SHA}") == 2 assert workflow.count(f"astral-sh/setup-uv@{SETUP_UV_SHA}") == 2 diff --git a/tests/test_create_release_workflow.py b/tests/test_create_release_workflow.py index d2ff95e..d41feb6 100644 --- a/tests/test_create_release_workflow.py +++ b/tests/test_create_release_workflow.py @@ -154,9 +154,11 @@ def test_verify_job_runs_complete_quality_gate_before_build_handoff(): ) positions = tuple(verify_block.index(command) for command in commands) assert positions == tuple(sorted(positions)) - assert "exactly one wheel and one source distribution" in verify_block - assert "rankweave-${version}-py3-none-any.whl" in verify_block - assert "rankweave-${version}.tar.gz" in verify_block + assert "rustup toolchain install 1.97.1" in verify_block + assert "scripts/verify_release_archives.py" in verify_block + assert '--version "$RELEASE_VERSION"' in verify_block + assert "--wheel-tag linux" in verify_block + assert "--require-sdist" in verify_block assert "Extract deterministic release notes" in verify_block assert "## [${version}]" in verify_block assert "name: rankweave-release-notes" in verify_block diff --git a/tests/test_fusion.py b/tests/test_fusion.py index 296b7b8..9d6d202 100644 --- a/tests/test_fusion.py +++ b/tests/test_fusion.py @@ -131,6 +131,12 @@ def test_rejects_invalid_rank_and_eta(self): with pytest.raises(ValueError): reciprocal_rank_fusion_score({"a": 1}, rank_constant_eta=0) + def test_preserves_unbounded_python_integer_contract(self): + rank = 2**64 + assert reciprocal_rank_fusion_score({"channel": rank}) == pytest.approx( + 1.0 / (60 + rank) + ) + class TestFusionSettings: def test_defaults_follow_research_grounding(self): diff --git a/tests/test_hourly_commercialization_workflow.py b/tests/test_hourly_commercialization_workflow.py index a1adc5f..b81f77d 100644 --- a/tests/test_hourly_commercialization_workflow.py +++ b/tests/test_hourly_commercialization_workflow.py @@ -148,7 +148,8 @@ def test_product_prompt_enforces_bounded_commercial_quality(): "exactly one highest-impact buyer-visible product gap", "write the failing", "full production docstrings", - "standard-library-only runtime", + "no-third-party Python runtime dependency", + "one Rust calculation core", "Update CHANGELOG.md", "Do not commit, push", "Figma is not applicable because RankWeave has no UI", @@ -171,16 +172,33 @@ def test_autonomous_diff_is_text_only_bounded_and_policy_safe(): for protected_path in ( '".gitmodules"', '"CODEOWNERS"', + '"pyproject.toml"', '"SECURITY.md"', '"AGENTS.md"', '".github/"', '".git/"', + '"crates/"', ): assert protected_path in workflow assert "non-regular file changed" in workflow assert "NUL byte found" in workflow assert "non-text or unsupported path changed" in workflow + assert '"crates/rankweave-core/src/"' not in workflow + assert '"crates/rankweave-python/src/"' not in workflow + assert '".rs"' not in workflow assert "must change a production Python module" in workflow + assert "This autonomous lane is Python-only" in workflow + + +def test_ignored_native_core_is_restored_from_trusted_copy_after_each_cleanup(): + workflow = _workflow_text() + + assert ( + "trusted editable install did not produce exactly one native core" in workflow + ) + assert workflow.count('git clean -fdX') == 2 + assert workflow.count('cp "$AUTOMATION_TRUSTED_NATIVE_CORE"') == 2 + assert workflow.count('sha256sum "$AUTOMATION_TRUSTED_NATIVE_CORE"') == 2 def test_untrusted_validation_has_no_network_or_inherited_environment(): @@ -198,6 +216,11 @@ def test_untrusted_validation_has_no_network_or_inherited_environment(): assert "python -m coverage report" in workflow assert "python -m pip wheel" in workflow assert "-m pip check" in workflow + assert "cargo +1.97.1 fetch --locked" in workflow + assert '"CARGO_NET_OFFLINE=true"' in workflow + assert '"CARGO_HOME=$SANDBOX_CARGO_HOME"' in workflow + assert '"CARGO_TARGET_DIR=$validation_home/cargo-target"' in workflow + assert '"RUSTC=$SANDBOX_RUST_TOOLCHAIN/bin/rustc"' in workflow def test_queue_and_base_are_checked_before_and_after_token_exchange(): diff --git a/tests/test_public_api_compatibility.py b/tests/test_public_api_compatibility.py new file mode 100644 index 0000000..ec4758b --- /dev/null +++ b/tests/test_public_api_compatibility.py @@ -0,0 +1,142 @@ +"""Enforce ADR 0005: the public API surface frozen at rankweave 0.18.0 stays exported. + +See docs/adr/0005-public-api-compatibility-policy.md. A name in this frozen +set is not removed or renamed within a minor version; removing one requires +updating this file, CHANGELOG.md's ``### Removed`` section, and the ADR in +the same reviewed change. Adding new public names does not require touching +this file — the assertion is a lower bound, not an exact match. +""" + +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility + import tomli as tomllib + +import rankweave + +FROZEN_PUBLIC_API_AT_0_18_0 = { + "AggregateRankingMetrics", + "ArtifactVerificationRecord", + "ArtifactVerificationReport", + "CANDIDATE_GREATER_ALTERNATIVE", + "CANDIDATE_LESS_ALTERNATIVE", + "CONVEX_COMBINATION_STRATEGY", + "COSINE_DISTANCE_THEORETICAL_BOUNDS", + "DEFAULT_MAX_QUERY_CHARACTER_LENGTH", + "DEFAULT_RANDOMIZATION_COUNT", + "DEFAULT_RANDOM_SEED", + "EXACT_RANDOMIZATION_METHOD", + "EXACT_RANDOMIZATION_PAIR_LIMIT", + "FAMILY_REPORT_SCHEMA_VERSION", + "FusedRankedItem", + "FusedScoredItem", + "FusedWeightedRankedItem", + "FusionSettings", + "MEAN_NDCG_OBJECTIVE", + "MEAN_PRECISION_OBJECTIVE", + "MEAN_RECALL_OBJECTIVE", + "MEAN_RECIPROCAL_RANK_OBJECTIVE", + "MONTE_CARLO_RANDOMIZATION_METHOD", + "NDCG_AT_K_METRIC", + "PAIRWISE_REPORT_SCHEMA_VERSION", + "PRECISION_AT_K_METRIC", + "PairedRandomizationResult", + "QueryMetricDifference", + "QueryRankingMetrics", + "RECALL_AT_K_METRIC", + "RECIPROCAL_RANK_AT_K_METRIC", + "RECIPROCAL_RANK_STRATEGY", + "RankingComparisonReport", + "RankingEvaluationReport", + "RankingMetrics", + "ReportSchemaDescriptor", + "SUPPORTED_COMPARISON_ALTERNATIVES", + "SUPPORTED_COMPARISON_METRICS", + "SUPPORTED_TUNING_OBJECTIVES", + "TWO_SIDED_ALTERNATIVE", + "TrecCandidateComparison", + "TrecQrelEntry", + "TrecQrels", + "TrecRun", + "TrecRunComparisonReport", + "TrecRunEntry", + "TrecRunFamilyComparisonReport", + "WORD_SIMILARITY_THEORETICAL_BOUNDS", + "WeightedChannelContribution", + "WeightedConvexBacktestReport", + "WeightedConvexBacktestWindow", + "WeightedConvexBacktestWindowDefinition", + "WeightedConvexCrossValidationFold", + "WeightedConvexCrossValidationReport", + "WeightedConvexTuningReport", + "WeightedConvexTuningTrial", + "WeightedRRFCrossValidationFold", + "WeightedRRFCrossValidationReport", + "WeightedRRFTuningReport", + "WeightedRRFTuningTrial", + "WeightedRankContribution", + "available_report_schemas", + "backtest_weighted_convex_fusion", + "compare_ranking_reports", + "compare_rankings", + "compare_trec_run_family", + "compare_trec_runs", + "convex_combination_score", + "cross_validate_weighted_convex_fusion", + "cross_validate_weighted_reciprocal_rank_fusion", + "evaluate_ranking", + "evaluate_rankings", + "evaluate_trec_run", + "format_trec_qrels", + "format_trec_run", + "fuse_channel_scores", + "load_report_schema", + "load_report_schema_text", + "normalize_search_text", + "parse_trec_qrels", + "parse_trec_run", + "reciprocal_rank_fuse", + "reciprocal_rank_fusion_score", + "theoretical_min_max_normalize", + "tune_weighted_convex_fusion", + "tune_weighted_reciprocal_rank_fusion", + "verify_report_artifacts", + "weighted_convex_combination_score", + "weighted_convex_fuse", + "weighted_reciprocal_rank_fuse", + "weighted_reciprocal_rank_fusion_score", +} + + +def test_frozen_0_18_0_public_api_remains_exported(): + """No frozen 0.18.0 name disappears from ``__all__`` within this minor version.""" + assert FROZEN_PUBLIC_API_AT_0_18_0 <= set(rankweave.__all__) + + +def test_frozen_0_18_0_public_api_remains_resolvable(): + """No name frozen at 0.18.0 becomes unimportable within this minor version.""" + for symbol_name in FROZEN_PUBLIC_API_AT_0_18_0: + assert hasattr(rankweave, symbol_name), symbol_name + + +def test_frozen_cli_entrypoint_remains_installed(): + """ADR 0005 keeps the documented console command mapped to its adapter.""" + + project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + assert project["project"]["scripts"]["rankweave"] == "rankweave.cli:main" + + +def test_frozen_cli_transport_versions_remain_available(): + """ADR 0005 keeps both established JSON transport versions discoverable.""" + + descriptors = rankweave.available_report_schemas() + versions = {descriptor.transport_schema_id for descriptor in descriptors} + assert { + "rankweave.artifact-verification.v1", + "rankweave.trec-comparison.v1", + "rankweave.trec-comparison.v2", + "rankweave.trec-family-comparison.v1", + "rankweave.trec-family-comparison.v2", + } <= versions diff --git a/tests/test_publish_workflow.py b/tests/test_publish_workflow.py index 51c7b53..f8ce9d0 100644 --- a/tests/test_publish_workflow.py +++ b/tests/test_publish_workflow.py @@ -71,29 +71,47 @@ def test_publish_workflow_pins_exact_current_actions(): references = _action_references(workflow_text) assert set(references) == set(EXPECTED_PUBLISH_ACTIONS.items()) - assert len(references) == 8 + assert len(references) == 15 def test_publish_workflow_separates_jobs_and_handoffs_one_artifact(): workflow_text = _publish_workflow() - build_block = _job_block(workflow_text, "build", "provenance") + build_block = _job_block(workflow_text, "build", "wheels") + wheels_block = _job_block(workflow_text, "wheels", "assemble") + assemble_block = _job_block(workflow_text, "assemble", "provenance") provenance_block = _job_block(workflow_text, "provenance", "publish") publish_block = _job_block(workflow_text, "publish", None) assert "needs:" not in build_block - assert "needs: build" in provenance_block - assert "needs: [build, provenance]" in publish_block + assert "needs: build" in wheels_block + assert "needs: [build, wheels]" in assemble_block + assert "needs: assemble" in provenance_block + assert "needs: [assemble, provenance]" in publish_block assert build_block.index("python -m coverage run -m pytest -q") < ( - build_block.index("uv build --wheel --sdist --out-dir dist") + build_block.index("uv build --sdist --out-dir dist") ) - assert "name: rankweave-distributions" in build_block + assert "runs-on: ${{ matrix.runner }}" in wheels_block + assert "runner: ubuntu-latest" in wheels_block + assert "runner: macos-14" in wheels_block + assert "runner: windows-latest" in wheels_block + assert "uvx --from maturin==1.14.1 maturin build" in wheels_block + assert workflow_text.count( + "ref: ${{ github.event.repository.default_branch }}" + ) == 3 + assert workflow_text.count("Revalidate and checkout released commit") == 2 + assert workflow_text.count( + 'git merge-base --is-ancestor "$RELEASE_SHA" "origin/$DEFAULT_BRANCH"' + ) == 2 + assert "ref: ${{ needs.build.outputs.release_sha }}" not in workflow_text + assert "compatibility: manylinux2014" in wheels_block + assert "name: rankweave-distributions" in assemble_block assert ( "path: |\n" " dist/\n" " release-handoff/SHA256SUMS" - ) in build_block - assert "if-no-files-found: error" in build_block - assert "include-hidden-files: false" in build_block + ) in assemble_block + assert "if-no-files-found: error" in assemble_block + assert "include-hidden-files: false" in assemble_block assert "retention-days: 7" in build_block assert "name: rankweave-distributions" in provenance_block assert "path: handoff/" in provenance_block @@ -104,22 +122,22 @@ def test_publish_workflow_separates_jobs_and_handoffs_one_artifact(): def test_distribution_handoff_is_checksum_verified_before_use(): workflow_text = _publish_workflow() - build_block = _job_block(workflow_text, "build", "provenance") + assemble_block = _job_block(workflow_text, "assemble", "provenance") provenance_block = _job_block(workflow_text, "provenance", "publish") publish_block = _job_block(workflow_text, "publish", None) assert ( "manifest_sha256: ${{ steps.distributions.outputs.manifest_sha256 }}" - in build_block + in assemble_block ) - assert ") > release-handoff/SHA256SUMS" in build_block - assert "sha256sum release-handoff/SHA256SUMS" in build_block - assert "manifest_sha256=%s" in build_block + assert ") > release-handoff/SHA256SUMS" in assemble_block + assert "sha256sum release-handoff/SHA256SUMS" in assemble_block + assert "manifest_sha256=%s" in assemble_block for job_block in (provenance_block, publish_block): assert "Verify immutable distribution handoff" in job_block assert ( "EXPECTED_MANIFEST_SHA256: " - "${{ needs.build.outputs.manifest_sha256 }}" + "${{ needs.assemble.outputs.manifest_sha256 }}" ) in job_block assert "handoff/release-handoff/SHA256SUMS" in job_block assert "sha256sum --check --strict -" in job_block @@ -135,9 +153,9 @@ def test_distribution_handoff_is_checksum_verified_before_use(): def test_build_job_checks_exact_release_identity_and_default_branch(): - build_block = _job_block(_publish_workflow(), "build", "provenance") + build_block = _job_block(_publish_workflow(), "build", "wheels") - assert "ref: ${{ inputs.release_sha || github.sha }}" in build_block + assert "ref: ${{ github.event.repository.default_branch }}" in build_block assert "fetch-depth: 0" in build_block assert "persist-credentials: false" in build_block for expected in ( @@ -161,10 +179,16 @@ def test_build_job_checks_exact_release_identity_and_default_branch(): "stable GitHub Release must not be a prerelease", ): assert expected in build_block + assert build_block.index("git merge-base --is-ancestor") < build_block.index( + 'git checkout --detach "$release_sha"' + ) def test_build_job_checks_package_version_and_complete_quality_gate(): - build_block = _job_block(_publish_workflow(), "build", "provenance") + workflow_text = _publish_workflow() + build_block = _job_block(workflow_text, "build", "wheels") + wheels_block = _job_block(workflow_text, "wheels", "assemble") + assemble_block = _job_block(workflow_text, "assemble", "provenance") assert '"$release_tag" != "v${version}"' in build_block assert "rankweave.__version__" in build_block @@ -173,16 +197,24 @@ def test_build_job_checks_package_version_and_complete_quality_gate(): assert "python -m ruff check ." in build_block assert "python -m coverage run -m pytest -q" in build_block assert "python -m coverage report" in build_block - assert "uv build --wheel --sdist --out-dir dist" in build_block - assert "release must contain exactly one wheel and one " in build_block - assert '"source distribution"' in build_block - assert "rankweave/schemas/artifact-verification-v1.schema.json" in build_block - assert "CHANGELOG.md" in build_block + assert "uv build --sdist --out-dir dist" in build_block + assert "uvx --from maturin==1.14.1 maturin build" in wheels_block + assert "Smoke-test built native wheel" in wheels_block + assert "pip install --no-index --find-links dist rankweave" in wheels_block + assert "from rankweave import SemanticUnitExactIndex" in wheels_block + assert '"$smoke_cli" --help' in wheels_block + assert "scripts/verify_release_archives.py" in build_block + assert "scripts/verify_release_archives.py" in wheels_block + assert "scripts/verify_release_archives.py" in assemble_block + assert "--wheel-tag manylinux --wheel-tag macosx" in assemble_block + assert "--wheel-tag win_amd64 --require-sdist" in assemble_block + assert build_block.count("rustup toolchain install 1.97.1") == 1 + assert wheels_block.count("rustup toolchain install 1.97.1") == 1 def test_release_jobs_use_least_privilege_and_protected_environment(): workflow_text = _publish_workflow() - build_block = _job_block(workflow_text, "build", "provenance") + build_block = _job_block(workflow_text, "build", "wheels") provenance_block = _job_block(workflow_text, "provenance", "publish") publish_block = _job_block(workflow_text, "publish", None) @@ -251,14 +283,31 @@ def test_normal_package_ci_builds_and_exercises_release_archives(): ci_workflow = _read_repository_file(".github/workflows/ci.yml") assert "uv build --wheel --sdist --out-dir dist" in ci_workflow - assert "Verify source distribution contents" in ci_workflow - assert "package job requires exactly one source distribution" in ci_workflow - assert 'source_root + "CHANGELOG.md"' in ci_workflow - assert 'source_root + "tests/test_version.py"' in ci_workflow + assert "Verify wheel and source distribution contents" in ci_workflow + assert "scripts/verify_release_archives.py" in ci_workflow + assert "--wheel-tag linux --require-sdist" in ci_workflow assert "Exercise release checksum handoff" in ci_workflow - assert "sha256sum *.whl *.tar.gz" in ci_workflow + assert "sha256sum ./*.whl ./*.tar.gz" in ci_workflow assert "release-handoff/SHA256SUMS" in ci_workflow assert "sha256sum --check --strict -" in ci_workflow assert "sha256sum --check --strict ../release-handoff/SHA256SUMS" in ( ci_workflow ) + + +def test_release_archive_verifier_has_one_complete_member_contract(): + verifier = _read_repository_file("scripts/verify_release_archives.py") + + for required_member in ( + "rankweave/_rankweave_core.pyi", + "rankweave/semantic_index.py", + "rankweave/semantic_vector_ranking.py", + "Cargo.lock", + "Cargo.toml", + "crates/rankweave-core/Cargo.toml", + "crates/rankweave-core/src/semantic_index.rs", + "crates/rankweave-python/Cargo.toml", + ): + assert required_member in verifier + assert "unexpected stable-ABI wheel name" in verifier + assert "wheel is missing the compiled RankWeave core" in verifier diff --git a/tests/test_semantic_index.py b/tests/test_semantic_index.py new file mode 100644 index 0000000..910bf23 --- /dev/null +++ b/tests/test_semantic_index.py @@ -0,0 +1,210 @@ +import ast +import struct +from pathlib import Path + +import pytest + +from rankweave import SemanticUnitExactIndex + + +def packed(*vectors: tuple[float, ...]) -> bytes: + return b"".join( + struct.pack(f">{len(vector)}d", *vector) for vector in vectors + ) + + +def packed_authorization(*identities: tuple[str, str]) -> bytes: + payload = [len(identities).to_bytes(8, "big")] + for identity in identities: + for value in identity: + encoded = value.encode("utf-8") + payload.extend((len(encoded).to_bytes(8, "big"), encoded)) + return b"".join(payload) + + +def exact_index(version: str = "snapshot-v1") -> SemanticUnitExactIndex: + return SemanticUnitExactIndex( + version, + "model-v1", + 2, + [("item-b", "unit-z"), ("item-a", "unit-z"), ("item-a", "unit-a")], + packed((1.0, 0.0), (1.0, 0.0), (0.0, 1.0)), + ) + + +def test_exact_index_returns_only_authorized_candidates() -> None: + index = exact_index() + + report = index.rank_authorized( + "model-v1", + [1.0, 0.0], + [("item-a", "unit-a")], + ) + + assert report.snapshot == index.snapshot_evidence + assert report.snapshot.schema_version == "rankweave.semantic-unit-index-snapshot.v1" + assert report.snapshot.vector_dimension == 2 + assert report.snapshot.candidate_count == 3 + assert report.execution_profile == "rankweave.semantic-unit-index.cpu-rayon.v1" + assert report.ordered_input_digest.startswith("sha256:") + assert report.output_digest.startswith("sha256:") + actual = [ + (row.item_id, row.winning_unit_id, row.score) for row in report.results + ] + assert actual == [("item-a", "unit-a", 0.0)] + + +def test_exact_index_packed_authorization_matches_row_transport() -> None: + index = exact_index() + identities = (("item-a", "unit-a"),) + + rows = index.rank_authorized("model-v1", [1.0, 0.0], identities) + packed_rows = index.rank_authorized_packed( + "model-v1", + [1.0, 0.0], + packed_authorization(*identities), + ) + + assert packed_rows == rows + + +def test_exact_index_preflights_one_real_packed_authorization_scope() -> None: + index = exact_index() + authorization = packed_authorization( + ("item-b", "unit-z"), ("item-a", "unit-z"), ("item-a", "unit-a") + ) + + report = index.preflight_authorized_packed("model-v1", authorization) + top_k = index.preflight_authorized_top_k_packed("model-v1", authorization, 1) + + assert report.snapshot == index.snapshot_evidence + assert {result.item_id for result in report.results} == {"item-a", "item-b"} + assert report.ordered_input_digest.startswith("sha256:") + assert report.output_digest.startswith("sha256:") + assert top_k.results == report.results[:1] + assert top_k.ordered_input_digest != report.ordered_input_digest + + +def test_exact_index_packed_batch_matches_independent_reports() -> None: + index = exact_index() + authorization = packed_authorization( + ("item-b", "unit-z"), ("item-a", "unit-z"), ("item-a", "unit-a") + ) + queries = ([1.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0]) + + batch = index.rank_authorized_batch_packed("model-v1", queries, authorization) + independent = tuple( + index.rank_authorized_packed("model-v1", query, authorization) + for query in queries + ) + + assert batch == independent + + +def test_exact_index_packed_batch_rejects_empty_queries() -> None: + with pytest.raises(ValueError, match="^empty_query_batch:"): + exact_index().rank_authorized_batch_packed( + "model-v1", [], packed_authorization(("item-a", "unit-a")) + ) + + +def test_exact_top_k_batch_matches_scalar_prefix_with_distinct_digest() -> None: + index = exact_index() + authorization = packed_authorization( + ("item-b", "unit-z"), ("item-a", "unit-z"), ("item-a", "unit-a") + ) + queries = ([1.0, 0.0], [0.0, 1.0]) + + top_k = index.rank_authorized_top_k_batch_packed( + "model-v1", queries, authorization, 1 + ) + full = index.rank_authorized_batch_packed("model-v1", queries, authorization) + + for top_k_report, full_report in zip(top_k, full, strict=True): + assert top_k_report.results == full_report.results[:1] + assert top_k_report.ordered_input_digest != full_report.ordered_input_digest + assert top_k_report.output_digest != full_report.output_digest + + +def test_exact_top_k_batch_rejects_zero_k() -> None: + with pytest.raises(ValueError, match="^empty_top_k:"): + exact_index().rank_authorized_top_k_batch_packed( + "model-v1", + [[1.0, 0.0]], + packed_authorization(("item-a", "unit-a")), + 0, + ) + + +def test_exact_index_replacement_is_atomic_after_validation() -> None: + index = exact_index() + + with pytest.raises(ValueError, match="^packed_vector_byte_length:"): + index.replace_snapshot( + "snapshot-v2", + "model-v1", + 2, + [("item", "unit")], + b"short", + ) + assert index.snapshot_evidence.snapshot_version == "snapshot-v1" + + replacement = exact_index("snapshot-v2") + evidence = replacement.snapshot_evidence + index.replace_snapshot( + evidence.snapshot_version, + "model-v1", + 2, + [("item-b", "unit-z"), ("item-a", "unit-z"), ("item-a", "unit-a")], + packed((1.0, 0.0), (1.0, 0.0), (0.0, 1.0)), + ) + assert index.snapshot_evidence.snapshot_version == "snapshot-v2" + + +@pytest.mark.parametrize( + ("model", "query", "authorization", "code"), + [ + ("other-model", [1.0, 0.0], [("item-a", "unit-a")], "model_mismatch"), + ("model-v1", [1.0], [("item-a", "unit-a")], "dimension_mismatch"), + ("model-v1", [1.0, 0.0], [], "empty_authorization"), + ( + "model-v1", + [1.0, 0.0], + [("missing", "unit")], + "unknown_authorized_candidate", + ), + ], +) +def test_exact_index_fails_closed( + model: str, + query: list[float], + authorization: list[tuple[str, str]], + code: str, +) -> None: + with pytest.raises(ValueError, match=f"^{code}:") as raised: + exact_index().rank_authorized(model, query, authorization) + assert str(raised.value).count(code) == 1 + assert "exact semantic index rejected input" in str(raised.value) + + +def test_native_stub_declares_every_packed_scope_operation() -> None: + stub = ast.parse( + (Path(__file__).parents[1] / "src/rankweave/_rankweave_core.pyi").read_text( + encoding="utf-8" + ) + ) + index_class = next( + node + for node in stub.body + if isinstance(node, ast.ClassDef) and node.name == "SemanticUnitIndex" + ) + methods = { + node.name for node in index_class.body if isinstance(node, ast.FunctionDef) + } + assert { + "rank_authorized_packed", + "preflight_authorized_packed", + "preflight_authorized_top_k_packed", + "rank_authorized_batch_packed", + "rank_authorized_top_k_batch_packed", + } <= methods diff --git a/tests/test_semantic_vector_ranking.py b/tests/test_semantic_vector_ranking.py new file mode 100644 index 0000000..11f933d --- /dev/null +++ b/tests/test_semantic_vector_ranking.py @@ -0,0 +1,103 @@ +import math +import struct + +import pytest + +from rankweave import ( + SemanticUnitCandidate, + rank_semantic_units, + rank_semantic_units_packed, +) + + +def test_semantic_units_return_versioned_winning_unit_evidence() -> None: + report = rank_semantic_units( + [1.0, 0.0], + [ + SemanticUnitCandidate("item-b", "unit-z", [1.0, 0.0]), + SemanticUnitCandidate("item-a", "unit-z", [1.0, 0.0]), + SemanticUnitCandidate("item-c", "unit-b", [-1.0, 0.0]), + SemanticUnitCandidate("item-c", "unit-a", [0.0, 1.0]), + ], + ) + + assert report.schema_version == "rankweave.semantic-unit-ranking.v1" + assert report.algorithm_version == "rankweave.semantic-unit-cosine.v1" + assert report.ordered_input_digest.startswith("sha256:") + assert report.vector_dimension == 2 + assert [ + (row.item_id, row.winning_unit_id, row.score) for row in report.results + ] == [ + ("item-a", "unit-z", 1.0), + ("item-b", "unit-z", 1.0), + ("item-c", "unit-a", 0.0), + ] + + +def test_packed_semantic_units_preserve_exact_report_and_digest() -> None: + """Packed binary64 transport is identical to the scalar public contract.""" + + query = [1.0, 0.0] + candidate_ids = [("item-b", "unit-z"), ("item-a", "unit-z")] + vectors = [[1.0, 0.0], [0.0, 1.0]] + scalar_report = rank_semantic_units( + query, + [ + SemanticUnitCandidate(item_id, unit_id, vector) + for (item_id, unit_id), vector in zip(candidate_ids, vectors, strict=True) + ], + ) + + packed_report = rank_semantic_units_packed( + query, + candidate_ids, + b"".join(struct.pack(">2d", *vector) for vector in vectors), + ) + + assert packed_report == scalar_report + + +def test_packed_semantic_units_reject_wrong_byte_length() -> None: + """Packed transport never pads or truncates a malformed vector payload.""" + + with pytest.raises(ValueError, match="^packed_vector_byte_length:"): + rank_semantic_units_packed([1.0, 0.0], [("item", "unit")], b"short") + + +@pytest.mark.parametrize( + ("query", "candidates", "error_code"), + [ + ([], [SemanticUnitCandidate("item", "unit", [1.0])], "empty_query_vector"), + ([1.0], [], "empty_candidates"), + ( + [math.inf], + [SemanticUnitCandidate("item", "unit", [1.0])], + "non_finite_vector", + ), + ( + [1.0], + [SemanticUnitCandidate("item", "unit", [1.0, 2.0])], + "dimension_mismatch", + ), + ( + [0.0], + [SemanticUnitCandidate("item", "unit", [1.0])], + "zero_norm_vector", + ), + ( + [1.0], + [ + SemanticUnitCandidate("item", "unit", [1.0]), + SemanticUnitCandidate("item", "unit", [1.0]), + ], + "duplicate_candidate", + ), + ], +) +def test_semantic_unit_failures_include_stable_codes( + query: list[float], + candidates: list[SemanticUnitCandidate], + error_code: str, +) -> None: + with pytest.raises(ValueError, match=f"^{error_code}:"): + rank_semantic_units(query, candidates) diff --git a/tests/test_temporal_release_contract.py b/tests/test_temporal_release_contract.py index e4e0cb3..a9ff68f 100644 --- a/tests/test_temporal_release_contract.py +++ b/tests/test_temporal_release_contract.py @@ -3,6 +3,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] CI_WORKFLOW = PROJECT_ROOT / ".github/workflows/ci.yml" PUBLISH_WORKFLOW = PROJECT_ROOT / ".github/workflows/publish.yml" +ARCHIVE_VERIFIER = PROJECT_ROOT / "scripts/verify_release_archives.py" TEMPORAL_MODULE = "rankweave/temporal_backtesting.py" RELEASE_VERSION = "0.18.0" @@ -10,9 +11,11 @@ def test_package_and_release_workflows_require_temporal_module(): ci_workflow = CI_WORKFLOW.read_text(encoding="utf-8") publish_workflow = PUBLISH_WORKFLOW.read_text(encoding="utf-8") + archive_verifier = ARCHIVE_VERIFIER.read_text(encoding="utf-8") - assert ci_workflow.count(TEMPORAL_MODULE) == 1 - assert publish_workflow.count(TEMPORAL_MODULE) == 1 + assert ci_workflow.count("scripts/verify_release_archives.py") == 1 + assert publish_workflow.count("scripts/verify_release_archives.py") == 3 + assert archive_verifier.count(TEMPORAL_MODULE) == 1 def test_installed_package_smoke_targets_current_temporal_release(): diff --git a/tests/test_verify_release_archives.py b/tests/test_verify_release_archives.py new file mode 100644 index 0000000..1e238fd --- /dev/null +++ b/tests/test_verify_release_archives.py @@ -0,0 +1,105 @@ +from io import BytesIO +from pathlib import Path +from tarfile import TarInfo +from tarfile import open as open_tarfile +from zipfile import ZIP_DEFLATED, ZipFile + +import pytest + +from scripts.verify_release_archives import ( + REQUIRED_SOURCE_MEMBERS, + REQUIRED_WHEEL_MEMBERS, + verify_release_archives, +) + +VERSION = "0.18.0" + + +def _write_wheel( + dist_dir: Path, + *, + omitted_member: str | None = None, + platform_tag: str = "linux_x86_64", + extension_suffix: str = ".so", +) -> None: + wheel_path = dist_dir / f"rankweave-{VERSION}-cp310-abi3-{platform_tag}.whl" + members = set(REQUIRED_WHEEL_MEMBERS) + members.add(f"rankweave/_rankweave_core.abi3{extension_suffix}") + if omitted_member is not None: + members.remove(omitted_member) + with ZipFile(wheel_path, "w", ZIP_DEFLATED) as archive: + for member in sorted(members): + archive.writestr(member, b"synthetic") + + +def _write_sdist(dist_dir: Path, *, omitted_member: str | None = None) -> None: + source_path = dist_dir / f"rankweave-{VERSION}.tar.gz" + members = set(REQUIRED_SOURCE_MEMBERS) + if omitted_member is not None: + members.remove(omitted_member) + with open_tarfile(source_path, "w:gz") as archive: + for member in sorted(members): + payload = b"synthetic" + member_info = TarInfo(f"rankweave-{VERSION}/{member}") + member_info.size = len(payload) + archive.addfile(member_info, BytesIO(payload)) + + +def test_verify_release_archives_accepts_complete_native_archives(tmp_path): + _write_wheel(tmp_path) + _write_sdist(tmp_path) + + verify_release_archives( + tmp_path, + VERSION, + expected_wheel_tags=("linux",), + require_sdist=True, + ) + + +def test_verify_release_archives_rejects_missing_native_stub(tmp_path): + _write_wheel(tmp_path, omitted_member="rankweave/_rankweave_core.pyi") + + with pytest.raises(ValueError, match="_rankweave_core.pyi"): + verify_release_archives( + tmp_path, + VERSION, + expected_wheel_tags=("linux",), + require_sdist=False, + ) + + +def test_verify_release_archives_rejects_missing_cargo_manifest(tmp_path): + _write_sdist(tmp_path, omitted_member="crates/rankweave-core/Cargo.toml") + + with pytest.raises(ValueError, match="rankweave-core/Cargo.toml"): + verify_release_archives( + tmp_path, + VERSION, + expected_wheel_tags=(), + require_sdist=True, + ) + + +def test_verify_release_archives_rejects_wrong_platform_set(tmp_path): + _write_wheel(tmp_path) + + with pytest.raises(ValueError, match="macosx"): + verify_release_archives( + tmp_path, + VERSION, + expected_wheel_tags=("macosx",), + require_sdist=False, + ) + + +def test_verify_release_archives_rejects_so_in_windows_wheel(tmp_path): + _write_wheel(tmp_path, platform_tag="win_amd64", extension_suffix=".so") + + with pytest.raises(ValueError, match="for its platform"): + verify_release_archives( + tmp_path, + VERSION, + expected_wheel_tags=("win_amd64",), + require_sdist=False, + )