fix: use OutputBatches metric variant for DF52 compatibility

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "1.0.0-beta.16"
+current_version = "3.1.0-beta.2"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -90,6 +90,11 @@ filename = "Cargo.toml"
 search = 'lance-namespace = {{ version = "={current_version}"'
 replace = 'lance-namespace = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-namespace-datafusion = {{ version = "={current_version}"'
+replace = 'lance-namespace-datafusion = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-namespace-impls = {{ version = "={current_version}"'

.cargo/config.toml

@@ -9,6 +9,11 @@ debug = true
 codegen-units = 16
 lto = "thin"
 
+[profile.bench]
+inherits = "release"
+lto = "thin"
+codegen-units = 16
+
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 

.github/CODEOWNERS

@@ -0,0 +1,14 @@
+# file specification
+protos/file*.proto @file-spec-team
+protos/encodings*.proto @file-spec-team
+docs/src/format/file/ @file-spec-team
+
+# table specification
+protos/table.proto @table-spec-team
+protos/rowids.proto @table-spec-team
+docs/src/format/table/ @table-spec-team
+
+# index specification
+protos/index.proto @index-spec-team
+protos/index_old.proto @index-spec-team
+docs/src/format/table/index/ @index-spec-team

.github/actions/setup-release-env/action.yml

@@ -11,7 +11,7 @@ runs:
     - name: Install dependencies
       shell: bash
       run: |
-        pip install bump-my-version packaging PyGithub
+        pip install bump-my-version packaging PyGithub PyYAML
 
     - name: Set up Rust
       uses: actions-rs/toolchain@v1

.github/workflows/approve-rc.yml

@@ -58,10 +58,7 @@ jobs:
 
         if [ -n "${PREVIOUS_TAG}" ]; then
           echo "Generating release notes from ${PREVIOUS_TAG} to ${STABLE_TAG}"
-          NOTES=$(gh api repos/${{ github.repository }}/releases/generate-notes \
-            -f tag_name="${STABLE_TAG}" \
-            -f previous_tag_name="${PREVIOUS_TAG}" \
-            --jq .body)
+          NOTES=$(python ci/generate_release_notes.py ${PREVIOUS_TAG} ${STABLE_TAG})
         else
           echo "No previous tag found, using automatic generation"
           NOTES=$(gh api repos/${{ github.repository }}/releases/generate-notes \

.github/workflows/benchmark-comment-trigger.yml

@@ -0,0 +1,51 @@
+# This workflow is used to trigger benchmarks on a PR when a specific comment is added
+# to the PR.  The workflow runs on all PR comments containing the string 'benchmark'
+# and the string '@bench-bot'.  The workflow collects some information about the PR
+# and then forwards the information to the lance-bench workflow.
+#
+# The lance-bench repository is a public repository in the lancedb organization which
+# runs benchmarks against the Lance repository on a regular basis and stores the results
+# in a historical database.
+
+name: Benchmark Comment Trigger
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  forward-to-bench:
+    # Only process comments on PRs that mention @bench-bot and contain 'benchmark'
+    if: |
+      github.event.issue.pull_request != null &&
+      contains(github.event.comment.body, '@bench-bot') &&
+      contains(github.event.comment.body, 'benchmark')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR details
+        id: pr
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+
+            core.setOutput('head_sha', pr.data.head.sha);
+
+      - name: Forward to lance-bench
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.LANCE_BENCH_DISPATCH_TOKEN }}
+          repository: lancedb/lance-bench
+          event-type: pr-comment
+          client-payload: |
+            {
+              "comment_body": ${{ toJson(github.event.comment.body) }},
+              "comment_user": "${{ github.event.comment.user.login }}",
+              "pr_number": ${{ github.event.issue.number }},
+              "pr_head_sha": "${{ steps.pr.outputs.head_sha }}",
+              "repository": "${{ github.repository }}"
+            }

.github/workflows/build_linux_wheel/action.yml

@@ -25,6 +25,11 @@ runs:
       shell: bash
       run: |
         echo "ARM BUILD: ${{ inputs.arm-build }}"
+    - name: Clean old wheels
+      shell: bash
+      run: |
+        # Ensure no cached pylance wheels linger across cache restores
+        rm -f python/target/wheels/pylance-*.whl || true
     - name: Build x86_64 Manylinux2014 wheel
       if: ${{ inputs.arm-build == 'false' && inputs.manylinux == '2_17' }}
       uses: PyO3/maturin-action@v1
@@ -34,6 +39,7 @@ runs:
         target: x86_64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
+        maturin-version: "1.10.2"
         before-script-linux: |
           set -e
           yum install -y openssl-devel \
@@ -52,6 +58,7 @@ runs:
           -e CC=clang
           -e CXX=clang++
         args: ${{ inputs.args }}
+        maturin-version: "1.10.2"
         before-script-linux: |
           set -e
           yum install -y openssl-devel clang \
@@ -67,6 +74,7 @@ runs:
         target: aarch64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
+        maturin-version: "1.10.2"
         before-script-linux: |
           set -e
           yum install -y openssl-devel clang \

.github/workflows/cargo-publish.yml

@@ -11,6 +11,11 @@ on:
         description: "Tag to publish (e.g., v1.0.0)"
         required: true
         type: string
+      skip_check_repo:
+        description: "Skip checking if packages have been modified (useful for backfilling missed releases)"
+        required: false
+        type: boolean
+        default: false
 
 env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
@@ -22,7 +27,7 @@ env:
 jobs:
   build:
     # Needs additional disk space for the full build.
-    runs-on: ubuntu-2404-8x-x64
+    runs-on: warp-ubuntu-latest-x64-8x
     permissions:
       id-token: write
     timeout-minutes: 60
@@ -87,6 +92,7 @@ jobs:
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
           args: "--all-features"
           path: .
+          check-repo: ${{ github.event_name != 'workflow_dispatch' || inputs.skip_check_repo != true }}
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest

.github/workflows/ci-benchmarks.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   bench_regress:
-    timeout-minutes: 30
+    timeout-minutes: 120
     runs-on: warp-custom-gcp-storage-benchmark
     env:
       # Need up-to-date compilers for kernels
@@ -44,17 +44,32 @@ jobs:
         run: |
           python -m venv venv
           source venv/bin/activate
-          pip install maturin duckdb requests pytest pytest-benchmark
-          maturin develop --locked --release
+          pip install maturin duckdb requests pytest pytest-benchmark datasets
+          maturin develop --locked --release --features datagen
+      - name: Build memtest
+        run: |
+          source venv/bin/activate
+          make -C ../memtest build-release
       - name: Generate datasets
         run: |
-          python -m venv venv
           source venv/bin/activate
           python python/ci_benchmarks/datagen/gen_all.py
       - name: Run benchmarks
         run: |
-          python -m venv venv
           source venv/bin/activate
           bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} --adapter python_pytest \
                       --branch main --testbed google-genoa --err --file results.json "python -mpytest --benchmark-json \
                       results.json python/ci_benchmarks"
+      - name: Run IO/memory benchmarks
+        run: |
+          source venv/bin/activate
+          LIB_PATH=$(lance-memtest)
+          LD_PRELOAD=$LIB_PATH pytest python/ci_benchmarks \
+            -k "io_mem_" \
+            --benchmark-stats-json io_mem_stats.json
+      - name: Upload IO/memory stats to bencher
+        run: |
+          source venv/bin/activate
+          bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} \
+            --adapter json --branch main --testbed google-genoa \
+            --err --file io_mem_stats.json

.github/workflows/claude-code-review.yml

@@ -0,0 +1,67 @@
+name: Claude Code Review
+
+on:
+  pull_request_target:
+    types: [opened]
+    # Optional: Only run on specific file changes
+    # paths:
+    #   - "src/**/*.ts"
+    #   - "src/**/*.tsx"
+    #   - "src/**/*.js"
+    #   - "src/**/*.jsx"
+
+jobs:
+  claude-review:
+    # Optional: Filter by PR author
+    # if: |
+    #   github.event.pull_request.user.login == 'external-contributor' ||
+    #   github.event.pull_request.user.login == 'new-developer' ||
+    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code Review
+        id: claude-review
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.CLAUDE_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          prompt: |
+            REPO: ${{ github.repository }}
+            PR NUMBER: ${{ github.event.pull_request.number }}
+
+            Please review this pull request and provide feedback on:
+            - Code quality and best practices
+            - Potential bugs or issues
+            - Performance considerations
+            - Security concerns
+            - Test coverage
+
+            Please note that the attention of contributors and maintainers is the MOST valuable resource.
+            Less is more: focus on the most important aspects.
+
+            - Your review output SHOULD be concise and clear.
+            - You SHOULD only highlight P0 and P1 level issues, such as severe bugs, performance degradation, or security concerns.
+            - You MUST not reiterate detailed changes in your review.
+            - You MUST not repeat aspects of the PR that are already well done.
+
+            Use the repository's CLAUDE.md for more guidance on style and conventions.
+
+            Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR.
+
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          claude_args: |
+            --allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"
+            --model "claude-opus-4-5"

.github/workflows/claude.yml

@@ -0,0 +1,49 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.CLAUDE_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          claude_args: |
+            --allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"
+            --model "claude-opus-4-5"