emilk/fix write starvation

.bumpversion.toml

@@ -1,8 +1,8 @@
 [tool.bumpversion]
-current_version = "0.32.1"
-parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>.+))?"
+current_version = "5.0.0-beta.1"
+parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
-    "{major}.{minor}.{patch}-{prerelease}",
+    "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
     "{major}.{minor}.{patch}"
 ]
 search = "{current_version}"
@@ -18,6 +18,13 @@ allow_dirty = false
 commit = false
 message = "chore: bump version {current_version} → {new_version}"
 
+[tool.bumpversion.parts.prerelease]
+optional_value = "stable"
+values = ["beta", "rc", "stable"]
+
+[tool.bumpversion.parts.prerelease_num]
+first_value = "0"
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'version = "{current_version}"'
@@ -58,6 +65,11 @@ filename = "Cargo.toml"
 search = 'lance-file = {{ version = "={current_version}"'
 replace = 'lance-file = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-geo = {{ version = "={current_version}"'
+replace = 'lance-geo = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-index = {{ version = "={current_version}"'
@@ -78,6 +90,11 @@ filename = "Cargo.toml"
 search = 'lance-namespace = {{ version = "={current_version}"'
 replace = 'lance-namespace = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-namespace-datafusion = {{ version = "={current_version}"'
+replace = 'lance-namespace-datafusion = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-namespace-impls = {{ version = "={current_version}"'
@@ -108,96 +125,8 @@ filename = "Cargo.toml"
 search = 'lance-bitpacking = {{ version = "={current_version}"'
 replace = 'lance-bitpacking = {{ version = "={new_version}"'
 
-# Update all rust crate Cargo.toml files
-[[tool.bumpversion.files]]
-filename = "rust/lance/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-arrow/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-core/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-datafusion/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-datagen/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-encoding/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-file/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-index/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-io/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-linalg/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-namespace/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-namespace-impls/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-table/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/compression/bitpacking/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/compression/fsst/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-test-macros/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/lance-testing/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
-
-[[tool.bumpversion.files]]
-filename = "rust/examples/Cargo.toml"
-search = 'version = "{current_version}"'
-replace = 'version = "{new_version}"'
+# Note: Individual rust crate Cargo.toml files use workspace = true,
+# so we don't need to update them individually
 
 # Python Cargo.toml
 [[tool.bumpversion.files]]

.cargo/config.toml

@@ -9,6 +9,11 @@ debug = true
 codegen-units = 16
 lto = "thin"
 
+[profile.bench]
+inherits = "release"
+lto = "thin"
+codegen-units = 16
+
 [target.x86_64-unknown-linux-gnu]
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 

.github/CODEOWNERS

@@ -0,0 +1,14 @@
+# file specification
+protos/file*.proto @file-spec-team
+protos/encodings*.proto @file-spec-team
+docs/src/format/file/ @file-spec-team
+
+# table specification
+protos/table.proto @table-spec-team
+protos/rowids.proto @table-spec-team
+docs/src/format/table/ @table-spec-team
+
+# index specification
+protos/index.proto @index-spec-team
+protos/index_old.proto @index-spec-team
+docs/src/format/table/index/ @index-spec-team

.github/actions/setup-release-env/action.yml

@@ -0,0 +1,26 @@
+name: 'Setup Release Environment'
+description: 'Sets up Python, Rust, and dependencies for release workflows (assumes repo is already checked out)'
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
+      with:
+        python-version: "3.11"
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        pip install bump-my-version packaging PyGithub PyYAML
+
+    - name: Set up Rust
+      uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1
+      with:
+        toolchain: stable
+        override: true
+
+    - name: Configure git identity
+      shell: bash
+      run: |
+        git config user.name 'Lance Release Bot'
+        git config user.email 'dev+gha@lance.org'

.github/release.yml

@@ -7,6 +7,9 @@ changelog:
     - title: Breaking Changes 🛠
       labels:
         - breaking-change
+    - title: Critical Fixes ‼️
+      labels:
+        - critical-fix
     - title: New Features 🎉
       labels:
         - enhancement

.github/workflows/approve-rc.yml

@@ -0,0 +1,110 @@
+name: Approve RC
+
+on:
+  workflow_dispatch:
+    inputs:
+      rc_tag:
+        description: 'RC tag to approve (e.g., v1.3.0-rc.2 or v1.3.1-rc.1)'
+        required: true
+        type: string
+      dry_run:
+        description: 'Dry run (simulate without pushing)'
+        required: true
+        default: false
+        type: boolean
+
+jobs:
+  approve-rc:
+    runs-on: ubuntu-latest
+    outputs:
+      stable_version: ${{ steps.approve.outputs.STABLE_VERSION }}
+      stable_tag: ${{ steps.approve.outputs.STABLE_TAG }}
+      release_branch: ${{ steps.approve.outputs.RELEASE_BRANCH }}
+      is_major_minor: ${{ steps.approve.outputs.IS_MAJOR_MINOR }}
+      previous_tag: ${{ steps.approve.outputs.PREVIOUS_TAG }}
+    steps:
+    - name: Output Inputs
+      run: echo "${{ toJSON(github.event.inputs) }}"
+
+    - name: Check out repository
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      with:
+        token: ${{ secrets.LANCE_RELEASE_TOKEN }}
+        fetch-depth: 0
+        lfs: true
+
+    - name: Setup release environment
+      uses: ./.github/actions/setup-release-env
+
+    - name: Approve RC
+      id: approve
+      run: |
+        bash ci/approve_rc.sh "${{ inputs.rc_tag }}"
+
+    - name: Push changes (if not dry run)
+      if: ${{ !inputs.dry_run }}
+      run: |
+        git push origin "${{ steps.approve.outputs.RELEASE_BRANCH }}"
+        git push origin "${{ steps.approve.outputs.STABLE_TAG }}"
+
+    - name: Generate Release Notes (if not dry run)
+      if: ${{ !inputs.dry_run }}
+      id: release_notes
+      env:
+        GH_TOKEN: ${{ secrets.LANCE_RELEASE_TOKEN }}
+      run: |
+        PREVIOUS_TAG="${{ steps.approve.outputs.PREVIOUS_TAG }}"
+        STABLE_TAG="${{ steps.approve.outputs.STABLE_TAG }}"
+
+        if [ -n "${PREVIOUS_TAG}" ]; then
+          echo "Generating release notes from ${PREVIOUS_TAG} to ${STABLE_TAG}"
+          NOTES=$(python ci/generate_release_notes.py ${PREVIOUS_TAG} ${STABLE_TAG})
+        else
+          echo "No previous tag found, using automatic generation"
+          NOTES=$(gh api repos/${{ github.repository }}/releases/generate-notes \
+            -f tag_name="${STABLE_TAG}" \
+            --jq .body)
+        fi
+
+        # Save to output
+        echo "notes<<EOF" >> $GITHUB_OUTPUT
+        echo "$NOTES" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
+
+    - name: Create GitHub Release (if not dry run)
+      if: ${{ !inputs.dry_run }}
+      uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
+      with:
+        tag_name: ${{ steps.approve.outputs.STABLE_TAG }}
+        name: ${{ steps.approve.outputs.STABLE_TAG }}
+        draft: false
+        prerelease: false
+        body: ${{ steps.release_notes.outputs.notes }}
+        token: ${{ secrets.LANCE_RELEASE_TOKEN }}
+
+    - name: Summary
+      run: |
+        echo "## Stable Release Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "- **RC Tag:** ${{ inputs.rc_tag }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Stable Version:** ${{ steps.approve.outputs.STABLE_VERSION }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Stable Tag:** ${{ steps.approve.outputs.STABLE_TAG }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Release Branch:** ${{ steps.approve.outputs.RELEASE_BRANCH }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Next Version:** ${{ steps.approve.outputs.NEXT_BETA_VERSION }}" >> $GITHUB_STEP_SUMMARY
+        echo "- **Release Type:** $( [ "${{ steps.approve.outputs.IS_MAJOR_MINOR }}" == "true" ] && echo "Major/Minor" || echo "Patch" )" >> $GITHUB_STEP_SUMMARY
+        if [ -n "${{ steps.approve.outputs.PREVIOUS_TAG }}" ]; then
+          echo "- **Release Notes From:** ${{ steps.approve.outputs.PREVIOUS_TAG }}" >> $GITHUB_STEP_SUMMARY
+        fi
+        echo "- **Dry Run:** ${{ inputs.dry_run }}" >> $GITHUB_STEP_SUMMARY
+
+        if [ "${{ inputs.dry_run }}" == "true" ]; then
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "⚠️ This was a dry run. No changes were pushed." >> $GITHUB_STEP_SUMMARY
+        else
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "✅ Stable release ${{ steps.approve.outputs.STABLE_TAG }} complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Publishing:** Stable artifacts will be published to PyPI, crates.io, Maven Central" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Auto-bumped:** Release branch bumped to ${{ steps.approve.outputs.NEXT_BETA_VERSION }}" >> $GITHUB_STEP_SUMMARY
+        fi

.github/workflows/benchmark-comment-trigger.yml

@@ -0,0 +1,51 @@
+# This workflow is used to trigger benchmarks on a PR when a specific comment is added
+# to the PR.  The workflow runs on all PR comments containing the string 'benchmark'
+# and the string '@bench-bot'.  The workflow collects some information about the PR
+# and then forwards the information to the lance-bench workflow.
+#
+# The lance-bench repository is a public repository in the lancedb organization which
+# runs benchmarks against the Lance repository on a regular basis and stores the results
+# in a historical database.
+
+name: Benchmark Comment Trigger
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  forward-to-bench:
+    # Only process comments on PRs that mention @bench-bot and contain 'benchmark'
+    if: |
+      github.event.issue.pull_request != null &&
+      contains(github.event.comment.body, '@bench-bot') &&
+      contains(github.event.comment.body, 'benchmark')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR details
+        id: pr
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
+        with:
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+
+            core.setOutput('head_sha', pr.data.head.sha);
+
+      - name: Forward to lance-bench
+        uses: peter-evans/repository-dispatch@bf47d102fdb849e755b0b0023ea3e81a44b6f570 # v2
+        with:
+          token: ${{ secrets.LANCE_BENCH_DISPATCH_TOKEN }}
+          repository: lancedb/lance-bench
+          event-type: pr-comment
+          client-payload: |
+            {
+              "comment_body": ${{ toJson(github.event.comment.body) }},
+              "comment_user": "${{ github.event.comment.user.login }}",
+              "pr_number": ${{ github.event.issue.number }},
+              "pr_head_sha": "${{ steps.pr.outputs.head_sha }}",
+              "repository": "${{ github.repository }}"
+            }

.github/workflows/benchmarks.yml

@@ -11,12 +11,12 @@ jobs:
       matrix:
         dataset: ["sift"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       with:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
       with:
         python-version: "3.10"
     - uses: ./.github/workflows/build_linux_wheel
@@ -32,7 +32,7 @@ jobs:
       run: |
         ./test_dataset.sh
     - name: Archive results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: ${{ matrix.dataset }}-results
         path: |