chore: use uv for dep management

.github/workflows/benchmarks.yml

@@ -16,17 +16,23 @@ jobs:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.10"
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
     - uses: ./.github/workflows/build_linux_wheel
       with:
         python-minor-version: "10"
-    - name: Pip install
+    - name: Install dependencies
       working-directory: python
       run: |
-        pip install $(ls target/wheels/*.whl)
-        pip install pandas
+        # Create a clean virtual environment
+        uv venv --seed
+        # Install the pre-built wheel with test extras (includes pandas and benchmark deps)
+        uv pip install $(ls target/wheels/*.whl)[tests,benchmarks]
     - name: Run test
       working-directory: benchmarks/${{ matrix.dataset }}
       run: |

.github/workflows/build_linux_wheel/action.yml

@@ -34,6 +34,7 @@ runs:
         target: x86_64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
+        container: off
         before-script-linux: |
           set -e
           yum install -y openssl-devel \
@@ -52,6 +53,7 @@ runs:
           -e CC=clang
           -e CXX=clang++
         args: ${{ inputs.args }}
+        container: off
         before-script-linux: |
           set -e
           yum install -y openssl-devel clang \
@@ -67,6 +69,7 @@ runs:
         target: aarch64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
+        container: off
         before-script-linux: |
           set -e
           yum install -y openssl-devel clang \

.github/workflows/ci-benchmarks.yml

@@ -33,6 +33,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.11 # Ray does not support 3.12 yet.
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -42,19 +46,13 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
       - name: Build
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install maturin duckdb requests pytest pytest-benchmark
-          maturin develop --locked --release
+          uv sync --group test --group dev  # test includes benchmark deps
+          uv run maturin develop --locked --release
       - name: Generate datasets
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          python python/ci_benchmarks/datagen/gen_all.py
+          uv run python python/ci_benchmarks/datagen/gen_all.py
       - name: Run benchmarks
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} --adapter python_pytest \
+          uv run bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} --adapter python_pytest \
                       --branch main --testbed google-genoa --err --file results.json "python -mpytest --benchmark-json \
                       results.json python/ci_benchmarks"

.github/workflows/file_verification.yml

@@ -15,6 +15,11 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
       - name: Install Build Requirements
         run: |
           sudo apt update
@@ -46,16 +51,13 @@ jobs:
       - name: Build Lance
         working-directory: ./python
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install pyarrow maturin
-          maturin develop --release
+          uv sync --group dev  # includes maturin and pyarrow is in core deps
+          uv run maturin develop --release
 
       - name: Test Lance File Write Read Round Trip
         run: |
-          source python/venv/bin/activate
           cd .github/workflows/file_verification
-          python test_write_read.py
+          uv run python test_write_read.py
 
 
       - name: Cleanup

.github/workflows/notebook.yml

@@ -25,20 +25,26 @@ jobs:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.10"
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
     - uses: ./.github/workflows/build_linux_wheel
       with:
         python-minor-version: "10"
-    - name: Pip install
+    - name: Install dependencies
       working-directory: python
       shell: bash
       run: |
-        pip3 install jupyter duckdb>=0.7
-        pip3 install $(ls target/wheels/*.whl)
-    - name: Run python tests
+        # Create a clean virtual environment
+        uv venv --seed
+        # Install the pre-built wheel with test extras (includes jupyter and duckdb)
+        uv pip install $(ls target/wheels/*.whl)[tests]
+    - name: Run notebook tests
       shell: bash
       working-directory: notebooks
       run: |
-        jupyter nbconvert --to notebook --execute quickstart.ipynb
+        uv run --no-project jupyter nbconvert --to notebook --execute quickstart.ipynb

.github/workflows/pypi-publish.yml

@@ -35,18 +35,22 @@ jobs:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.${{ matrix.python-minor-version }}
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
     - name: Handle tag
       id: handle_tag
       run: |
         # If the tag ends with -beta.N, we need to call setup_version.py 
         # and export repo as "fury" instead of "pypi"
         if [[ ${{ github.ref }} == refs/tags/*-beta.* ]]; then
           TAG=$(echo ${{ github.ref }} | sed 's/refs\/tags\///')
-          pip install packaging
-          python ci/setup_version.py $TAG
+          uv sync --group ci  # includes packaging
+          uv run python ci/setup_version.py $TAG
           echo "repo=fury" >> $GITHUB_OUTPUT
         else
           echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -82,18 +86,22 @@ jobs:
         fetch-depth: 0
         lfs: true
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.12
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
     - name: Handle tag
       id: handle_tag
       run: |
         # If the tag ends with -beta.N, we need to call setup_version.py 
         # and export repo as "fury" instead of "pypi"
         if [[ ${{ github.ref }} == refs/tags/*-beta.* ]]; then
           TAG=$(echo ${{ github.ref }} | sed 's/refs\/tags\///')
-          pip install packaging
-          python ci/setup_version.py $TAG
+          uv sync --group ci  # includes packaging
+          uv run python ci/setup_version.py $TAG
           echo "repo=fury" >> $GITHUB_OUTPUT
         else
           echo "repo=pypi" >> $GITHUB_OUTPUT
@@ -120,9 +128,13 @@ jobs:
           fetch-depth: 0
           lfs: true
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.${{ matrix.python-minor-version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - name: Handle tag
         id: handle_tag
         shell: bash
@@ -131,8 +143,8 @@ jobs:
           # and export repo as "fury" instead of "pypi"
           if [[ ${{ github.ref }} == refs/tags/*-beta.* ]]; then
             TAG=$(echo ${{ github.ref }} | sed 's/refs\/tags\///')
-            pip install packaging
-            python ci/setup_version.py $TAG
+            uv pip install packaging
+            uv run python ci/setup_version.py $TAG
             echo "repo=fury" >> $GITHUB_OUTPUT
           else
             echo "repo=pypi" >> $GITHUB_OUTPUT

.github/workflows/python.yml

@@ -56,41 +56,42 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.11 # Ray does not support 3.12 yet.
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
           cache-targets: false
           cache-workspace-crates: true
-      - name: Install linting tools
-        run: |
-          pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas]
-          pip install torch --index-url https://download.pytorch.org/whl/cpu
-      - name: Lint Python
-        run: |
-          ruff format --check --diff python
-          ruff check python
-          pyright
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install linting tools
+        run: |
+          # Install most dependencies first
+          uv sync --no-install-project --group dev --group test --group ray
+          # Install torch separately with CPU-only index as extra index
+          uv pip install torch --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu
+      - name: Lint Python
+        run: |
+          uv run --no-sync ruff format --check --diff python
+          uv run --no-sync ruff check python
+          uv run --no-sync pyright
       - name: Lint Rust
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
           cargo fmt --all -- --check
           cargo clippy --locked --features ${ALL_FEATURES} --tests -- -D warnings
       - name: Build
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu
-          pip install maturin
-          maturin develop --locked --extras tests,ray
+          uv run maturin develop --locked
       - name: Run doctest
         run: |
-          source venv/bin/activate
-          pytest --doctest-modules python/lance
+          uv run pytest --doctest-modules python/lance
   linux:
     timeout-minutes: 45
     strategy:
@@ -111,6 +112,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.${{ matrix.python-minor-version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -120,18 +125,18 @@ jobs:
       - uses: ./.github/workflows/build_linux_wheel
       - uses: ./.github/workflows/run_tests
       - name: Generate forward compatibility files
-        run: python python/tests/forward_compat/datagen.py
+        run: uv run --no-project python python/tests/forward_compat/datagen.py
       - name: Run forward compatibility tests (pylance 0.16.0)
         run: |
-          python -m venv venv
-          source venv/bin/activate
-          pip install pytest pylance==0.16.0
-          pytest python/tests/forward_compat --run-forward
+          # Create a separate venv for testing specific pylance version
+          uv venv --python 3.${{ matrix.python-minor-version }}
+          uv pip install pytest pylance==0.16.0
+          uv run --no-project pytest python/tests/forward_compat --run-forward
       - name: Run forward compatibility tests (pylance 0.29.1.beta2)
         run: |
-          source venv/bin/activate
-          pip install pytest --pre --extra-index-url https://pypi.fury.io/lancedb/ pylance==0.29.1.beta2
-          pytest python/tests/forward_compat --run-forward
+          # Reuse venv but install different version
+          uv pip install pytest --pre --extra-index-url https://pypi.fury.io/lancedb/ pylance==0.29.1.beta2
+          uv run --no-project pytest python/tests/forward_compat --run-forward
 
   linux-arm:
     timeout-minutes: 45
@@ -150,6 +155,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -182,6 +191,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -205,6 +218,10 @@ jobs:
         with:
           fetch-depth: 0
           lfs: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -229,6 +246,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11" # TODO: upgrade when ray supports 3.12
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: python
@@ -238,6 +259,6 @@ jobs:
       - uses: ./.github/workflows/build_linux_wheel
       - name: Install dependencies
         run: |
-          pip install ray[data]
-          pip install torch --index-url https://download.pytorch.org/whl/cpu
+          uv sync --no-project --group ray
+          uv pip install torch --index-url https://pypi.org/simple --extra-index-url https://download.pytorch.org/whl/cpu
       - uses: ./.github/workflows/run_integtests