From 1b39263b7c9de911868532acfe601146053bd9a2 Mon Sep 17 00:00:00 2001 From: Sammy Sidhu Date: Fri, 23 Aug 2024 13:22:35 -0700 Subject: [PATCH 1/5] unit tests --- .github/workflows/python-package.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index afaf36cfa1..d690504562 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -64,6 +64,15 @@ jobs: pyproject.toml requirements-dev.txt + - name: Set up uv + if: ${{ (runner.os != 'Windows') }} + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Set up uv + if: ${{ (runner.os == 'Windows') }} + run: irm https://astral.sh/uv/install.ps1 | iex + shell: powershell + - name: Setup Virtual Env run: | python -m venv venv @@ -72,15 +81,14 @@ jobs: - name: Install dependencies if: ${{ (runner.os != 'Windows') }} run: | - pip install --upgrade pip - pip install -r requirements-dev.txt + activate + uv pip install -r requirements-dev.txt - name: Install dependencies (Windows) if: ${{ (runner.os == 'Windows') }} run: | .\venv\Scripts\activate - python -m pip install --upgrade pip - python -m pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Override pyarrow if: ${{ (matrix.pyarrow-version) && (runner.os != 'Windows') }} From e11b7a13dcdfa865b1e0260df9f66af2c56bfb71 Mon Sep 17 00:00:00 2001 From: Sammy Sidhu Date: Fri, 23 Aug 2024 13:26:16 -0700 Subject: [PATCH 2/5] switch everything to uv --- .github/workflows/daft-profiling.yml | 4 ++-- .github/workflows/python-package.yml | 34 +++++++++++++++------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/daft-profiling.yml b/.github/workflows/daft-profiling.yml index e431e347c4..5a081f79cb 100644 --- a/.github/workflows/daft-profiling.yml +++ b/.github/workflows/daft-profiling.yml @@ -37,11 +37,11 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Build Rust Library run: | diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d690504562..6e195a3fc7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -246,14 +246,16 @@ jobs: pattern: wheels-* merge-multiple: true path: dist + - name: Setup Virtual Env run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv + - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall + uv pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall rm -rf daft - uses: actions/cache@v4 env: @@ -324,10 +326,10 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall + uv pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall rm -rf daft - name: Prepare tmpdirs for IO services run: | @@ -407,10 +409,10 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall + uv pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall rm -rf daft - name: Prepare tmpdirs for IO services run: | @@ -496,10 +498,10 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall + uv pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall rm -rf daft - uses: docker/setup-buildx-action@v3 - uses: docker/build-push-action@v5 @@ -574,10 +576,10 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall + uv pip install -r requirements-dev.txt dist/${{ env.package-name }}-*x86_64*.whl --force-reinstall rm -rf daft - name: Spin up services run: | @@ -650,10 +652,10 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Build Rust Library run: | source activate @@ -790,10 +792,10 @@ jobs: run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Run doctests run: | source activate @@ -948,8 +950,8 @@ jobs: - name: Install pre-commit run: | - pip install --upgrade pip - pip install pre-commit + pip install uv + uv pip install pre-commit - uses: moonrepo/setup-rust@v1 with: cache: false From 1f7dafb2a4b37572e4af6267c02c21ecd0f849c6 Mon Sep 17 00:00:00 2001 From: Sammy Sidhu Date: Fri, 23 Aug 2024 13:29:14 -0700 Subject: [PATCH 3/5] use system python --- .github/workflows/daft-profiling.yml | 1 + .github/workflows/python-package.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/daft-profiling.yml b/.github/workflows/daft-profiling.yml index 5a081f79cb..a67f203e6e 100644 --- a/.github/workflows/daft-profiling.yml +++ b/.github/workflows/daft-profiling.yml @@ -10,6 +10,7 @@ env: TPCH_SCALE_FACTOR: '4' TPCH_NUM_PARTS: '32' PYTHON_VERSION: '3.9' + UV_SYSTEM_PYTHON: 1 jobs: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6e195a3fc7..9aceb83aa8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,6 +14,7 @@ on: env: DAFT_ANALYTICS_ENABLED: '0' + UV_SYSTEM_PYTHON: 1 jobs: From bbe1e09bda5675797eb2372789c9495a78ed9c18 Mon Sep 17 00:00:00 2001 From: Sammy Sidhu Date: Fri, 23 Aug 2024 13:29:48 -0700 Subject: [PATCH 4/5] use system python --- .github/workflows/python-package.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9aceb83aa8..dbcad84055 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -65,19 +65,11 @@ jobs: pyproject.toml requirements-dev.txt - - name: Set up uv - if: ${{ (runner.os != 'Windows') }} - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - - name: Set up uv - if: ${{ (runner.os == 'Windows') }} - run: irm https://astral.sh/uv/install.ps1 | iex - shell: powershell - - name: Setup Virtual Env run: | python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH + pip install uv - name: Install dependencies if: ${{ (runner.os != 'Windows') }} From e82066522b066f7b676e7ec0eb8459531f6770a9 Mon Sep 17 00:00:00 2001 From: Sammy Sidhu Date: Fri, 23 Aug 2024 13:56:22 -0700 Subject: [PATCH 5/5] switch rest to uv --- .github/workflows/nightlies-tests.yml | 9 +++++---- .github/workflows/python-package.yml | 4 ++-- .github/workflows/python-publish.yml | 6 ++++-- .github/workflows/ray-compatibility.yml | 12 ++++++------ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/nightlies-tests.yml b/.github/workflows/nightlies-tests.yml index 3029e632c3..b9a6c40f78 100644 --- a/.github/workflows/nightlies-tests.yml +++ b/.github/workflows/nightlies-tests.yml @@ -11,6 +11,7 @@ on: env: DAFT_ANALYTICS_ENABLED: '0' + UV_SYSTEM_PYTHON: 1 jobs: integration-test-tpch: @@ -34,8 +35,8 @@ jobs: echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt getdaft --pre --extra-index-url https://pypi.anaconda.org/daft-nightly/simple --force-reinstall + pip install uv + uv pip install -r requirements-dev.txt getdaft --pre --extra-index-url https://pypi.anaconda.org/daft-nightly/simple --force-reinstall rm -rf daft - uses: actions/cache@v4 env: @@ -98,8 +99,8 @@ jobs: echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH - name: Install Daft and dev dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt getdaft --pre --extra-index-url https://pypi.anaconda.org/daft-nightly/simple --force-reinstall + pip install uv + uv pip install -r requirements-dev.txt getdaft --pre --extra-index-url https://pypi.anaconda.org/daft-nightly/simple --force-reinstall rm -rf daft - name: Prepare tmpdirs for IO services run: | diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index dbcad84055..b17f676946 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -85,11 +85,11 @@ jobs: - name: Override pyarrow if: ${{ (matrix.pyarrow-version) && (runner.os != 'Windows') }} - run: pip install pyarrow==${{ matrix.pyarrow-version }} + run: uv pip install pyarrow==${{ matrix.pyarrow-version }} - name: Override deltalake for pyarrow if: ${{ (matrix.pyarrow-version == '7.0.0') }} - run: pip install deltalake==0.10.0 + run: uv pip install deltalake==0.10.0 - name: Build library and Test with pytest (unix) if: ${{ (runner.os != 'Windows') }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index dda193f4f6..7493109dfe 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -18,6 +18,7 @@ env: PACKAGE_NAME: getdaft PYTHON_VERSION: 3.11 DAFT_ANALYTICS_ENABLED: '0' + UV_SYSTEM_PYTHON: 1 IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && ( ! endsWith(github.ref, 'dev0')) }} IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} @@ -45,7 +46,8 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} architecture: x64 - - run: pip install -U twine toml + - run: pip install uv + - run: uv pip install twine toml - run: python tools/patch_package_version.py - name: Build wheels - Mac and Windows x86 if: ${{ ((matrix.os == 'macos') || (matrix.os == 'windows')) && (matrix.compile_arch == 'x86_64') }} @@ -90,7 +92,7 @@ jobs: - name: Install and test built wheel - Linux and Mac x86_64 if: ${{ (matrix.os == 'ubuntu') && (matrix.compile_arch == 'x86_64') }} run: | - pip install -r requirements-dev.txt dist/${{ env.PACKAGE_NAME }}-*x86_64*.whl --force-reinstall + uv pip install -r requirements-dev.txt dist/${{ env.PACKAGE_NAME }}-*x86_64*.whl --force-reinstall rm -rf daft pytest -v # Disable until we figure out why are we getting FileNotFoundError: [WinError 3] Failed to open local file diff --git a/.github/workflows/ray-compatibility.yml b/.github/workflows/ray-compatibility.yml index 623e627cf3..a79c9bad3a 100644 --- a/.github/workflows/ray-compatibility.yml +++ b/.github/workflows/ray-compatibility.yml @@ -9,6 +9,7 @@ on: env: DAFT_ANALYTICS_ENABLED: '0' + UV_SYSTEM_PYTHON: 1 jobs: build: @@ -40,21 +41,20 @@ jobs: python -m venv venv echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH - - name: Upgrade Pip - run: pip install --upgrade pip + - name: Install uv + run: pip install uv - name: Install to pin Ray versions - run: pip install ray[data,default]==${{ matrix.ray-version }} + run: uv pip install ray[data,default]==${{ matrix.ray-version }} # Ray requires a pinned protobuf<4.0 for versions of Ray <1.13 (see: https://github.com/ray-project/ray/pull/25211) - name: Install to pin Protobuf version (for Ray <1.13) if: ${{ matrix.ray-version == '1.12.0' || matrix.ray-version == '1.11.0' || matrix.ray-version == '1.10.0' }} - run: pip install protobuf==3.20.3 + run: uv pip install protobuf==3.20.3 - name: Install dependencies run: | - pip install --upgrade pip - pip install -r requirements-dev.txt + uv pip install -r requirements-dev.txt - name: Build Rust Library run: |