Skip to content

Commit 6dd1989

Browse files
potiukpotiuk
committed
[v2-10-test] Backport pull_request_target removal
This is a bulk change that synchronizes dev/ci scripts for v2-10-test branch with main #45266 - including follow-ups. Rather than cherry-picking relevant PRs, this one gets the latest version of the scripts from main and updates the branch with some changes to adapt them to v2-10-test (such as bringing back python 3.8 support, removing some providers checks after the bulk move of providers and making sure all tests are passing. This is far easier than cherry-picking the changes, because for the v2-10-test we stopped cherry-picking CI changes which was deemed unnecessary (we used to do it for all previous branches) but this made it far more difficult (if not impossible) to cherry-pick individual changes. Fortunately, the CI scripts are maintained in the way that their latest version **should** in principle work for a v2-* branch and hopefully after just a few adjustments we should be able to synchronize the changes from main by updating all relevant CI/DEV scripts, dockerfile images, workflows, pre-commits etc.
1 parent 586f1ea commit 6dd1989

File tree

366 files changed

+15753
-12042
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

366 files changed

+15753
-12042
lines changed

‎.dockerignore

+7
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,15 @@
3434
!chart
3535
!docs
3636
!licenses
37+
!providers/
38+
!task_sdk/
3739

3840
# Add those folders to the context so that they are available in the CI container
3941
!scripts
4042

4143
# Add tests and kubernetes_tests to context.
4244
!tests
45+
!tests_common
4346
!kubernetes_tests
4447
!helm_tests
4548
!docker_tests
@@ -79,6 +82,7 @@
7982
airflow/git_version
8083
# Exclude mode_modules pulled by "yarn" for compilation of www files generated by NPM
8184
airflow/www/node_modules
85+
airflow/ui/node_modules
8286

8387
# Exclude link to docs
8488
airflow/www/static/docs
@@ -121,6 +125,9 @@ docs/_build/
121125
docs/_api/
122126
docs/_doctrees/
123127

128+
# Exclude new providers docs generated files
129+
providers/**/docs/_api/
130+
124131
# files generated by memray
125132
*.py.*.html
126133
*.py.*.bin

‎.github/actions/breeze/action.yml

+5-7
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ description: 'Sets up Python and Breeze'
2121
inputs:
2222
python-version:
2323
description: 'Python version to use'
24-
# Version of Python used for reproducibility of the packages built
25-
# Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
26-
# Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
2724
default: "3.9"
25+
use-uv:
26+
description: 'Whether to use uv tool'
27+
required: true
2828
outputs:
2929
host-python-version:
3030
description: Python version used in host
@@ -36,13 +36,11 @@ runs:
3636
uses: actions/setup-python@v5
3737
with:
3838
python-version: ${{ inputs.python-version }}
39-
cache: 'pip'
40-
cache-dependency-path: ./dev/breeze/pyproject.toml
39+
# NOTE! Installing Breeze without using cache is FASTER than when using cache - uv is so fast and has
40+
# so low overhead, that just running upload cache/restore cache is slower than installing it from scratch
4141
- name: "Install Breeze"
4242
shell: bash
4343
run: ./scripts/ci/install_breeze.sh
44-
env:
45-
PYTHON_VERSION: ${{ inputs.python-version }}
4644
- name: "Free space"
4745
shell: bash
4846
run: breeze ci free-space

‎.github/actions/checkout_target_commit/action.yml

-81
This file was deleted.

‎.github/actions/install-pre-commit/action.yml

+43-15
Original file line numberDiff line numberDiff line change
@@ -21,30 +21,58 @@ description: 'Installs pre-commit and related packages'
2121
inputs:
2222
python-version:
2323
description: 'Python version to use'
24-
default: 3.9
24+
default: "3.9"
2525
uv-version:
2626
description: 'uv version to use'
27-
default: 0.5.11
27+
default: "0.5.14" # Keep this comment to allow automatic replacement of uv version
2828
pre-commit-version:
2929
description: 'pre-commit version to use'
30-
default: 4.0.1
30+
default: "4.0.1" # Keep this comment to allow automatic replacement of pre-commit version
3131
pre-commit-uv-version:
3232
description: 'pre-commit-uv version to use'
33-
default: 4.1.4
33+
default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version
3434
runs:
3535
using: "composite"
3636
steps:
3737
- name: Install pre-commit, uv, and pre-commit-uv
3838
shell: bash
39-
run: >
40-
pip install
41-
pre-commit==${{inputs.pre-commit-version}}
42-
uv==${{inputs.uv-version}}
43-
pre-commit-uv==${{inputs.pre-commit-uv-version}}
44-
- name: Cache pre-commit envs
45-
uses: actions/cache@v4
39+
env:
40+
UV_VERSION: ${{inputs.uv-version}}
41+
PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}}
42+
PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}}
43+
run: |
44+
pip install uv==${UV_VERSION} || true
45+
uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \
46+
--with pre-commit-uv==${PRE_COMMIT_UV_VERSION}
47+
working-directory: ${{ github.workspace }}
48+
# We need to use tar file with archive to restore all the permissions and symlinks
49+
- name: "Delete ~.cache"
50+
run: |
51+
du ~/ --max-depth=2
52+
echo
53+
echo Deleting ~/.cache
54+
echo
55+
rm -rf ~/.cache
56+
echo
57+
shell: bash
58+
- name: "Restore pre-commit cache"
59+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
4660
with:
47-
path: ~/.cache/pre-commit
48-
key: "pre-commit-${{inputs.python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
49-
restore-keys: |
50-
pre-commit-${{inputs.python-version}}-
61+
key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
62+
path: /tmp/
63+
id: restore-pre-commit-cache
64+
- name: "Restore .cache from the tar file"
65+
run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz
66+
shell: bash
67+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
68+
- name: "Show restored files"
69+
run: |
70+
echo "Restored files"
71+
du ~/ --max-depth=2
72+
echo
73+
shell: bash
74+
if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
75+
- name: Install pre-commit hooks
76+
shell: bash
77+
run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1)
78+
working-directory: ${{ github.workspace }}

‎.github/actions/migration_tests/action.yml

+14
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,20 @@ runs:
4545
COMPOSE_PROJECT_NAME: "docker-compose"
4646
env:
4747
COMPOSE_PROJECT_NAME: "docker-compose"
48+
- name: "Bring compose down again"
49+
shell: bash
50+
run: breeze down
51+
env:
52+
COMPOSE_PROJECT_NAME: "docker-compose"
53+
- name: "Test offline migration ${{env.BACKEND}}"
54+
shell: bash
55+
run: >
56+
breeze shell "airflow db reset -y &&
57+
airflow db downgrade -n 2.7.0 -y &&
58+
airflow db migrate -s"
59+
env:
60+
COMPOSE_PROJECT_NAME: "docker-compose"
61+
if: env.BACKEND != 'sqlite'
4862
- name: "Bring any containers left down"
4963
shell: bash
5064
run: breeze down

‎.github/actions/post_tests_success/action.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ runs:
3333
- name: "Upload artifact for warnings"
3434
uses: actions/upload-artifact@v4
3535
with:
36-
name: test-warnings-${{env.JOB_ID}}
36+
name: test-warnings-${{ env.JOB_ID }}
3737
path: ./files/warnings-*.txt
3838
retention-days: 7
3939
if-no-files-found: ignore
@@ -50,5 +50,5 @@ runs:
5050
if: env.ENABLE_COVERAGE == 'true' && env.TEST_TYPES != 'Helm' && inputs.python-version != '3.12'
5151
with:
5252
name: coverage-${{env.JOB_ID}}
53-
flags: python-${{env.PYTHON_MAJOR_MINOR_VERSION}},${{env.BACKEND}}-${{env.BACKEND_VERSION}}
53+
flags: python-${{ env.PYTHON_MAJOR_MINOR_VERSION }},${{ env.BACKEND }}-${{ env.BACKEND_VERSION }}
5454
directory: "./files/coverage-reports/"

‎.github/actions/prepare_all_ci_images/action.yml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
---
19+
name: 'Prepare all CI images'
20+
description: 'Recreates current python CI images from artifacts for all python versions'
21+
inputs:
22+
python-versions-list-as-string:
23+
description: 'Stringified array of all Python versions to test - separated by spaces.'
24+
required: true
25+
platform:
26+
description: 'Platform for the build - linux/amd64 or linux/arm64'
27+
required: true
28+
runs:
29+
using: "composite"
30+
steps:
31+
- name: "Cleanup docker"
32+
run: ./scripts/ci/cleanup_docker.sh
33+
shell: bash
34+
# TODO: Currently we cannot loop through the list of python versions and have dynamic list of
35+
# tasks. Instead we hardcode all possible python versions and they - but
36+
# this should be implemented in stash action as list of keys to download.
37+
# That includes 3.8 - 3.12 as we are backporting it to v2-10-test branch
38+
# This is captured in https://github.com/apache/airflow/issues/45268
39+
- name: "Restore CI docker image ${{ inputs.platform }}:3.8"
40+
uses: ./.github/actions/prepare_single_ci_image
41+
with:
42+
platform: ${{ inputs.platform }}
43+
python: "3.8"
44+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
45+
- name: "Restore CI docker image ${{ inputs.platform }}:3.9"
46+
uses: ./.github/actions/prepare_single_ci_image
47+
with:
48+
platform: ${{ inputs.platform }}
49+
python: "3.9"
50+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
51+
- name: "Restore CI docker image ${{ inputs.platform }}:3.10"
52+
uses: ./.github/actions/prepare_single_ci_image
53+
with:
54+
platform: ${{ inputs.platform }}
55+
python: "3.10"
56+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
57+
- name: "Restore CI docker image ${{ inputs.platform }}:3.11"
58+
uses: ./.github/actions/prepare_single_ci_image
59+
with:
60+
platform: ${{ inputs.platform }}
61+
python: "3.11"
62+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
63+
- name: "Restore CI docker image ${{ inputs.platform }}:3.12"
64+
uses: ./.github/actions/prepare_single_ci_image
65+
with:
66+
platform: ${{ inputs.platform }}
67+
python: "3.12"
68+
python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}

‎.github/actions/prepare_breeze_and_image/action.yml

+32-15
Original file line numberDiff line numberDiff line change
@@ -16,30 +16,47 @@
1616
# under the License.
1717
#
1818
---
19-
name: 'Prepare breeze && current python image'
20-
description: 'Installs breeze and pulls current python image'
19+
name: 'Prepare breeze && current image (CI or PROD)'
20+
description: 'Installs breeze and recreates current python image from artifact'
2121
inputs:
22-
pull-image-type:
23-
description: 'Which image to pull'
24-
default: CI
22+
python:
23+
description: 'Python version for image to prepare'
24+
required: true
25+
image-type:
26+
description: 'Which image type to prepare (ci/prod)'
27+
default: "ci"
28+
platform:
29+
description: 'Platform for the build - linux/amd64 or linux/arm64'
30+
required: true
31+
use-uv:
32+
description: 'Whether to use uv'
33+
required: true
2534
outputs:
2635
host-python-version:
2736
description: Python version used in host
2837
value: ${{ steps.breeze.outputs.host-python-version }}
2938
runs:
3039
using: "composite"
3140
steps:
41+
- name: "Cleanup docker"
42+
run: ./scripts/ci/cleanup_docker.sh
43+
shell: bash
3244
- name: "Install Breeze"
3345
uses: ./.github/actions/breeze
46+
with:
47+
use-uv: ${{ inputs.use-uv }}
3448
id: breeze
35-
- name: Login to ghcr.io
36-
shell: bash
37-
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
38-
- name: Pull CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
39-
shell: bash
40-
run: breeze ci-image pull --tag-as-latest
41-
if: inputs.pull-image-type == 'CI'
42-
- name: Pull PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG }}
49+
- name: "Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}"
50+
uses: apache/infrastructure-actions/stash/restore@c94b890bbedc2fc61466d28e6bd9966bc6c6643c
51+
with:
52+
key: ${{ inputs.image-type }}-image-save-${{ inputs.platform }}-${{ inputs.python }}
53+
path: "/tmp/"
54+
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"
55+
env:
56+
PLATFORM: ${{ inputs.platform }}
57+
PYTHON: ${{ inputs.python }}
58+
IMAGE_TYPE: ${{ inputs.image-type }}
59+
run: >
60+
breeze ${IMAGE_TYPE}-image load
61+
--platform ${PLATFORM} --python ${PYTHON}
4362
shell: bash
44-
run: breeze prod-image pull --tag-as-latest
45-
if: inputs.pull-image-type == 'PROD'

0 commit comments

Comments
 (0)