diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index a2a28cbed14ac..7dd05fc53ee2b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -26,7 +26,6 @@ permissions: contents: read pull-requests: read env: - MOUNT_SELECTED_LOCAL_SOURCES: "false" ANSWER: "yes" DB_RESET: "true" VERBOSE: "true" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7b6c6cd91ab4..405272c20fadd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ permissions: contents: read packages: read env: - MOUNT_SELECTED_LOCAL_SOURCES: "false" ANSWER: "yes" DB_RESET: "true" VERBOSE: "true" @@ -963,7 +962,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" needs: [build-info, wait-for-ci-images] env: RUNS_ON: ${{ needs.build-info.outputs.runs-on }} - MOUNT_SELECTED_LOCAL_SOURCES: "true" TEST_TYPES: "Helm" BACKEND: "" DB_RESET: "false" @@ -1055,10 +1053,28 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Test Offline SQL generation" - run: ./scripts/ci/testing/run_offline_sql_test.sh + - name: "Test downgrade migration file ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset --skip-init -y && + airflow db upgrade --to-revision heads && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Test downgrade ORM ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset -y && + airflow db upgrade && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Test Offline SQL generation ${{ env.BACKEND }}" + run: > + breeze shell "airflow db upgrade --from-version 2.0.0 -r heads --show-sql-only && + airflow db downgrade --to-version 2.0.0 --show-sql-only -y" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: breeze testing tests --run-in-parallel env: @@ -1129,8 +1145,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh + - name: "Test downgrade migration file ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset --skip-init -y && + airflow db upgrade --to-revision heads && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Test downgrade ORM ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset -y && + airflow db upgrade && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: breeze testing tests --run-in-parallel env: @@ -1201,8 +1231,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh + - name: "Test downgrade migration file ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset --skip-init -y && + airflow db upgrade --to-revision heads && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Test downgrade ORM ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset -y && + airflow db upgrade && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: breeze testing tests --run-in-parallel env: @@ -1271,8 +1315,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze ci-image pull --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh + - name: "Test downgrade migration file ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset --skip-init -y && + airflow db upgrade --to-revision heads && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: "Test downgrade ORM ${{ env.BACKEND }}" + run: > + breeze shell "airflow db reset -y && + airflow db upgrade && + airflow db downgrade -r e959f08ac86c -y && + airflow db upgrade" + env: + IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: breeze testing tests --run-in-parallel env: @@ -1448,11 +1506,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze ci free-space - - name: "Cache virtualenv environment" - uses: actions/cache@v3 - with: - path: '.build/.docker_venv' - key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} # We wait for the images to be available either from "build-images.yml' run as pull_request_target # or from build-prod-images above. diff --git a/CI.rst b/CI.rst index b155ccc370559..5c937e579431f 100644 --- a/CI.rst +++ b/CI.rst @@ -487,7 +487,8 @@ the sources are not mapped from your host machine. Depending whether the scripts are run locally via `Breeze `_ or whether they are run in ``Build Images`` or ``Tests`` workflows they can take different values. -You can use those variables when you try to reproduce the build locally. +You can use those variables when you try to reproduce the build locally (alternatively you can pass +those via command line flags passed to ``breeze`` command. +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | Variable | Local | Build Images | CI | Comment | @@ -505,29 +506,7 @@ You can use those variables when you try to reproduce the build locally. | | | | | it requires to perform manual init/reset | | | | | | if you stop the environment. | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| Mount variables | -+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| ``MOUNT_SELECTED_LOCAL_SOURCES`` | true | false | false | Determines whether local sources are | -| | | | | mounted to inside the container. Useful for | -| | | | | local development, as changes you make | -| | | | | locally can be immediately tested in | -| | | | | the container. We mount only selected, | -| | | | | important folders. We do not mount the whole | -| | | | | project folder in order to avoid accidental | -| | | | | use of artifacts (such as ``egg-info`` | -| | | | | directories) generated locally on the | -| | | | | host during development. | -+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| ``MOUNT_ALL_LOCAL_SOURCES`` | false | false | false | Determines whether all local sources are | -| | | | | mounted to inside the container. Useful for | -| | | | | local development when you need to access .git | -| | | | | folders and other folders excluded when | -| | | | | ``MOUNT_SELECTED_LOCAL_SOURCES`` is true. | -| | | | | You might need to manually delete egg-info | -| | | | | folder when you enter breeze and the folder was | -| | | | | generated using different Python versions. | -+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| Force variables | +| Forcing answer | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | ``ANSWER`` | | yes | yes | This variable determines if answer to questions | | | | | | during the build process should be | @@ -550,7 +529,7 @@ You can use those variables when you try to reproduce the build locally. +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | ``COMMIT_SHA`` | | GITHUB_SHA | GITHUB_SHA | SHA of the commit of the build is run | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| Initialization | +| In container environment initialization | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | ``SKIP_ENVIRONMENT_INITIALIZATION`` | false\* | false\* | false\* | Skip initialization of test environment | | | | | | | @@ -560,35 +539,12 @@ You can use those variables when you try to reproduce the build locally. | | | | | | | | | | | \* set to true in GitHub CodeSpaces | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| Verbosity variables | -+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| ``PRINT_INFO_FROM_SCRIPTS`` | true\* | true\* | true\* | Allows to print output to terminal from running | -| | | | | scripts. It prints some extra outputs if true | -| | | | | including what the commands do, results of some | -| | | | | operations, summary of variable values, exit | -| | | | | status from the scripts, outputs of failing | -| | | | | commands. If verbose is on it also prints the | -| | | | | commands executed by docker, kind, helm, | -| | | | | kubectl. Disabled in pre-commit checks. | -| | | | | | -| | | | | \* set to false in pre-commits | -+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| ``VERBOSE`` | false | true | true | Determines whether docker, helm, kind, | -| | | | | kubectl commands should be printed before | -| | | | | execution. This is useful to determine | -| | | | | what exact commands were executed for | -| | | | | debugging purpose as well as allows | -| | | | | to replicate those commands easily by | -| | | | | copy&pasting them from the output. | -| | | | | requires ``PRINT_INFO_FROM_SCRIPTS`` set to | -| | | | | true. | -+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | ``VERBOSE_COMMANDS`` | false | false | false | Determines whether every command | -| | | | | executed in bash should also be printed | +| | | | | executed in docker should also be printed | | | | | | before execution. This is a low-level | -| | | | | debugging feature of bash (set -x) and | -| | | | | it should only be used if you are lost | -| | | | | at where the script failed. | +| | | | | debugging feature of bash (set -x) enabled in | +| | | | | entrypoint and it should only be used if you | +| | | | | need to debug the bash scripts in container. | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | Image build variables | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index d6939fb5760e8..ac44f289455c4 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -358,31 +358,7 @@ Run script to re-tag images from the ``main`` branch to the ``vX-Y-test`` branc ### Update default branches -#### In the legacy, bash breeze (to be removed when the bash breeze is entirely gone) - -In ``./scripts/ci/libraries/_intialization.sh`` update branches to reflect the new branch: - -```bash -export DEFAULT_BRANCH=${DEFAULT_BRANCH="main"} -export DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH="constraints-main"} -``` - -should become this, where ``X-Y`` is your new branch version: - -```bash -export DEFAULT_BRANCH=${DEFAULT_BRANCH="vX-Y-test"} -export DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH="constraints-X-Y"} -``` - -In ``./scripts/ci/libraries/_build_images.sh`` add branch to preload packages from (replace X and Y in -values for comparison and regexp): - -```bash - elif [[ ${AIRFLOW_VERSION} =~ v?X\.Y* ]]; then - AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="vX-Y-stable" -``` - -#### In the new breeze +#### In Breeze In ``./dev/breeze/src/airflow_breeze/branch_defaults.py`` update branches to reflect the new branch: diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 53f3a3cb1e299..df6387af99e54 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -589,7 +589,6 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, 'LOAD_DEFAULT_CONNECTIONS', "false") set_value_to_default_if_not_set(env, 'LOAD_EXAMPLES', "false") set_value_to_default_if_not_set(env, 'PACKAGE_FORMAT', ALLOWED_PACKAGE_FORMATS[0]) - set_value_to_default_if_not_set(env, 'PRINT_INFO_FROM_SCRIPTS', "true") set_value_to_default_if_not_set(env, 'PYTHONDONTWRITEBYTECODE', "true") set_value_to_default_if_not_set(env, 'RUN_SYSTEM_TESTS', "false") set_value_to_default_if_not_set(env, 'RUN_TESTS', "false") diff --git a/scripts/ci/constraints/ci_branch_constraints.sh b/scripts/ci/constraints/ci_branch_constraints.sh index b8a3b272a1c0d..4883da4d28ee0 100755 --- a/scripts/ci/constraints/ci_branch_constraints.sh +++ b/scripts/ci/constraints/ci_branch_constraints.sh @@ -15,9 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - if [[ ${GITHUB_REF} == 'refs/heads/main' ]]; then echo "::set-output name=branch::constraints-main" elif [[ ${GITHUB_REF} =~ refs/heads/v([0-9\-]*)\-(test|stable) ]]; then diff --git a/scripts/ci/constraints/ci_commit_constraints.sh b/scripts/ci/constraints/ci_commit_constraints.sh index e3da5e20e26df..b066f773e9cf5 100755 --- a/scripts/ci/constraints/ci_commit_constraints.sh +++ b/scripts/ci/constraints/ci_commit_constraints.sh @@ -15,9 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - cp -v ./files/constraints-*/constraints*.txt repo/ cd repo || exit 1 git config --local user.email "dev@airflow.apache.org" diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index 30907589dba05..2fdc9079b4034 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -52,7 +52,6 @@ MYSQL_VERSION NUM_RUNS PACKAGE_FORMAT POSTGRES_VERSION -PRINT_INFO_FROM_SCRIPTS PYTHONDONTWRITEBYTECODE PYTHON_MAJOR_MINOR_VERSION RUN_TESTS diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 8db997f50e30f..f42820bb2f461 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -65,7 +65,6 @@ services: - NUM_RUNS=${NUM_RUNS} - PACKAGE_FORMAT=${PACKAGE_FORMAT} - POSTGRES_VERSION=${POSTGRES_VERSION} - - PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS} - PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE} - PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} - RUN_TESTS=${RUN_TESTS} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index f59af39a7524f..5b60ce529e785 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -52,7 +52,6 @@ MYSQL_VERSION=5.7 NUM_RUNS= PACKAGE_FORMAT= POSTGRES_VERSION=10 -PRINT_INFO_FROM_SCRIPTS="false" PYTHONDONTWRITEBYTECODE="true" RUN_TESTS="false" LIST_OF_INTEGRATION_TESTS_TO_RUN="" diff --git a/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh b/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh index 5cf486e611e88..f54f9dc84ab0e 100755 --- a/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh +++ b/scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh @@ -15,9 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" # This is an AMI that is based on Basic Amazon Linux AMI with installed and configured docker service WORKING_DIR="/tmp/armdocker" diff --git a/scripts/ci/images/ci_stop_arm_instance.sh b/scripts/ci/images/ci_stop_arm_instance.sh index 6833c55670f59..94e14623e011d 100755 --- a/scripts/ci/images/ci_stop_arm_instance.sh +++ b/scripts/ci/images/ci_stop_arm_instance.sh @@ -15,9 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - # This is an AMI that is based on Basic Amazon Linux AMI with installed and configured docker service WORKING_DIR="/tmp/armdocker" INSTANCE_INFO="${WORKING_DIR}/instance_info.json" diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh deleted file mode 100755 index 5f64e4cd05b86..0000000000000 --- a/scripts/ci/libraries/_all_libs.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -LIBRARIES_DIR=$(dirname "${BASH_SOURCE[0]}") -export LIBRARIES_DIR -readonly LIBRARIES_DIR - -SCRIPTS_CI_DIR=$(cd "${LIBRARIES_DIR}/.." && pwd) -export SCRIPTS_CI_DIR -readonly SCRIPTS_CI_DIR - -# shellcheck source=scripts/ci/libraries/_traps.sh -. "${LIBRARIES_DIR}"/_traps.sh -# shellcheck source=scripts/ci/libraries/_initialization.sh -. "${LIBRARIES_DIR}"/_initialization.sh -# shellcheck source=scripts/ci/libraries/_sanity_checks.sh -. "${LIBRARIES_DIR}"/_sanity_checks.sh -# shellcheck source=scripts/ci/libraries/_local_mounts.sh -. "${LIBRARIES_DIR}"/_local_mounts.sh -# shellcheck source=scripts/ci/libraries/_start_end.sh -. "${LIBRARIES_DIR}"/_start_end.sh -# shellcheck source=scripts/ci/libraries/_testing.sh -. "${LIBRARIES_DIR}"/_testing.sh -# shellcheck source=scripts/ci/libraries/_verbosity.sh -. "${LIBRARIES_DIR}"/_verbosity.sh diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh deleted file mode 100644 index 42b6896319e68..0000000000000 --- a/scripts/ci/libraries/_initialization.sh +++ /dev/null @@ -1,852 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Needs to be declared outside function in MacOS -# shellcheck disable=SC2034 -CURRENT_PYTHON_MAJOR_MINOR_VERSIONS=() -CURRENT_POSTGRES_VERSIONS=() -CURRENT_MYSQL_VERSIONS=() -CURRENT_MSSQL_VERSIONS=() -CURRENT_EXECUTOR=() -ALL_PYTHON_MAJOR_MINOR_VERSIONS=() -INSTALLED_PROVIDERS=() - -# Creates directories for Breeze -function initialization::create_directories() { - # This folder is mounted to inside the container in /files folder. This is the way how - # We can exchange DAGs, scripts, packages etc with the container environment - export FILES_DIR="${AIRFLOW_SOURCES}/files" - readonly FILES_DIR - - # Directory where all the build cache is stored - we keep there status of all the docker images - # As well as hashes of the important files, but also we generate build scripts there that are - # Used to execute the commands for breeze - export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build" - readonly BUILD_CACHE_DIR - - # In case of tmpfs backend for docker, mssql fails because TMPFS does not support - # O_DIRECT parameter for direct writing to the filesystem - # https://github.com/microsoft/mssql-docker/issues/13 - # so we need to mount an external volume for its db location - # the external db must allow for parallel testing so external volume is mapped - # to the data volume - export MSSQL_DATA_VOLUME="${BUILD_CACHE_DIR}/tmp_mssql_volume" - - # Create those folders above in case they do not exist - mkdir -p "${BUILD_CACHE_DIR}" >/dev/null - mkdir -p "${FILES_DIR}" >/dev/null - mkdir -p "${MSSQL_DATA_VOLUME}" >/dev/null - # MSSQL 2019 runs with non-root user by default so we have to make the volumes world-writeable - # This is a bit scary and we could get by making it group-writeable but the group would have - # to be set to "root" (GID=0) for the volume to work and this cannot be accomplished without sudo - chmod a+rwx "${MSSQL_DATA_VOLUME}" - - # By default we are not in CI environment GitHub Actions sets CI to "true" - export CI="${CI="false"}" - - # Create useful directories if not yet created - mkdir -p "${AIRFLOW_SOURCES}/logs" - mkdir -p "${AIRFLOW_SOURCES}/dist" - - CACHE_TMP_FILE_DIR=$(mktemp -d) - export CACHE_TMP_FILE_DIR - readonly CACHE_TMP_FILE_DIR - - if [[ ${SKIP_CACHE_DELETION=} != "true" ]]; then - traps::add_trap "rm -rf -- '${CACHE_TMP_FILE_DIR}'" EXIT HUP INT TERM - fi - - OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log" - export OUTPUT_LOG - readonly OUTPUT_LOG -} - -# Very basic variables that MUST be set -function initialization::initialize_base_variables() { - # until we have support for ARM images, we set docker default platform to linux/AMD - # so that all breeze commands use emulation - local machine - if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then - machine="arm64" - else - machine="amd64" - fi - - export PLATFORM=${PLATFORM:="linux/${machine}"} - - # enable buildkit for builds - export DOCKER_BUILDKIT=1 - - # Default port numbers for forwarded ports - export SSH_PORT=${SSH_PORT:="12322"} - export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"} - export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"} - export MYSQL_HOST_PORT=${MYSQL_HOST_PORT:="23306"} - export MSSQL_HOST_PORT=${MSSQL_HOST_PORT:="21433"} - export FLOWER_HOST_PORT=${FLOWER_HOST_PORT:="25555"} - export REDIS_HOST_PORT=${REDIS_HOST_PORT:="26379"} - - # The SQLite URL used for sqlite runs - export SQLITE_URL="sqlite:////root/airflow/airflow.db" - - # Disable writing .pyc files - slightly slower imports but not messing around when switching - # Python version and avoids problems with root-owned .pyc files in host - export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"} - - # By default we build CI images but we can switch to production image with PRODUCTION_IMAGE="true" - export PRODUCTION_IMAGE="false" - - # All supported major/minor versions of python in all versions of Airflow - ALL_PYTHON_MAJOR_MINOR_VERSIONS+=("3.7" "3.8" "3.9" "3.10") - export ALL_PYTHON_MAJOR_MINOR_VERSIONS - - # Currently supported major/minor versions of python - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS+=("3.7" "3.8" "3.9" "3.10") - export CURRENT_PYTHON_MAJOR_MINOR_VERSIONS - - # Currently supported versions of Postgres - CURRENT_POSTGRES_VERSIONS+=("10" "14") - export CURRENT_POSTGRES_VERSIONS - - # Currently supported versions of MySQL - CURRENT_MYSQL_VERSIONS+=("5.7" "8") - export CURRENT_MYSQL_VERSIONS - - # Currently supported versions of MSSQL - CURRENT_MSSQL_VERSIONS+=("2017-latest" "2019-latest") - export CURRENT_MSSQL_VERSIONS - - BACKEND=${BACKEND:="sqlite"} - export BACKEND - - # Default Postgres versions - export POSTGRES_VERSION=${POSTGRES_VERSION:=${CURRENT_POSTGRES_VERSIONS[0]}} - - # Default MySQL versions - export MYSQL_VERSION=${MYSQL_VERSION:=${CURRENT_MYSQL_VERSIONS[0]}} - - #Default MS SQL version - export MSSQL_VERSION=${MSSQL_VERSION:=${CURRENT_MSSQL_VERSIONS[0]}} - - # If set to true, the database will be reset at entry. Works for Postgres and MySQL - export DB_RESET=${DB_RESET:="false"} - - # If set to true, the database will be initialized, a user created and webserver and scheduler started - export START_AIRFLOW=${START_AIRFLOW:="false"} - - # If set to true, the sample dags will be used - export LOAD_EXAMPLES=${LOAD_EXAMPLES:="false"} - - # If set to true, the test connections will be created - export LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS:="false"} - - # If set to true, Breeze db volumes will be persisted when breeze is stopped and reused next time - # Which means that you do not have to start from scratch - export PRESERVE_VOLUMES="false" - - # Cleans up docker context files if specified - export CLEANUP_CONTEXT="false" - - # if set to true, the ci image will look for packages in dist folder and will install them - # during entering the container - export USE_PACKAGES_FROM_DIST=${USE_PACKAGES_FROM_DIST:="false"} - - # If set the specified file will be used to initialize Airflow after the environment is created, - # otherwise it will use files/airflow-breeze-config/init.sh - export INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE:=""} - - # Read airflow version from the setup.py. - AIRFLOW_VERSION=$(awk '/^version =/ {print $3}' "${AIRFLOW_SOURCES}/setup.py" | sed "s/['+]//g") - export AIRFLOW_VERSION - - # Whether credentials should be forwarded to inside the docker container - export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"} - - # If no Airflow Home defined - fallback to ${HOME}/airflow - AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow} - export AIRFLOW_HOME_DIR - - # Dry run - only show docker-compose and docker commands but do not execute them - export DRY_RUN_DOCKER=${DRY_RUN_DOCKER:="false"} - -} - -# Determine current branch -function initialization::initialize_branch_variables() { - # Default branch used - this will be different in different branches - export DEFAULT_BRANCH=${DEFAULT_BRANCH="main"} - export DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH="constraints-main"} - readonly DEFAULT_BRANCH - readonly DEFAULT_CONSTRAINTS_BRANCH - - # Default branch name for triggered builds is the one configured in default branch - # We need to read it here as it comes from _common_values.sh - export BRANCH_NAME=${BRANCH_NAME:=${DEFAULT_BRANCH}} -} - -# Determine available integrations -function initialization::initialize_available_integrations() { - export AVAILABLE_INTEGRATIONS="cassandra kerberos mongo openldap pinot rabbitmq redis statsd trino" -} - -# Needs to be declared outside of function for MacOS -FILES_FOR_REBUILD_CHECK=() - -# Determine which files trigger rebuild check -function initialization::initialize_files_for_rebuild_check() { - FILES_FOR_REBUILD_CHECK+=( - "setup.py" - "setup.cfg" - "Dockerfile.ci" - ".dockerignore" - "scripts/docker/common.sh" - "scripts/docker/install_additional_dependencies.sh" - "scripts/docker/install_airflow.sh" - "scripts/docker/install_airflow_dependencies_from_branch_tip.sh" - "scripts/docker/install_from_docker_context_files.sh" - "scripts/docker/install_mysql.sh" - "scripts/docker/install_postgres.sh" - ) -} - -# Needs to be declared outside of function for MacOS - -# extra flags passed to docker run for PROD image -# shellcheck disable=SC2034 -EXTRA_DOCKER_PROD_BUILD_FLAGS=() - -# files that should be cleaned up when the script exits -# shellcheck disable=SC2034 -FILES_TO_CLEANUP_ON_EXIT=() - -# extra flags passed to docker run for CI image -# shellcheck disable=SC2034 -EXTRA_DOCKER_FLAGS=() - -# Determine behaviour of mounting sources to the container -function initialization::initialize_mount_variables() { - - # Whether necessary for airflow run local sources are mounted to docker - export MOUNT_SELECTED_LOCAL_SOURCES=${MOUNT_SELECTED_LOCAL_SOURCES:="true"} - - # Whether all airflow sources are mounted to docker - export MOUNT_ALL_LOCAL_SOURCES=${MOUNT_ALL_LOCAL_SOURCES:="false"} - - if [[ ${MOUNT_SELECTED_LOCAL_SOURCES} == "true" ]]; then - verbosity::print_info - verbosity::print_info "Mounting necessary host volumes to Docker" - verbosity::print_info - read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)" - elif [[ ${MOUNT_ALL_LOCAL_SOURCES} == "true" ]]; then - verbosity::print_info - verbosity::print_info "Mounting whole airflow volume to Docker" - verbosity::print_info - EXTRA_DOCKER_FLAGS+=("-v" "${AIRFLOW_SOURCES}:/opt/airflow/:cached") - else - verbosity::print_info - verbosity::print_info "Skip mounting host volumes to Docker" - verbosity::print_info - fi - - EXTRA_DOCKER_FLAGS+=( - "-v" "${AIRFLOW_SOURCES}/files:/files" - "-v" "${AIRFLOW_SOURCES}/dist:/dist" - "--rm" - "--env-file" "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" - ) - export EXTRA_DOCKER_FLAGS -} - -# Determine values of force settings -function initialization::initialize_force_variables() { - - # Can be set to "yes/no/quit" in order to force specified answer to all questions asked to the user. - export ANSWER=${ANSWER:=""} - - # integrations are disabled by default - export ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""} - - # systems are disabled by default - export ENABLED_SYSTEMS=${ENABLED_SYSTEMS:=""} - - # no issue id by default (quarantined builds only) - export ISSUE_ID=${ISSUE_ID:=""} - - # no NUM_RUNS by default (quarantined builds only) - export NUM_RUNS=${NUM_RUNS:=""} - -} - -# Determine information about the host -function initialization::initialize_host_variables() { - # Set host user id to current user. This is used to set the ownership properly when exiting - # The container on Linux - all files created inside docker are created with root user - # but they should be restored back to the host user - HOST_USER_ID="$(id -ur)" - export HOST_USER_ID - - # Set host group id to current group This is used to set the ownership properly when exiting - # The container on Linux - all files created inside docker are created with root user - # but they should be restored back to the host user - HOST_GROUP_ID="$(id -gr)" - export HOST_GROUP_ID - - # Set host OS. This is used to set the ownership properly when exiting - # The container on Linux - all files created inside docker are created with root user - # but they should be restored back to the host user - HOST_OS="$(uname -s | tr '[:upper:]' '[:lower:]')" - export HOST_OS - - # In case of MacOS we need to use gstat - gnu version of the stats - export STAT_BIN=stat - if [[ "${OSTYPE}" == "darwin"* ]]; then - export STAT_BIN=gstat - fi -} - -# Determine image augmentation parameters -function initialization::initialize_image_build_variables() { - # Default build id - export CI_BUILD_ID="${CI_BUILD_ID:="0"}" - - # Default extras used for building Production image. The canonical source of this information is in the Dockerfile - DEFAULT_PROD_EXTRAS=$(grep "ARG AIRFLOW_EXTRAS=" "${AIRFLOW_SOURCES}/Dockerfile" | - awk 'BEGIN { FS="=" } { print $2 }' | tr -d '"') - export DEFAULT_PROD_EXTRAS - - # By default we are not upgrading to latest version of constraints when building Docker CI image - # This will only be done in cron jobs - export UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES:="false"} - - # Checks if the image should be rebuilt - export CHECK_IMAGE_FOR_REBUILD="${CHECK_IMAGE_FOR_REBUILD:="true"}" - - # Skips building production images altogether (assume they are already built) - export SKIP_BUILDING_PROD_IMAGE="${SKIP_BUILDING_PROD_IMAGE:="false"}" - - # Additional airflow extras on top of the default ones - export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}" - # Additional python dependencies on top of the default ones - export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}" - # Use default DEV_APT_COMMAND - export DEV_APT_COMMAND="" - # Use default DEV_APT_DEPS - export DEV_APT_DEPS="" - # Use empty ADDITIONAL_DEV_APT_COMMAND - export ADDITIONAL_DEV_APT_COMMAND="" - # additional development apt dependencies on top of the default ones - export ADDITIONAL_DEV_APT_DEPS="${ADDITIONAL_DEV_APT_DEPS:=""}" - # Use empty ADDITIONAL_DEV_APT_ENV - export ADDITIONAL_DEV_APT_ENV="${ADDITIONAL_DEV_APT_ENV:=""}" - # Use default RUNTIME_APT_COMMAND - export RUNTIME_APT_COMMAND="" - # Use default RUNTIME_APT_DEPS - export RUNTIME_APT_DEPS="" - # Use empty ADDITIONAL_RUNTIME_APT_COMMAND - export ADDITIONAL_RUNTIME_APT_COMMAND="" - # additional runtime apt dependencies on top of the default ones - export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}" - export ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS:=""}" - # Use empty ADDITIONAL_RUNTIME_APT_ENV - export ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV:=""}" - # whether pre cached pip packages are used during build - export AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES:="true"}" - # by default install mysql client - export INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT:="true"} - # by default install mssql client - export INSTALL_MSSQL_CLIENT=${INSTALL_MSSQL_CLIENT:="true"} - # by default install postgres client - export INSTALL_POSTGRES_CLIENT=${INSTALL_POSTGRES_CLIENT:="true"} - # additional tag for the image - export IMAGE_TAG=${IMAGE_TAG:=""} - - INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES:="true"} - export INSTALL_PROVIDERS_FROM_SOURCES - - SKIP_SSH_SETUP=${SKIP_SSH_SETUP:="false"} - export SKIP_SSH_SETUP - - SKIP_ENVIRONMENT_INITIALIZATION=${SKIP_ENVIRONMENT_INITIALIZATION:="false"} - export SKIP_ENVIRONMENT_INITIALIZATION - - export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv" - - AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="22.2.2"} - export AIRFLOW_PIP_VERSION - - # We also pin version of wheel used to get consistent builds - WHEEL_VERSION=${WHEEL_VERSION:="0.36.2"} - export WHEEL_VERSION - - # And installed from there (breeze and ci) - AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION:=""} - export AIRFLOW_VERSION_SPECIFICATION - - # By default no sources are copied to image - AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="Dockerfile"} - export AIRFLOW_SOURCES_FROM - - AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/Dockerfile"} - export AIRFLOW_SOURCES_TO - - # By default in scripts production docker image is installed from PyPI package - export AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD:="apache-airflow"} - - # Installs different airflow version than current from the sources - export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""} - - # Determines if airflow should be installed from a specified reference in GitHub - export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} - - # Determines which providers are used to generate constraints - source, pypi or no providers - export AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE:="constraints-source-providers"} - - # whether installation should be performed from the local wheel packages in "docker-context-files" folder - export INSTALL_PACKAGES_FROM_CONTEXT="${INSTALL_PACKAGES_FROM_CONTEXT:="false"}" - - # reference to CONSTRAINTS. they can be overwritten manually or replaced with AIRFLOW_CONSTRAINTS_LOCATION - export AIRFLOW_CONSTRAINTS_REFERENCE="${AIRFLOW_CONSTRAINTS_REFERENCE:=""}" - - # direct constraints Location - can be URL or path to local file. If empty, it will be calculated - # based on which Airflow version is installed and from where - export AIRFLOW_CONSTRAINTS_LOCATION="${AIRFLOW_CONSTRAINTS_LOCATION:=""}" - - # Suffix for constraints. Can be: - # * 'constraints' = for constraints with PyPI released providers (default for installations) - # * 'constraints-source-providers' for constraints with source version of providers (defaults in Breeze and CI) - # * 'constraints-no-providers' for constraints without providers - export AIRFLOW_CONSTRAINTS_MODE="${AIRFLOW_CONSTRAINTS_MODE:="constraints-source-providers"}" - - # Replace airflow at runtime in CI image with the one specified - # * none - just removes airflow - # * wheel - replaces airflow with one specified in the wheel file in /dist - # * wheel - replaces airflow with one specified in the sdist file in /dist - # * - replaces airflow with the specific version from PyPI - export USE_AIRFLOW_VERSION=${USE_AIRFLOW_VERSION:=""} - - # whether images should be pushed to registry cache after they are built - export PREPARE_BUILDX_CACHE=${PREPARE_BUILDX_CACHE:="false"} -} - -function initialization::initialize_virtualenv_variables() { - # The extras to install when initializing a virtual env with breeze - export VIRTUALENV_EXTRAS=${VIRTUALENV_EXTRAS:="devel"} -} - -function initialization::initialize_git_variables() { - # SHA of the commit for the current sources - COMMIT_SHA="$(git rev-parse HEAD 2>/dev/null || echo "Unknown")" - export COMMIT_SHA -} - -function initialization::initialize_github_variables() { - - export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"} - # Allows to override the repository which is used as source of constraints during the build - export CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY:="apache/airflow"} - - # Used only in CI environment - export GITHUB_TOKEN="${GITHUB_TOKEN=""}" - export GITHUB_USERNAME="${GITHUB_USERNAME=""}" -} - -function initialization::initialize_test_variables() { - - #Enables test coverage - export ENABLE_TEST_COVERAGE=${ENABLE_TEST_COVERAGE:=""} - - # In case we want to force certain test type to run, this variable should be set to this type - # Otherwise TEST_TYPEs to run will be derived from TEST_TYPES space-separated string - export FORCE_TEST_TYPE=${FORCE_TEST_TYPE:=""} - - # Do not run tests by default - export RUN_TESTS=${RUN_TESTS:="false"} - - # Do not run integration tests by default - export LIST_OF_INTEGRATION_TESTS_TO_RUN=${LIST_OF_INTEGRATION_TESTS_TO_RUN:=""} - - # Do not run system tests by default (they can be enabled by setting the RUN_SYSTEM_TESTS variable to "true") - export RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS:=""} - -} - -function initialization::set_output_color_variables() { - COLOR_BLUE=$'\e[34m' - COLOR_GREEN=$'\e[32m' - COLOR_RED=$'\e[31m' - COLOR_RESET=$'\e[0m' - COLOR_YELLOW=$'\e[33m' - COLOR_CYAN=$'\e[36m' - export COLOR_BLUE - export COLOR_GREEN - export COLOR_RED - export COLOR_RESET - export COLOR_YELLOW - export COLOR_CYAN -} - -# Common environment that is initialized by both Breeze and CI scripts -function initialization::initialize_common_environment() { - initialization::set_output_color_variables - initialization::initialize_base_variables - initialization::initialize_branch_variables - initialization::initialize_available_integrations - initialization::initialize_files_for_rebuild_check - initialization::initialize_mount_variables - initialization::initialize_force_variables - initialization::initialize_host_variables - initialization::initialize_image_build_variables - initialization::initialize_virtualenv_variables - initialization::initialize_git_variables - initialization::initialize_github_variables - initialization::initialize_test_variables -} - -function initialization::set_default_python_version_if_empty() { - # default version of python used to tag the "main" and "latest" images in DockerHub - export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.7 - export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}} -} - - -# Retrieves GitHub Container Registry image prefix from repository name -# GitHub Container Registry stores all images at the organization level, they are just -# linked to the repository via docker label - however we assume a convention where we will -# add repository name to organisation separated by '-' and convert everything to lowercase -# this is because in order for it to work for internal PR for users or other organisation's -# repositories, the other organisations and repositories can be uppercase -# container registry image name has to be lowercase -function initialization::get_github_container_registry_image_prefix() { - echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]' -} - -function initialization::get_docker_cache_image_names() { - # Python base image to use - export PYTHON_BASE_IMAGE="python:${PYTHON_MAJOR_MINOR_VERSION}-slim-bullseye" - - local image_name - image_name="ghcr.io/$(initialization::get_github_container_registry_image_prefix)" - - # Example: - # ghcr.io/apache/airflow/main/ci/python3.8 - export AIRFLOW_CI_IMAGE="${image_name}/${BRANCH_NAME}/ci/python${PYTHON_MAJOR_MINOR_VERSION}" - - # Example: - # ghcr.io/apache/airflow/main/ci/python3.8:latest - # ghcr.io/apache/airflow/main/ci/python3.8: - export AIRFLOW_CI_IMAGE_WITH_TAG="${image_name}/${BRANCH_NAME}/ci/python${PYTHON_MAJOR_MINOR_VERSION}:latest" - - # File that is touched when the CI image is built for the first time locally - export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}" - - # Example: - # ghcr.io/apache/airflow/main/prod/python3.8 - export AIRFLOW_PROD_IMAGE="${image_name}/${BRANCH_NAME}/prod/python${PYTHON_MAJOR_MINOR_VERSION}" -} - -function initialization::summarize_build_environment() { - cat <>"${GITHUB_ENV}" - fi -} - -function initialization::ver() { - # convert SemVer number to comparable string (strips pre-release version) - # shellcheck disable=SC2086,SC2183 - printf "%03d%03d%03d%.0s" ${1//[.-]/ } -} - -function initialization::check_docker_version() { - local permission_denied - permission_denied=$(docker info 2>/dev/null | grep "ERROR: Got permission denied while trying " || true) - if [[ ${permission_denied} != "" ]]; then - echo - echo "${COLOR_RED}ERROR: You have 'permission denied' error when trying to communicate with docker.${COLOR_RESET}" - echo - echo "${COLOR_YELLOW}Most likely you need to add your user to 'docker' group: https://docs.docker.com/engine/install/linux-postinstall/ .${COLOR_RESET}" - echo - exit 1 - fi - local docker_version - # In GitHub Code QL, the version of docker has +azure suffix which we should remove - docker_version=$(docker version --format '{{.Client.Version}}' | sed 's/\+.*$//' || true) - if [ "${docker_version}" == "" ]; then - echo - echo "${COLOR_YELLOW}Your version of docker is unknown. If the scripts fail, please make sure to install docker at least: ${min_docker_version} version.${COLOR_RESET}" - echo - return - fi - local comparable_docker_version - comparable_docker_version=$(initialization::ver "${docker_version}") - local min_docker_version="20.10.0" - local min_comparable_docker_version - min_comparable_docker_version=$(initialization::ver "${min_docker_version}") - # The #0 Strips leading zeros - if [[ ${comparable_docker_version#0} -lt ${min_comparable_docker_version#0} ]]; then - echo - echo "${COLOR_RED}Your version of docker is too old: ${docker_version}. Please upgrade to at least ${min_docker_version}.${COLOR_RESET}" - echo - exit 1 - else - if [[ ${PRINT_INFO_FROM_SCRIPTS} != "false" ]]; then - echo "${COLOR_GREEN}Good version of docker ${docker_version}.${COLOR_RESET}" - fi - fi -} diff --git a/scripts/ci/libraries/_local_mounts.sh b/scripts/ci/libraries/_local_mounts.sh deleted file mode 100644 index a51840577b658..0000000000000 --- a/scripts/ci/libraries/_local_mounts.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Those are files that are mounted locally when mounting local sources is requested -# By default not the whole airflow sources directory is mounted because there are often -# artifacts created there (for example .egg-info files) that are breaking the capability -# of running different python versions in Breeze. So we only mount what is needed by default. -function local_mounts::generate_local_mounts_list { - local prefix="$1" - LOCAL_MOUNTS=( - "$prefix".bash_aliases:/root/.bash_aliases:cached - "$prefix".bash_history:/root/.bash_history:cached - "$prefix".coveragerc:/opt/airflow/.coveragerc:cached - "$prefix".dockerignore:/opt/airflow/.dockerignore:cached - "$prefix".flake8:/opt/airflow/.flake8:cached - "$prefix".github:/opt/airflow/.github:cached - "$prefix".inputrc:/root/.inputrc:cached - "$prefix".rat-excludes:/opt/airflow/.rat-excludes:cached - "$prefix"RELEASE_NOTES.rst:/opt/airflow/RELEASE_NOTES.rst:cached - "$prefix"LICENSE:/opt/airflow/LICENSE:cached - "$prefix"MANIFEST.in:/opt/airflow/MANIFEST.in:cached - "$prefix"NOTICE:/opt/airflow/NOTICE:cached - "$prefix"airflow:/opt/airflow/airflow:cached - "$prefix"provider_packages:/opt/airflow/provider_packages:cached - "$prefix"dags:/opt/airflow/dags:cached - "$prefix"dev:/opt/airflow/dev:cached - "$prefix"docs:/opt/airflow/docs:cached - "$prefix"hooks:/opt/airflow/hooks:cached - "$prefix"logs:/root/airflow/logs:cached - "$prefix"pyproject.toml:/opt/airflow/pyproject.toml:cached - "$prefix"pytest.ini:/opt/airflow/pytest.ini:cached - "$prefix"scripts:/opt/airflow/scripts:cached - "$prefix"scripts/docker/entrypoint_ci.sh:/entrypoint:cached - "$prefix"setup.cfg:/opt/airflow/setup.cfg:cached - "$prefix"setup.py:/opt/airflow/setup.py:cached - "$prefix"tests:/opt/airflow/tests:cached - "$prefix"kubernetes_tests:/opt/airflow/kubernetes_tests:cached - "$prefix"docker_tests:/opt/airflow/docker_tests:cached - "$prefix"chart:/opt/airflow/chart:cached - "$prefix"metastore_browser:/opt/airflow/metastore_browser:cached - ) -} - -# Converts the local mounts that we defined above to the right set of -v -# volume mappings in docker-compose file. This is needed so that we only -# maintain the volumes in one place (above) -function local_mounts::convert_local_mounts_to_docker_params() { - local_mounts::generate_local_mounts_list "${AIRFLOW_SOURCES}/" - # Bash can't "return" arrays, so we need to quote any special characters - printf -- '-v %q ' "${LOCAL_MOUNTS[@]}" -} diff --git a/scripts/ci/libraries/_sanity_checks.sh b/scripts/ci/libraries/_sanity_checks.sh deleted file mode 100644 index 6a4ed38c44565..0000000000000 --- a/scripts/ci/libraries/_sanity_checks.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Fixes a file that is expected to be a file. If - for whatever reason - the local file is not created -# When mounting it to container, docker assumes it is a missing directory and creates it. Such mistakenly -# Created directories should be removed and replaced with files -function sanity_checks::sanitize_file() { - if [[ -d "${1}" ]]; then - rm -rf "${1}" - fi - touch "${1}" -} - -# -# Creates cache directory where we will keep temporary files needed for the docker build -# -# This directory will be automatically deleted when the script is killed or exists (via trap) -# Unless SKIP_CACHE_DELETION variable is set. You can set this variable and then see -# the output/files generated by the scripts in this directory. -# -# Most useful is out.log file in this directory storing verbose output of the scripts. -# -# -# Checks if core utils required in the host system are installed and explain what needs to be done if not -# -function sanity_checks::check_if_coreutils_installed() { - local getopt_retval - local stat_present - local md5sum_present - - set +e - getopt -T >/dev/null - getopt_retval=$? - - if [[ $(uname -s| tr '[:upper:]' '[:lower:]') == 'darwin' ]] ; then - command -v gstat >/dev/null - stat_present=$? - else - command -v stat >/dev/null - stat_present=$? - fi - - command -v md5sum >/dev/null - md5sum_present=$? - - set -e - - CMDNAME="$(basename -- "$0")" - export CMDNAME - readonly CMDNAME - - #################### Parsing options/arguments - if [[ ${getopt_retval} != 4 || "${stat_present}" != "0" || "${md5sum_present}" != "0" ]]; then - verbosity::print_info - if [[ $(uname -s| tr '[:upper:]' '[:lower:]') == 'darwin' ]] ; then - echo """ -${COLOR_RED}ERROR: You are running ${CMDNAME} in OSX environment and you need to install gnu commands - -Run 'brew install gnu-getopt coreutils' - -Then link the gnu-getopt to become default as suggested by brew. - -If you use bash, you should run these commands: - -echo 'export PATH=\"$(brew --prefix)/opt/gnu-getopt/bin:\$PATH\"' >> ~/.bash_profile -. ~/.bash_profile - -If you use zsh, you should run these commands: - -echo 'export PATH=\"$(brew --prefix)/opt/gnu-getopt/bin:\$PATH\"' >> ~/.zprofile -. ~/.zprofile - -Either source the profile file as shown above, or re-login afterwards. - -After that, your PATH variable should start with \"$(brew --prefix)/opt/gnu-getopt/bin\" -Your current path is ${PATH} -${COLOR_RESET} -""" - else - echo """ -${COLOR_RED}ERROR: You do not have necessary tools in your path (getopt, stat, md5sum). -Please install latest/GNU version of getopt and coreutils. -This can usually be done with 'apt install util-linux coreutils' -${COLOR_RESET} -""" - fi - verbosity::print_info - exit 1 - fi -} - -# -# Asserts that we are not inside of the container -# -function sanity_checks::assert_not_in_container() { - if [[ ${SKIP_IN_CONTAINER_CHECK:=} == "true" ]]; then - return - fi - if [[ -f /.dockerenv ]]; then - echo """ -\e[31mERROR: You are inside the Airflow docker container - -You should only run this script from the host. -Learn more about how we develop and test airflow at: -https://github.com/apache/airflow/blob/main/TESTING.rst - -""" - exit 1 - fi -} - -# Changes directory to local sources -function sanity_checks::go_to_airflow_sources { - pushd "${AIRFLOW_SOURCES}" >/dev/null 2>&1 || exit 1 -} - -# -# Performs basic quick checks common for most of the scripts in this directory -# -function sanity_checks::basic_sanity_checks() { - sanity_checks::assert_not_in_container - initialization::set_default_python_version_if_empty - sanity_checks::go_to_airflow_sources - sanity_checks::check_if_coreutils_installed -} diff --git a/scripts/ci/libraries/_script_init.sh b/scripts/ci/libraries/_script_init.sh deleted file mode 100755 index a85877e7cba1f..0000000000000 --- a/scripts/ci/libraries/_script_init.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eo pipefail - -if [[ $(uname -s | tr '[:upper:]' '[:lower:]') != "darwin" ]]; then - # do not fail with undefined variable on MacOS. The old Bash which is default on Mac OS - # fails with undefined variable when you are passing an empty variable and this causes - # problems for example when you try to pass empty list of arguments "${@}" - set -u -fi - -export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}" -readonly AIRFLOW_SOURCES - -# shellcheck source=scripts/ci/libraries/_all_libs.sh -. "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh" - -initialization::create_directories - -initialization::initialize_common_environment - -initialization::check_docker_version - -sanity_checks::basic_sanity_checks - -start_end::script_start - -initialization::get_docker_cache_image_names - -initialization::get_environment_for_builds_on_ci - -initialization::make_constants_read_only - -# Work around occasional unexplained failure on CI. Clear file flags on -# STDOUT (which is connected to a tmp file by GitHub Runner). -# The one error I did see: BlockingIOError: [Errno 11] write could not complete without blocking -[[ "$CI" == "true" ]] && python3 -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)" - -traps::add_trap start_end::script_end EXIT HUP INT TERM diff --git a/scripts/ci/libraries/_start_end.sh b/scripts/ci/libraries/_start_end.sh deleted file mode 100644 index fbca97dcea22c..0000000000000 --- a/scripts/ci/libraries/_start_end.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Starts group for GitHub Actions - makes logs much more readable -function start_end::group_start { - if [[ ${PRINT_INFO_FROM_SCRIPTS} != "false" ]]; then - if [[ ${GITHUB_ACTIONS=} == "true" ]]; then - echo "::group::${1}" - else - echo - echo "${1}" - echo - fi - fi -} - -# Ends group for GitHub Actions -function start_end::group_end { - if [[ ${PRINT_INFO_FROM_SCRIPTS} != "false" ]]; then - if [[ ${GITHUB_ACTIONS=} == "true" ]]; then - echo -e "\033[0m" # Disable any colors set in the group - echo "::endgroup::" - fi - fi -} - - -# -# Starts the script. -# If VERBOSE_COMMANDS variable is set to true, it enables verbose output of commands executed -# Also prints some useful diagnostics information at start of the script if VERBOSE is set to true -# -function start_end::script_start { - START_SCRIPT_TIME=$(date +%s) - verbosity::print_info "Running '${COLOR_GREEN}$(basename "$0")${COLOR_RESET}'" - if [[ "${GITHUB_ACTIONS=}" == "true" && ${VERBOSE_COMMANDS:="false"} == "false" ]]; then - return - fi - - verbosity::print_info - verbosity::print_info "${COLOR_BLUE}Log is redirected to '${OUTPUT_LOG}'${COLOR_RESET}" - verbosity::print_info - if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then - verbosity::print_info - verbosity::print_info "${COLOR_BLUE}Variable VERBOSE_COMMANDS Set to \"true\"${COLOR_RESET}" - verbosity::print_info "${COLOR_BLUE}You will see a lot of output${COLOR_RESET}" - verbosity::print_info - set -x - else - verbosity::print_info "You can increase verbosity by running 'export VERBOSE_COMMANDS=\"true\"" - if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then - verbosity::print_info "And skip deleting the output file with 'export SKIP_CACHE_DELETION=\"true\"" - fi - verbosity::print_info - set +x - fi -} - -# -# Trap function executed always at the end of the script. In case of verbose output it also -# Prints the exit code that the script exits with. Removes verbosity of commands in case it was run with -# command verbosity and in case the script was not run from Breeze (so via ci scripts) it displays -# total time spent in the script so that we can easily see it. -# -function start_end::script_end { - #shellcheck disable=2181 - local exit_code=$? - if [[ ${exit_code} != 0 ]]; then - # Finish previous group so that output can be written - # Cat output log in case we exit with error but only if we do not PRINT_INFO_FROM_SCRIPTS - # Because it will be printed immediately by "tee" - if [[ -f "${OUTPUT_LOG}" && ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then - cat "${OUTPUT_LOG}" - fi - start_end::group_end - echo - echo "${COLOR_RED}ERROR: The previous step completed with error. Please take a look at output above ${COLOR_RESET}" - echo - verbosity::print_info "${COLOR_RED}###########################################################################################${COLOR_RESET}" - verbosity::print_info "${COLOR_RED} EXITING WITH STATUS CODE ${exit_code}${COLOR_RESET}" - verbosity::print_info "${COLOR_RED}###########################################################################################${COLOR_RESET}" - fi - if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then - set +x - fi - - if [[ ${#FILES_TO_CLEANUP_ON_EXIT[@]} -gt 0 ]]; then - rm -rf -- "${FILES_TO_CLEANUP_ON_EXIT[@]}" - fi - - local end_script_time - end_script_time=$(date +%s) - local run_script_time - run_script_time=$((end_script_time-START_SCRIPT_TIME)) - if [[ ${BREEZE:=} != "true" && ${RUN_TESTS=} != "true" ]]; then - verbosity::print_info - verbosity::print_info "Finished the script ${COLOR_GREEN}$(basename "$0")${COLOR_RESET}" - verbosity::print_info "Elapsed time spent in the script: ${COLOR_BLUE}${run_script_time} seconds${COLOR_RESET}" - if [[ ${exit_code} == "0" ]]; then - verbosity::print_info "Exit code ${COLOR_GREEN}${exit_code}${COLOR_RESET}" - else - verbosity::print_info "Exit code ${COLOR_RED}${exit_code}${COLOR_RESET}" - fi - verbosity::print_info - fi -} diff --git a/scripts/ci/libraries/_testing.sh b/scripts/ci/libraries/_testing.sh deleted file mode 100644 index a198f5d8fd49b..0000000000000 --- a/scripts/ci/libraries/_testing.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export MEMORY_REQUIRED_FOR_HEAVY_TEST_PARALLEL_RUN=33000 - -function testing::get_docker_compose_local() { - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/files.yml") - if [[ ${MOUNT_SELECTED_LOCAL_SOURCES} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local.yml") - fi - if [[ ${MOUNT_ALL_LOCAL_SOURCES} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/local-all-sources.yml") - fi - - if [[ ${GITHUB_ACTIONS} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/ga.yml") - fi - - if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml") - fi - - if [[ -n ${USE_AIRFLOW_VERSION=} ]]; then - DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml") - fi - readonly DOCKER_COMPOSE_LOCAL -} - - -function testing::dump_container_logs() { - start_end::group_start "${COLOR_BLUE}Dumping container logs ${container}${COLOR_RESET}" - local container="${1}" - local dump_file - dump_file=${AIRFLOW_SOURCES}/files/container_logs_${container}_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}.log - echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}" - echo " Dumping logs from ${container} container" - echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}" - docker_v logs "${container}" > "${dump_file}" - echo " Container ${container} logs dumped to ${dump_file}" - echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}" - start_end::group_end -} - -function testing::setup_docker_compose_backend() { - local TEST_TYPE - TEST_TYPE="${1}" - if [[ ${BACKEND} == "mssql" ]]; then - local backend_docker_compose=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml") - local docker_filesystem - docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker 2>/dev/null || echo "unknown") - if [[ ${docker_filesystem} == "tmpfs" ]]; then - # In case of tmpfs backend for docker, mssql fails because TMPFS does not support - # O_DIRECT parameter for direct writing to the filesystem - # https://github.com/microsoft/mssql-docker/issues/13 - # so we need to mount an external volume for its db location - # the external db must allow for parallel testing so TEST_TYPE - # is added to the volume name - export MSSQL_DATA_VOLUME="${HOME}/tmp-mssql-volume-${TEST_TYPE/\[*\]/}-${MSSQL_VERSION}" - mkdir -p "${MSSQL_DATA_VOLUME}" - # MSSQL 2019 runs with non-root user by default so we have to make the volumes world-writeable - # This is a bit scary and we could get by making it group-writeable but the group would have - # to be set to "root" (GID=0) for the volume to work and this cannot be accomplished without sudo - chmod a+rwx "${MSSQL_DATA_VOLUME}" - backend_docker_compose+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-tmpfs-volume.yml") - - # Runner user doesn't have blanket sudo access, but we can run docker as root. Go figure - traps::add_trap "docker run -u 0 --rm -v ${MSSQL_DATA_VOLUME}:/mssql alpine sh -c 'rm -rvf -- /mssql/.* /mssql/*' || true" EXIT - - # Clean up at start too, in case a previous runner left it messy - docker run --rm -u 0 -v "${MSSQL_DATA_VOLUME}":/mssql alpine sh -c 'rm -rfv -- /mssql/.* /mssql/*' || true - export BACKEND_DOCKER_COMPOSE=("${backend_docker_compose[@]}") - else - backend_docker_compose+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-docker-volume.yml") - export BACKEND_DOCKER_COMPOSE=("${backend_docker_compose[@]}") - fi - else - local backend_docker_compose=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml") - export BACKEND_DOCKER_COMPOSE=("${backend_docker_compose[@]}") - fi -} - -function testing::run_command_in_docker(){ - set +u - set +e - local exit_code - local docker_test_name - local docker_test_name="${1}" - local docker_cmd - local docker_cmd="${2}" - echo - echo "Semaphore grabbed. Running ${docker_test_name} tests for ${BACKEND}" - echo - echo "Making sure docker-compose is down and remnants removed" - echo - docker-compose -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ - --project-name "airflow-${docker_test_name}-${BACKEND}" \ - down --remove-orphans \ - --volumes --timeout 10 - docker-compose --log-level INFO \ - -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ - "${BACKEND_DOCKER_COMPOSE[@]}" \ - "${DOCKER_COMPOSE_LOCAL[@]}" \ - --project-name "airflow-${docker_test_name}-${BACKEND}" \ - run airflow -c "${docker_cmd}" - exit_code=$? - docker ps - if [[ ${exit_code} != "0" && ${CI} == "true" ]]; then - docker ps --all - local container - for container in $(docker ps --all --format '{{.Names}}') - do - testing::dump_container_logs "${container}" - done - fi - - docker-compose --log-level INFO -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \ - --project-name "airflow-${docker_test_name}-${BACKEND}" \ - down --remove-orphans \ - --volumes --timeout 10 - set -u - set -e - if [[ ${exit_code} == 0 ]]; then - echo - echo "${COLOR_GREEN}Test ${docker_test_name} succeeded.${COLOR_RESET}" - else - echo - echo "${COLOR_RED}Test ${docker_test_name} failed.${COLOR_RESET}" - fi - return "${exit_code}" -} diff --git a/scripts/ci/libraries/_traps.sh b/scripts/ci/libraries/_traps.sh deleted file mode 100644 index 491f254eb3bfd..0000000000000 --- a/scripts/ci/libraries/_traps.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -####################################################################################################### -# -# Adds trap to the traps already set. -# -# Arguments: -# trap to set -# .... list of signals to handle -####################################################################################################### -function traps::add_trap() { - trap="${1}" - shift - for signal in "${@}" - do - # adding trap to exiting trap - local handlers - # shellcheck disable=SC2046 - handlers="$(_parse_current_traps "${signal}")" - # shellcheck disable=SC2064 - trap "${trap};${handlers}" "${signal}" - done -} - -function _parse_current_traps() { - local signal="$1" - # Yes, eval is evil, but this is the only way I was able to "parse" the output of trap which is "already" quoted - eval "set -- $(trap -p "$signal")" - # Output format is `trap -- 'command' ` - echo "${3-}" -} diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh deleted file mode 100644 index cd225a0b2196e..0000000000000 --- a/scripts/ci/libraries/_verbosity.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# In case "VERBOSE_COMMANDS" is set to "true" set -x is used to enable debugging - -DOCKER_BINARY_PATH="${DOCKER_BINARY_PATH:=$(command -v docker || echo "/bin/docker")}" -export DOCKER_BINARY_PATH - -function verbosity::store_exit_on_error_status() { - exit_on_error="false" - # If 'set -e' is set before entering the function, remember it, so you can restore before return! - if [[ $- == *e* ]]; then - exit_on_error="true" - fi - set +e -} - -function verbosity::restore_exit_on_error_status() { - if [[ ${exit_on_error} == "true" ]]; then - set -e - fi - unset exit_on_error -} - -# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be -# printed before execution. In case of DRY_RUN_DOCKER flag set to "true" -# show the command to execute instead of executing them -function docker_v { - if [[ ${DRY_RUN_DOCKER=} != "false" ]]; then - echo - echo "${COLOR_CYAN}docker" "${@}" "${COLOR_RESET}" - echo - return - fi - verbosity::store_exit_on_error_status - if [[ ${VERBOSE:="false"} == "true" && \ - # do not print echo if VERBOSE_COMMAND is set (set -x does it already) - ${VERBOSE_COMMANDS:=} != "true" && \ - # And when generally printing info is disabled - ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then - >&2 echo "${COLOR_CYAN}docker ${*} ${COLOR_RESET}" - fi - if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then - ${DOCKER_BINARY_PATH} "${@}" >>"${OUTPUT_LOG}" 2>&1 - else - "${DOCKER_BINARY_PATH}" "${@}" - fi - res="$?" - if [[ ${res} == "0" || ${exit_on_error} == "false" ]]; then - # The log is removed on success or when exit_on_error is set to false - # This way in the output log we only see the most recent failed command and what was echoed before - # But if we expect that the command might fail, we also will not print it's output - rm -f "${OUTPUT_LOG}" - fi - verbosity::restore_exit_on_error_status - return ${res} -} - -# Prints verbose information in case VERBOSE variable is set -function verbosity::print_info() { - if [[ ${VERBOSE:="false"} == "true" && ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then - echo "$@" - fi -} - -function verbosity::set_verbosity() { - # whether verbose output should be produced - export VERBOSE=${VERBOSE:="false"} - - # whether every bash statement should be printed as they are executed - export VERBOSE_COMMANDS=${VERBOSE_COMMANDS:="false"} - - # whether the output from script should be printed at all - export PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS:="true"} -} - -verbosity::set_verbosity diff --git a/scripts/ci/openapi/client_codegen_diff.sh b/scripts/ci/openapi/client_codegen_diff.sh index c84adf43859cb..7e2f40184708a 100755 --- a/scripts/ci/openapi/client_codegen_diff.sh +++ b/scripts/ci/openapi/client_codegen_diff.sh @@ -15,9 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh" - set -eu # HEAD^1 says the "first" parent. For PR merge commits, or main commits, this is the "right" commit. diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index dca87457bf072..9ffd881e82240 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -50,8 +50,6 @@ "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", - "PRINT_INFO_FROM_SCRIPTS=false", - "-e", "BACKEND=sqlite", "--pull", "never", diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py index c786b422708e9..0451cba112791 100755 --- a/scripts/ci/pre_commit/pre_commit_migration_reference.py +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -49,8 +49,6 @@ *get_extra_docker_flags(MOUNT_SELECTED), "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", - "-e", - "PRINT_INFO_FROM_SCRIPTS=false", "--pull", "never", airflow_image, diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 0bf220c47a5f2..8ff73e888fb20 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -53,8 +53,6 @@ "-e", "SKIP_ENVIRONMENT_INITIALIZATION=true", "-e", - "PRINT_INFO_FROM_SCRIPTS=false", - "-e", "BACKEND=sqlite", "--pull", "never", diff --git a/scripts/ci/testing/run_downgrade_test.sh b/scripts/ci/testing/run_downgrade_test.sh deleted file mode 100755 index 06a3b405a6c76..0000000000000 --- a/scripts/ci/testing/run_downgrade_test.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -testing::get_docker_compose_local -testing::setup_docker_compose_backend "downgrade" -# This runs downgrades for DBs created from the migration file -testing::run_command_in_docker "downgrade" "airflow db reset --skip-init -y \ - && airflow db upgrade --to-revision heads && airflow db downgrade -r e959f08ac86c -y \ - && airflow db upgrade" -# This tests upgrade/downgrade for DBs created from the ORM -testing::run_command_in_docker "downgrade" "airflow db reset -y \ - && airflow db upgrade && airflow db downgrade -r e959f08ac86c -y \ - && airflow db upgrade" diff --git a/scripts/ci/testing/run_offline_sql_test.sh b/scripts/ci/testing/run_offline_sql_test.sh deleted file mode 100755 index 9245f7134ee9b..0000000000000 --- a/scripts/ci/testing/run_offline_sql_test.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -testing::get_docker_compose_local -testing::setup_docker_compose_backend "offline-sql-test" -# We test from 2.0.0 upwards -testing::run_command_in_docker "offline-sql-test" "airflow db upgrade --from-version 2.0.0 -r heads --show-sql-only \ - && airflow db downgrade --to-version 2.0.0 --show-sql-only -y"