Add verification steps when releasing the images. (apache#24520)

After the images are pushed in CI we are running the verification
of the AMD image now.

This cannot be really done during building and pushing the image,
because we are using multi-platform images using remote builders
so the image is not even available locally, so we need to actually
pull the images after they are built in order to verify them.

This PR adds those features:
* ability to pull images for verification with --pull-flag
* ability to verify slim images (regular tests are skipped and
  we only expect the preinstalled providers to be available
* the steps to verify the images (both regular and slim) are
  added to the workflow

Author: potiuk

.github/workflows/release_dockerhub_image.yml

@@ -119,12 +119,27 @@ jobs:
           ${{ needs.build-info.outputs.skipLatest }}
           ${{ needs.build-info.outputs.limitPlatform }}
           --limit-python ${{ matrix.python-version }} --slim-images
-      - name: "Docker logout"
-        run: docker logout
-        if: always()
       - name: "Stop ARM instance"
         run: ./scripts/ci/images/ci_stop_arm_instance.sh
         if: always() && github.repository == 'apache/airflow'
+      - name: >
+          Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
+        run: >
+          breeze verify-prod-image
+          --pull-image
+          --image-name
+          ${{github.repository}}:${{github.event.inputs.airflowVersion}}-python${{matrix.python-version}}
+      - name: >
+          Verify slim AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
+        run: >
+          breeze verify-prod-image
+          --pull-image
+          --slim-image
+          --image-name
+          ${{github.repository}}:slim-${{github.event.inputs.airflowVersion}}-python${{matrix.python-version}}
+      - name: "Docker logout"
+        run: docker logout
+        if: always()
       - name: "Fix ownership"
         run: breeze fix-ownership
         if: always()

dev/breeze/src/airflow_breeze/commands/ci_image_commands.py

@@ -54,6 +54,7 @@
     option_parallelism,
     option_platform,
     option_prepare_buildx_cache,
+    option_pull_image,
     option_push_image,
     option_python,
     option_python_versions,
@@ -181,6 +182,7 @@
                 "--image-name",
                 "--python",
                 "--image-tag",
+                "--pull-image",
             ],
         }
     ],
@@ -270,7 +272,7 @@ def run_build(ci_image_params: BuildCiParams) -> None:
 @option_image_tag
 @option_tag_as_latest
 @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def pull_image(
+def pull_ci_image(
     verbose: bool,
     dry_run: bool,
     python: str,
@@ -338,27 +340,33 @@ def pull_image(
 @option_github_repository
 @option_image_tag
 @option_image_name
+@option_pull_image
 @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def verify_image(
+def verify_ci_image(
     verbose: bool,
     dry_run: bool,
     python: str,
     github_repository: str,
     image_name: str,
     image_tag: str,
+    pull_image: bool,
     extra_pytest_args: Tuple,
 ):
     """Verify CI image."""
     perform_environment_checks(verbose=verbose)
     if image_name is None:
         build_params = BuildCiParams(python=python, image_tag=image_tag, github_repository=github_repository)
         image_name = build_params.airflow_image_name_with_tag
+    if pull_image:
+        command_to_run = ["docker", "pull", image_name]
+        run_command(command_to_run, verbose=verbose, dry_run=dry_run, check=True)
     get_console().print(f"[info]Verifying CI image: {image_name}[/]")
     return_code, info = verify_an_image(
         image_name=image_name,
         verbose=verbose,
         dry_run=dry_run,
         image_type='CI',
+        slim_image=False,
         extra_pytest_args=extra_pytest_args,
     )
     sys.exit(return_code)

dev/breeze/src/airflow_breeze/commands/production_image_commands.py

@@ -52,6 +52,7 @@
     option_parallelism,
     option_platform,
     option_prepare_buildx_cache,
+    option_pull_image,
     option_push_image,
     option_python,
     option_python_versions,
@@ -184,6 +185,7 @@
                 "--image-name",
                 "--python",
                 "--image-tag",
+                "--pull-image",
             ],
         }
     ],
@@ -382,6 +384,12 @@ def pull_prod_image(
 @option_github_repository
 @option_image_tag
 @option_image_name
+@option_pull_image
+@click.option(
+    '--slim-image',
+    help='The image to verify is slim and non-slim tests should be skipped.',
+    is_flag=True,
+)
 @click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
 def verify_prod_image(
     verbose: bool,
@@ -390,6 +398,8 @@ def verify_prod_image(
     github_repository: str,
     image_name: str,
     image_tag: str,
+    pull_image: bool,
+    slim_image: bool,
     extra_pytest_args: Tuple,
 ):
     """Verify Production image."""
@@ -399,13 +409,17 @@ def verify_prod_image(
             python=python, image_tag=image_tag, github_repository=github_repository
         )
         image_name = build_params.airflow_image_name_with_tag
+    if pull_image:
+        command_to_run = ["docker", "pull", image_name]
+        run_command(command_to_run, verbose=verbose, dry_run=dry_run, check=True)
     get_console().print(f"[info]Verifying PROD image: {image_name}[/]")
     return_code, info = verify_an_image(
         image_name=image_name,
         verbose=verbose,
         dry_run=dry_run,
         image_type='PROD',
         extra_pytest_args=extra_pytest_args,
+        slim_image=slim_image,
     )
     sys.exit(return_code)
 

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -434,3 +434,9 @@
     show_default=True,
     help='Mode of constraints for PROD image building',
 )
+option_pull_image = click.option(
+    '--pull-image',
+    help="Pull image is missing before attempting to verify it.",
+    is_flag=True,
+    envvar='PULL_IMAGE',
+)

dev/breeze/src/airflow_breeze/utils/image.py

@@ -201,6 +201,7 @@ def run_pull_and_verify_image(
         image_type=image_params.image_type,
         dry_run=dry_run,
         verbose=verbose,
+        slim_image=False,
         extra_pytest_args=extra_pytest_args,
     )
 

dev/breeze/src/airflow_breeze/utils/run_tests.py

@@ -26,7 +26,7 @@
 
 
 def verify_an_image(
-    image_name: str, image_type: str, dry_run: bool, verbose: bool, extra_pytest_args: Tuple
+    image_name: str, image_type: str, dry_run: bool, verbose: bool, slim_image: bool, extra_pytest_args: Tuple
 ) -> Tuple[int, str]:
     command_result = run_command(
         ["docker", "inspect", image_name], dry_run=dry_run, verbose=verbose, check=False, stdout=DEVNULL
@@ -43,6 +43,8 @@ def verify_an_image(
         test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" / "test_ci_image.py"
     env = os.environ.copy()
     env['DOCKER_IMAGE'] = image_name
+    if slim_image:
+        env['TEST_SLIM_IMAGE'] = 'true'
     command_result = run_command(
         [sys.executable, "-m", "pytest", str(test_path), *pytest_args, *extra_pytest_args],
         dry_run=dry_run,

docker_tests/test_prod_image.py

@@ -16,6 +16,7 @@
 # under the License.
 
 import json
+import os
 import subprocess
 import tempfile
 from pathlib import Path
@@ -30,6 +31,7 @@
     run_bash_in_docker,
     run_python_in_docker,
 )
+from setup import PREINSTALLED_PROVIDERS
 
 INSTALLED_PROVIDER_PATH = SOURCE_ROOT / "scripts" / "ci" / "installed_providers.txt"
 
@@ -74,8 +76,11 @@ def test_bash_version(self):
 
 class TestPythonPackages:
     def test_required_providers_are_installed(self):
-        lines = (d.strip() for d in INSTALLED_PROVIDER_PATH.read_text().splitlines())
-        lines = (d for d in lines)
+        if os.environ.get("TEST_SLIM_IMAGE"):
+            lines = PREINSTALLED_PROVIDERS
+        else:
+            lines = (d.strip() for d in INSTALLED_PROVIDER_PATH.read_text().splitlines())
+            lines = (d for d in lines)
         packages_to_install = {f"apache-airflow-providers-{d.replace('.', '-')}" for d in lines}
         assert len(packages_to_install) != 0
 
@@ -163,6 +168,7 @@ def test_pip_dependencies_conflict(self):
         "virtualenv": ["virtualenv"],
     }
 
+    @pytest.mark.skipif(os.environ.get("TEST_SLIM_IMAGE") == "true", reason="Skipped with slim image")
     @pytest.mark.parametrize("package_name,import_names", PACKAGE_IMPORTS.items())
     def test_check_dependencies_imports(self, package_name, import_names):
         run_python_in_docker(f"import {','.join(import_names)}")

images/breeze/output-commands-hash.txt

@@ -0,0 +1 @@
+8b4116c1808c84d491961283a4ddbec2