From bc763b5dec1ae477704e7104ff8a6f4b89cfef8b Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Wed, 25 Oct 2023 22:28:29 +0200 Subject: [PATCH 1/3] Fix K8S executor override config using pod_override_object --- .../providers/cncf/kubernetes/pod_generator.py | 4 ++-- .../cncf/kubernetes/test_pod_generator.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/pod_generator.py b/airflow/providers/cncf/kubernetes/pod_generator.py index 581aa61d16e22..69ca6201c4514 100644 --- a/airflow/providers/cncf/kubernetes/pod_generator.py +++ b/airflow/providers/cncf/kubernetes/pod_generator.py @@ -434,8 +434,8 @@ def construct_pod( ) # Reconcile the pods starting with the first chronologically, - # Pod from the pod_template_File -> Pod from executor_config arg -> Pod from the K8s executor - pod_list = [base_worker_pod, pod_override_object, dynamic_pod] + # Pod from the pod_template_File -> Pod from the K8s executor -> Pod from executor_config arg + pod_list = [base_worker_pod, dynamic_pod, pod_override_object] try: pod = reduce(PodGenerator.reconcile_pods, pod_list) diff --git a/tests/providers/cncf/kubernetes/test_pod_generator.py b/tests/providers/cncf/kubernetes/test_pod_generator.py index 2739781858f69..969bf26bc5e5a 100644 --- a/tests/providers/cncf/kubernetes/test_pod_generator.py +++ b/tests/providers/cncf/kubernetes/test_pod_generator.py @@ -401,17 +401,29 @@ def test_reconcile_pods(self, mock_rand_str): pytest.param(None, "busybox", id="no_image_in_cfg"), ], ) - def test_construct_pod(self, config_image, expected_image): + @pytest.mark.parametrize( + "pod_override_object_namespace, expected_namespace", + [ + ("new_namespace", "new_namespace"), # pod_override_object namespace should be used + (None, "test_namespace"), # if it is not provided, we use default one + ], + ) + def test_construct_pod( + self, config_image, expected_image, pod_override_object_namespace, expected_namespace + ): template_file = sys.path[0] + "/tests/providers/cncf/kubernetes/pod_generator_base_with_secrets.yaml" worker_config = PodGenerator.deserialize_model_file(template_file) executor_config = k8s.V1Pod( + metadata=k8s.V1ObjectMeta( + namespace=pod_override_object_namespace, + ), spec=k8s.V1PodSpec( containers=[ k8s.V1Container( name="", resources=k8s.V1ResourceRequirements(limits={"cpu": "1m", "memory": "1G"}) ) ] - ) + ), ) result = PodGenerator.construct_pod( @@ -432,7 +444,7 @@ def test_construct_pod(self, config_image, expected_image): expected.metadata.labels["app"] = "myapp" expected.metadata.annotations = self.annotations expected.metadata.name = "pod_id" - expected.metadata.namespace = "test_namespace" + expected.metadata.namespace = expected_namespace expected.spec.containers[0].args = ["command"] expected.spec.containers[0].image = expected_image expected.spec.containers[0].resources = {"limits": {"cpu": "1m", "memory": "1G"}} From 689306e6fe722a2a788902820e3bc3aa53422f38 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 26 Oct 2023 01:42:52 +0200 Subject: [PATCH 2/3] Activate multi namespace mode for K8S tests --- .github/workflows/ci.yml | 2 +- .../commands/kubernetes_commands.py | 23 ++++++ .../commands/kubernetes_commands_config.py | 2 + images/breeze/output-commands-hash.txt | 6 +- images/breeze/output_k8s_deploy-airflow.svg | 50 ++++++------ .../breeze/output_k8s_run-complete-tests.svg | 80 ++++++++++--------- 6 files changed, 98 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3dd00e20944b9..f9e87571c6618 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1677,7 +1677,7 @@ jobs: ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\ 'setup.py','pyproject.toml','generated/provider_dependencies.json') }}" - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} - run: breeze k8s run-complete-tests --run-in-parallel --upgrade + run: breeze k8s run-complete-tests --run-in-parallel --multi-namespace-mode --upgrade env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py index c46d2262adcf6..559d248135759 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py @@ -167,6 +167,13 @@ def kubernetes_group(): envvar="USE_STANDARD_NAMING", ) +option_multi_namespace_mode = click.option( + "--multi-namespace-mode", + help="Use multi namespace mode.", + is_flag=True, + envvar="MULTI_NAMESPACE_MODE", +) + option_rebuild_base_image = click.option( "--rebuild-base-image", help="Rebuilds base Airflow image before building K8S image.", @@ -954,6 +961,7 @@ def _deploy_helm_chart( executor: str, use_standard_naming: bool, extra_options: tuple[str, ...] | None = None, + multi_namespace_mode: bool = False, ) -> RunCommandResult: cluster_name = get_kubectl_cluster_name(python=python, kubernetes_version=kubernetes_version) action = "Deploying" if not upgrade else "Upgrading" @@ -994,6 +1002,8 @@ def _deploy_helm_chart( "config.logging.logging_level=DEBUG", "--set", f"executor={executor}", + "--set", + f"multiNamespaceMode={'true' if multi_namespace_mode else 'false'}", ] if upgrade: # force upgrade @@ -1024,6 +1034,7 @@ def _deploy_airflow( wait_time_in_seconds: int, use_standard_naming: bool, extra_options: tuple[str, ...] | None = None, + multi_namespace_mode: bool = False, ) -> tuple[int, str]: action = "Deploying" if not upgrade else "Upgrading" cluster_name = get_kind_cluster_name(python=python, kubernetes_version=kubernetes_version) @@ -1036,6 +1047,7 @@ def _deploy_airflow( executor=executor, use_standard_naming=use_standard_naming, extra_options=extra_options, + multi_namespace_mode=multi_namespace_mode, ) if result.returncode == 0: get_console(output=output).print( @@ -1073,6 +1085,7 @@ def _deploy_airflow( @option_debug_resources @option_include_success_outputs @option_use_standard_naming +@option_multi_namespace_mode @option_python_versions @option_kubernetes_versions @option_verbose @@ -1093,6 +1106,7 @@ def deploy_airflow( python_versions: str, kubernetes_versions: str, extra_options: tuple[str, ...], + multi_namespace_mode: bool = False, ): if run_in_parallel: python_version_array: list[str] = python_versions.split(" ") @@ -1121,6 +1135,7 @@ def deploy_airflow( "wait_time_in_seconds": wait_time_in_seconds, "extra_options": extra_options, "output": outputs[index], + "multi_namespace_mode": multi_namespace_mode, }, ) for index, combo in enumerate(combos) @@ -1142,6 +1157,7 @@ def deploy_airflow( use_standard_naming=use_standard_naming, wait_time_in_seconds=wait_time_in_seconds, extra_options=extra_options, + multi_namespace_mode=multi_namespace_mode, ) if return_code == 0: get_console().print( @@ -1430,6 +1446,7 @@ def _run_complete_tests( extra_options: tuple[str, ...] | None, test_args: tuple[str, ...], output: Output | None, + multi_namespace_mode: bool = False, ) -> tuple[int, str]: get_console(output=output).print(f"\n[info]Rebuilding k8s image for Python {python}\n") returncode, message = _rebuild_k8s_image( @@ -1486,6 +1503,7 @@ def _run_complete_tests( use_standard_naming=use_standard_naming, wait_time_in_seconds=wait_time_in_seconds, extra_options=extra_options, + multi_namespace_mode=multi_namespace_mode, ) if returncode != 0: _logs(python=python, kubernetes_version=kubernetes_version) @@ -1516,6 +1534,7 @@ def _run_complete_tests( use_standard_naming=use_standard_naming, wait_time_in_seconds=wait_time_in_seconds, extra_options=extra_options, + multi_namespace_mode=multi_namespace_mode, ) if returncode != 0: _logs(python=python, kubernetes_version=kubernetes_version) @@ -1562,6 +1581,7 @@ def _run_complete_tests( @option_debug_resources @option_include_success_outputs @option_use_standard_naming +@option_multi_namespace_mode @option_python_versions @option_kubernetes_versions @option_verbose @@ -1585,6 +1605,7 @@ def run_complete_tests( use_standard_naming: bool, python_versions: str, kubernetes_versions: str, + multi_namespace_mode: bool, test_args: tuple[str, ...], ): result = create_virtualenv(force_venv_setup=force_venv_setup) @@ -1623,6 +1644,7 @@ def run_complete_tests( "extra_options": None, "test_args": pytest_args, "output": outputs[index], + "multi_namespace_mode": multi_namespace_mode, }, ) for index, combo in enumerate(combos) @@ -1649,6 +1671,7 @@ def run_complete_tests( extra_options=None, test_args=test_args, output=None, + multi_namespace_mode=multi_namespace_mode, ) if result != 0: sys.exit(result) diff --git a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py index 5049100c59863..ae0a20b902b04 100644 --- a/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/kubernetes_commands_config.py @@ -160,6 +160,7 @@ "--upgrade", "--wait-time-in-seconds", "--use-standard-naming", + "--multi-namespace-mode", ], }, { @@ -224,6 +225,7 @@ "--upgrade", "--wait-time-in-seconds", "--use-standard-naming", + "--multi-namespace-mode", ], }, { diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 283ca94693f22..563c6ac8ce924 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -22,16 +22,16 @@ k8s:build-k8s-image:b625255c3e8f3f794ee404f9a4476836 k8s:configure-cluster:727d3d25b67fc02d8f9b45fd30981911 k8s:create-cluster:9428fa86c41f868db80bda7c232cead1 k8s:delete-cluster:12aed4c701000ae5ebe264efcdcd1e5a -k8s:deploy-airflow:a24b39835acf66263e561ce5eaee332c +k8s:deploy-airflow:c17477e1e08326ef5ff1bd15f4047efe k8s:k9s:26504ebfc24f3599bbda4e4110c80fd0 k8s:logs:a1eb1d8046bbc542fd210c00ceb23ee9 -k8s:run-complete-tests:6ad037e6986aa7ae136028f4a7640e91 +k8s:run-complete-tests:1f5007f36cc4c57696ea59d35f31f687 k8s:setup-env:a34e94744ca4e0592371fe55478c3d54 k8s:shell:cb7b8f22ce35f94cce5b0ca0c64bfbb3 k8s:status:0a472b05c23aa70098241414ac86687f k8s:tests:b981dbbfbfe531e358ebb2aaa0739525 k8s:upload-k8s-image:8ae1ad3745e2f34a142ecff8a714af1d -k8s:e23599728fd67b64e756e6dbe7f256f0 +k8s:ea758d2967ba71d158745e80bf6e407e prod-image:build:20f84ddadc2fe4ae2723b7ccdde0197f prod-image:pull:3817ef211b023b76df84ee1110ef64dd prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 diff --git a/images/breeze/output_k8s_deploy-airflow.svg b/images/breeze/output_k8s_deploy-airflow.svg index 94401d0bdf974..a3aba34e7e093 100644 --- a/images/breeze/output_k8s_deploy-airflow.svg +++ b/images/breeze/output_k8s_deploy-airflow.svg @@ -1,4 +1,4 @@ - +