diff --git a/src/containerapp/HISTORY.rst b/src/containerapp/HISTORY.rst index e5c5cfcbd4b..a322117c4ed 100644 --- a/src/containerapp/HISTORY.rst +++ b/src/containerapp/HISTORY.rst @@ -16,6 +16,7 @@ upcoming * 'az containerapp env http-route-config': Add commands for the http-route-config feature area. * 'az containerapp env java-component': Support more flexible configuration updates with new parameters `--set-configurations`, `--replace-configurations`, `--remove-configurations` and `--remove-all-configurations`. * 'az containerapp env java-component gateway-for-spring create/update': Support `--bind` and `--unbind` +* 'az containerapp arc': Enable setup custom core dns for Aks AzureCore on Arc. 1.1.0b1 ++++++ diff --git a/src/containerapp/azext_containerapp/_arc_utils.py b/src/containerapp/azext_containerapp/_arc_utils.py new file mode 100644 index 00000000000..f8f14601d3d --- /dev/null +++ b/src/containerapp/azext_containerapp/_arc_utils.py @@ -0,0 +1,358 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +# pylint: disable=line-too-long, consider-using-f-string, no-else-return, duplicate-string-formatting-argument, expression-not-assigned, too-many-locals, logging-fstring-interpolation, broad-except, pointless-statement, bare-except, unused-variable, redefined-outer-name, reimported, unused-import, consider-using-generator, broad-exception-raised + +import json +import os +import shutil + +from http import HTTPStatus +from knack.log import get_logger + +from kubernetes import client, config, utils + +from azure.cli.core.azclierror import (ValidationError, ResourceNotFoundError, CLIError, InvalidArgumentValueError) +from ._constants import (CUSTOM_CORE_DNS_VOLUME_NAME, CUSTOM_CORE_DNS_VOLUME_MOUNT_PATH, + CUSTOM_CORE_DNS, CORE_DNS, KUBE_SYSTEM, EMPTY_CUSTOM_CORE_DNS) + +logger = get_logger(__name__) + + +def create_kube_client(kube_config, kube_context, skip_ssl_verification=False): + default_config = get_kube_config(kube_config, kube_context, skip_ssl_verification) + return client.ApiClient(default_config) + + +def get_kube_config(kube_config, kube_context, skip_ssl_verification=False): + try: + config.load_kube_config(config_file=kube_config, context=kube_context) + default_config = client.Configuration.get_default_copy() + if skip_ssl_verification: + default_config.verify_ssl = False + + return default_config + + except config.config_exception.ConfigException as ce: + raise ValidationError("Problem loading the kubeconfig file. " + "You should either save the kube configuration in the default kubernetes config location, " + "Or you should specify the kube config and context in parameter. Error: " + str(ce)) + except Exception as e: + raise ValidationError("Problem loading the kubeconfig file." + str(e)) + + +def check_kube_connection(kube_config, kube_context, skip_ssl_verification=False): + logger.debug("Checking Connectivity to Cluster") + get_kube_config(kube_config, kube_context, skip_ssl_verification) + api_instance = client.VersionApi() + try: + api_response = api_instance.get_code() + logger.debug(f"Api Server Version: {api_response.git_version}") + return api_response.git_version + except Exception as e: # pylint: disable=broad-except + logger.warning("Unable to verify connectivity to the Kubernetes cluster.") + raise ValidationError(f"Unable to verify connectivity to the Kubernetes cluster. {e}") + + +def create_folder(folder_name, time_stamp): + error = "" + # Fetching path to user directory to create the arc diagnostic folder + home_dir = os.path.expanduser("~") + filepath = os.path.join(home_dir, ".azure", folder_name) + filepath_with_timestamp = os.path.join(filepath, time_stamp) + try: + + # Creating Diagnostic folder and its subfolder with the given timestamp and cluster name to store all the logs + try: + os.mkdir(filepath) + except FileExistsError: + pass + try: + os.mkdir(filepath_with_timestamp) + except FileExistsError: + # Deleting the folder if present with the same timestamp to prevent overriding in the same folder and then + # creating it again + shutil.rmtree(filepath_with_timestamp, ignore_errors=True) + os.mkdir(filepath_with_timestamp) + + return filepath_with_timestamp, True, "" + + # For handling storage or OS exception that may occur during the execution + except OSError as e: + if "[Errno 28]" in str(e): + shutil.rmtree(filepath_with_timestamp, ignore_errors=False, onexc=None) + error = "No space left on device" + else: + error = f"Error while trying to create diagnostic logs folder. Exception: {str(e)}" + + # To handle any exception that may occur during the execution + except Exception as e: + error = f"Error while trying to create diagnostic logs folder. Exception: {str(e)}" + + return "", False, error + + +def create_sub_folder(parent_path, subfolder_name): + if parent_path is None: + return "", False, "parent_path is required." + + if subfolder_name is None: + return "", False, "subfolder_name is required." + + error = "" + filepath = os.path.join(parent_path, subfolder_name) + try: + os.mkdir(filepath) + return filepath, True, "" + except FileExistsError: + return filepath, True, "" + # For handling storage or OS exception that may occur during the execution + except OSError as e: + if "[Errno 28]" in str(e): + shutil.rmtree(filepath, ignore_errors=False, onexc=None) + error = "No space left on device" + else: + error = f"Error while trying to create diagnostic logs folder. Exception: {str(e)}" + + # To handle any exception that may occur during the execution + except Exception as e: + error = f"Error while trying to create diagnostic logs folder. Exception: {str(e)}" + + return "", False, error + + +def patch_coredns(kube_client, coredns_configmap, coredns_deployment, new_filepath_with_timestamp, + custom_core_dns_configmap_exists, create_volume, create_volume_mount): + import re + + if not create_volume and not create_volume_mount: + return + + # create custom core dns config map + if not custom_core_dns_configmap_exists: + logger.info("coredns-custom configmap doesn't exist in namespace kube-system, create a new one") + filepath = os.path.join(new_filepath_with_timestamp, "coredns-custom.yaml") + with open(filepath, "w") as f: # Opens file and casts as f + f.write(EMPTY_CUSTOM_CORE_DNS) + utils.create_from_yaml(kube_client, filepath, verbose=True) + + # update core dns config map. + core_file_data = coredns_configmap.data.get('Corefile') + lines = core_file_data.split("\n") + has_import_custom_server = False + for line in lines: + if re.match(r'^\S*import custom/\*\.server$', line): + has_import_custom_server = True + break + if not has_import_custom_server: + core_file_data = rreplace(core_file_data, "\n", "\nimport custom/*.server\n", 1) + coredns_configmap.data['Corefile'] = core_file_data + update_configmap(CORE_DNS, KUBE_SYSTEM, kube_client, coredns_configmap) + + # update deployment + if create_volume: + logger.info(f"create volume {CUSTOM_CORE_DNS_VOLUME_NAME}") + custom_coredns_volume = client.V1Volume( + name=CUSTOM_CORE_DNS_VOLUME_NAME, + config_map=client.V1ConfigMapVolumeSource( + default_mode=420, + name=CUSTOM_CORE_DNS, + optional=True + ) + ) + coredns_deployment.spec.template.spec.volumes.append(custom_coredns_volume) + + if create_volume_mount: + logger.info(f"create volume mount {CUSTOM_CORE_DNS_VOLUME_NAME}") + custom_coredns_volume_mount = client.V1VolumeMount( + mount_path=CUSTOM_CORE_DNS_VOLUME_MOUNT_PATH, + name=CUSTOM_CORE_DNS_VOLUME_NAME, + read_only=True + ) + coredns_deployment.spec.template.spec.containers[0].volume_mounts.append(custom_coredns_volume_mount) + + deployment = client.V1Deployment( + spec=client.V1DeploymentSpec( + selector=coredns_deployment.spec.selector, + template=client.V1PodTemplateSpec( + spec=client.V1PodSpec( + volumes=coredns_deployment.spec.template.spec.volumes, + containers=[client.V1Container( + name=coredns_deployment.spec.template.spec.containers[0].name, + volume_mounts=coredns_deployment.spec.template.spec.containers[0].volume_mounts + )] + ) + ) + ) + ) + update_deployment(CORE_DNS, KUBE_SYSTEM, kube_client, deployment) + + +def rreplace(s, old, new, occurrence): + li = s.rsplit(old, occurrence) + return new.join(li) + + +def get_core_dns_deployment(kube_client, folder=None): + response = get_and_save_deployment(CORE_DNS, KUBE_SYSTEM, kube_client, folder) + if response is None: + raise ValidationError("CoreDns deployment cannot be found in kube-system namespace") + + return response + + +def get_core_dns_configmap(kube_client, folder=None): + response = get_and_save_configmap(CORE_DNS, KUBE_SYSTEM, kube_client, folder) + if response is None: + raise ValidationError("CoreDns Configmap cannot be found in kube-system namespace") + + return response + + +def backup_custom_core_dns_configmap(kube_client, folder=None): + return get_and_save_configmap(CUSTOM_CORE_DNS, KUBE_SYSTEM, kube_client, folder) + + +def get_and_save_configmap(resource_name, resource_namespace, kube_client, folder=None): + configmap = get_configmap(resource_name, resource_namespace, kube_client) + if configmap is not None: + if folder is not None: + filepath = os.path.join(folder, f"configmap-{resource_name}.json") + try: + logger.info(f"Save ConfigMap '{resource_name}' in namespace '{resource_namespace}' to {filepath} ") + with open(filepath, "w") as f: # Opens file and casts as f + configmap_dict = json.loads(configmap.data) + f.write(json.dumps(configmap_dict, indent=2)) + except Exception as e: + raise ValidationError(f"Failed to save file {filepath} with error '{str(e)}'") + return kube_client.deserialize(configmap, client.models.v1_config_map.V1ConfigMap) + + logger.info(f"ConfigMap '{resource_name}' in namespace '{resource_namespace}' does not exist") + return None + + +def get_and_save_deployment(resource_name, resource_namespace, kube_client, folder=None): + deployment = get_deployment(resource_name, resource_namespace, kube_client) + if deployment is not None: + if folder is not None: + filepath = os.path.join(folder, f"deployment-{resource_name}.json") + try: + logger.info(f"Save Deployment '{resource_name}' in namespace '{resource_namespace}' to {filepath} ") + with open(filepath, "w") as f: # Opens file and casts as f + deployment_dict = json.loads(deployment.data) + f.write(json.dumps(deployment_dict, indent=2)) + except Exception as e: + raise ValidationError(f"Failed to save file {filepath} with error '{str(e)}'") + + return kube_client.deserialize(deployment, client.models.v1_deployment.V1Deployment) + + logger.info(f"Deployment '{resource_name}' in namespace '{resource_namespace}' does not exist") + return None + + +def get_deployment(resource_name, resource_namespace, kube_client): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Get Deployment '{resource_name}' from namespace '{resource_namespace}'") + apps_v1_api = client.AppsV1Api(kube_client) + deployment = apps_v1_api.read_namespaced_deployment(name=resource_name, namespace=resource_namespace, pretty=True, _preload_content=False) + except client.exceptions.ApiException as e: + if e.status == 404: + deployment = None + else: + raise e + except Exception as e: + raise ValidationError(f"other errors while getting deployment coredns in kube-system {str(e)}") + + return deployment + + +def update_deployment(resource_name, resource_namespace, kube_client, deployment): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Start to patch deployment {resource_name} in namespace {resource_namespace}") + apps_v1_api = client.AppsV1Api(kube_client) + apps_v1_api.patch_namespaced_deployment(name=resource_name, namespace=resource_namespace, body=deployment) + except Exception as e: + raise ValidationError(f"other errors while patching deployment coredns in kube-system {str(e)}") + + +def replace_deployment(resource_name, resource_namespace, kube_client, deployment): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Start to replace deployment {resource_name} in namespace {resource_namespace}") + apps_v1_api = client.AppsV1Api(kube_client) + apps_v1_api.replace_namespaced_deployment(name=resource_name, namespace=resource_namespace, body=deployment) + except Exception as e: + raise ValidationError(f"other errors while replacing deployment coredns in kube-system {str(e)}") + + +def get_configmap(resource_name, resource_namespace, kube_client): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Get ConfigMap '{resource_name}' from namespace '{resource_namespace}'") + core_v1_api = client.api.core_v1_api.CoreV1Api(kube_client) + config_map = core_v1_api.read_namespaced_config_map(name=resource_name, namespace=resource_namespace, pretty=True, _preload_content=False) + except client.exceptions.ApiException as e: + if e.status == 404: + config_map = None + else: + raise e + except Exception as e: + raise ValidationError(f"other errors while getting config map coredns in kube-system {str(e)}") + + return config_map + + +def update_configmap(resource_name, resource_namespace, kube_client, config_map): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Start to update configmap {resource_name} in namespace {resource_namespace}") + core_v1_api = client.api.core_v1_api.CoreV1Api(kube_client) + core_v1_api.patch_namespaced_config_map(name=resource_name, namespace=resource_namespace, body=config_map) + + except Exception as e: + raise CLIError(f"other errors while patching config map coredns in kube-system {str(e)}") + + +def replace_configmap(resource_name, resource_namespace, kube_client, config_map): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Start to replace configmap {resource_name} in namespace {resource_namespace}") + core_v1_api = client.api.core_v1_api.CoreV1Api(kube_client) + core_v1_api.replace_namespaced_config_map(name=resource_name, namespace=resource_namespace, body=config_map) + + except Exception as e: + raise CLIError(f"other errors while replacing config map coredns in kube-system {str(e)}") + + +def delete_configmap(resource_name, resource_namespace, kube_client): + validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace) + + try: + logger.info(f"Start to delete configmap {resource_name} in namespace {resource_namespace}") + core_v1_api = client.api.core_v1_api.CoreV1Api(kube_client) + core_v1_api.delete_namespaced_config_map(name=resource_name, namespace=resource_namespace) + + except client.rest.ApiException as e: + if e.status == HTTPStatus.NOT_FOUND: + logger.info(f"Configmap {resource_name} in namespace {resource_namespace} has been deleted") + return + raise CLIError(f"other ApiException while deleting config map coredns in kube-system {str(e)}") + + except Exception as e: + raise CLIError(f"other errors while deleting config map coredns in kube-system {str(e)}") + + +def validate_resource_name_and_resource_namespace_not_empty(resource_name, resource_namespace): + if resource_name is None or len(resource_name) == 0: + raise InvalidArgumentValueError("Arg resource_name should not be None or Empty") + if resource_namespace is None or len(resource_namespace) == 0: + raise InvalidArgumentValueError("Arg resource_namespace should not be None or Empty") diff --git a/src/containerapp/azext_containerapp/_constants.py b/src/containerapp/azext_containerapp/_constants.py index 4dc65e7f323..0cd7ba655b1 100644 --- a/src/containerapp/azext_containerapp/_constants.py +++ b/src/containerapp/azext_containerapp/_constants.py @@ -140,3 +140,23 @@ RUNTIME_GENERIC = "generic" RUNTIME_JAVA = "java" SUPPORTED_RUNTIME_LIST = [RUNTIME_GENERIC, RUNTIME_JAVA] + +AKS_AZURE_LOCAL_DISTRO = "AksAzureLocal" +SETUP_CORE_DNS_SUPPORTED_DISTRO = [AKS_AZURE_LOCAL_DISTRO] +CUSTOM_CORE_DNS_VOLUME_NAME = 'custom-config-volume' +CUSTOM_CORE_DNS_VOLUME_MOUNT_PATH = '/etc/coredns/custom' +CUSTOM_CORE_DNS = 'coredns-custom' +CORE_DNS = 'coredns' +KUBE_SYSTEM = 'kube-system' +EMPTY_CUSTOM_CORE_DNS = """ +apiVersion: v1 +data: +kind: ConfigMap +metadata: + labels: + addonmanager.kubernetes.io/mode: EnsureExists + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + name: coredns-custom + namespace: kube-system +""" diff --git a/src/containerapp/azext_containerapp/_help.py b/src/containerapp/azext_containerapp/_help.py index a85f97b37c5..28b37af9865 100644 --- a/src/containerapp/azext_containerapp/_help.py +++ b/src/containerapp/azext_containerapp/_help.py @@ -1015,6 +1015,23 @@ az containerapp connected-env list -g MyResourceGroup """ +helps['containerapp arc'] = """ + type: group + short-summary: Install prerequisites for Kubernetes cluster on Arc +""" + +helps['containerapp arc setup-core-dns'] = """ + type: command + short-summary: Setup CoreDNS for Kubernetes cluster on Arc + examples: + - name: Setup CoreDNS for Aks on Azure Local on Arc + text: | + az containerapp arc setup-core-dns --distro AksAzureLocal + - name: Setup CoreDNS for Aks on Azure Local on Arc by specifying the kubeconfig and kubecontext. + text: | + az containerapp arc setup-core-dns --distro AksAzureLocal --kube-config /path/to/kubeconfig --kube-context kubeContextName +""" + helps['containerapp connected-env dapr-component'] = """ type: group short-summary: Commands to manage Dapr components for Container Apps connected environments. diff --git a/src/containerapp/azext_containerapp/_params.py b/src/containerapp/azext_containerapp/_params.py index 4529c85b217..f314ef2b9ad 100644 --- a/src/containerapp/azext_containerapp/_params.py +++ b/src/containerapp/azext_containerapp/_params.py @@ -19,7 +19,8 @@ from ._validators import (validate_env_name_or_id, validate_build_env_vars, validate_custom_location_name_or_id, validate_env_name_or_id_for_up, validate_otlp_headers, validate_target_port_range, validate_timeout_in_seconds) -from ._constants import MAXIMUM_CONTAINER_APP_NAME_LENGTH, MAXIMUM_APP_RESILIENCY_NAME_LENGTH, MAXIMUM_COMPONENT_RESILIENCY_NAME_LENGTH +from ._constants import (MAXIMUM_CONTAINER_APP_NAME_LENGTH, MAXIMUM_APP_RESILIENCY_NAME_LENGTH, MAXIMUM_COMPONENT_RESILIENCY_NAME_LENGTH, + AKS_AZURE_LOCAL_DISTRO) def load_arguments(self, _): @@ -365,6 +366,12 @@ def load_arguments(self, _): c.argument('environment_name', options_list=['--name', '-n'], help="The environment name.") c.argument('yaml', type=file_type, help='Path to a .yaml file with the configuration of a Dapr component. All other parameters will be ignored. For an example, see https://learn.microsoft.com/en-us/azure/container-apps/dapr-overview?tabs=bicep1%2Cyaml#component-schema') + with self.argument_context('containerapp arc setup-core-dns') as c: + c.argument('distro', arg_type=get_enum_type([AKS_AZURE_LOCAL_DISTRO]), required=True, help="The distro supported to setup CoreDNS.") + c.argument('kube_config', help="Path to the kube config file.") + c.argument('kube_context', help="Kube context from current machine.") + c.argument('skip_ssl_verification', help="Skip SSL verification for any cluster connection.") + with self.argument_context('containerapp github-action add') as c: c.argument('build_env_vars', nargs='*', help="A list of environment variable(s) for the build. Space-separated values in 'key=value' format.", validator=validate_build_env_vars, is_preview=True) diff --git a/src/containerapp/azext_containerapp/_utils.py b/src/containerapp/azext_containerapp/_utils.py index 3f4db609cef..932aa25bd06 100644 --- a/src/containerapp/azext_containerapp/_utils.py +++ b/src/containerapp/azext_containerapp/_utils.py @@ -13,6 +13,7 @@ import hashlib import re import requests +import shutil import packaging.version as SemVer from enum import Enum diff --git a/src/containerapp/azext_containerapp/commands.py b/src/containerapp/azext_containerapp/commands.py index be175cb1228..22c77317338 100644 --- a/src/containerapp/azext_containerapp/commands.py +++ b/src/containerapp/azext_containerapp/commands.py @@ -187,6 +187,9 @@ def load_command_table(self, args): g.custom_command('set', 'connected_env_create_or_update_storage', supports_no_wait=True, exception_handler=ex_handler_factory()) g.custom_command('remove', 'connected_env_remove_storage', supports_no_wait=True, confirmation=True, exception_handler=ex_handler_factory()) + with self.command_group('containerapp arc', is_preview=True) as g: + g.custom_command('setup-core-dns', 'setup_core_dns', confirmation=True, exception_handler=ex_handler_factory()) + with self.command_group('containerapp env java-component') as g: g.custom_command('list', 'list_java_components') diff --git a/src/containerapp/azext_containerapp/custom.py b/src/containerapp/azext_containerapp/custom.py index 1af47845d04..a638fd9d55e 100644 --- a/src/containerapp/azext_containerapp/custom.py +++ b/src/containerapp/azext_containerapp/custom.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse import json import requests +import copy import subprocess from concurrent.futures import ThreadPoolExecutor @@ -127,7 +128,13 @@ from ._ssh_utils import (SSH_DEFAULT_ENCODING, DebugWebSocketConnection, read_debug_ssh) -from ._utils import connected_env_check_cert_name_availability, get_oryx_run_image_tags, patchable_check, get_pack_exec_path, is_docker_running, parse_build_env_vars, env_has_managed_identity +from ._utils import (connected_env_check_cert_name_availability, get_oryx_run_image_tags, patchable_check, + get_pack_exec_path, is_docker_running, parse_build_env_vars, env_has_managed_identity) + +from ._arc_utils import (get_core_dns_deployment, get_core_dns_configmap, backup_custom_core_dns_configmap, + replace_configmap, replace_deployment, delete_configmap, patch_coredns, + create_folder, create_sub_folder, + check_kube_connection, create_kube_client) from ._constants import (CONTAINER_APPS_RP, NAME_INVALID, NAME_ALREADY_EXISTS, ACR_IMAGE_SUFFIX, DEV_POSTGRES_IMAGE, DEV_POSTGRES_SERVICE_TYPE, @@ -136,7 +143,8 @@ DEV_QDRANT_CONTAINER_NAME, DEV_QDRANT_SERVICE_TYPE, DEV_WEAVIATE_IMAGE, DEV_WEAVIATE_CONTAINER_NAME, DEV_WEAVIATE_SERVICE_TYPE, DEV_MILVUS_IMAGE, DEV_MILVUS_CONTAINER_NAME, DEV_MILVUS_SERVICE_TYPE, DEV_SERVICE_LIST, CONTAINER_APPS_SDK_MODELS, BLOB_STORAGE_TOKEN_STORE_SECRET_SETTING_NAME, DAPR_SUPPORTED_STATESTORE_DEV_SERVICE_LIST, DAPR_SUPPORTED_PUBSUB_DEV_SERVICE_LIST, - JAVA_COMPONENT_CONFIG, JAVA_COMPONENT_EUREKA, JAVA_COMPONENT_ADMIN, JAVA_COMPONENT_NACOS, JAVA_COMPONENT_GATEWAY, DOTNET_COMPONENT_RESOURCE_TYPE) + JAVA_COMPONENT_CONFIG, JAVA_COMPONENT_EUREKA, JAVA_COMPONENT_ADMIN, JAVA_COMPONENT_NACOS, JAVA_COMPONENT_GATEWAY, DOTNET_COMPONENT_RESOURCE_TYPE, + CUSTOM_CORE_DNS, CORE_DNS, KUBE_SYSTEM) logger = get_logger(__name__) @@ -2064,6 +2072,105 @@ def connected_env_remove_storage(cmd, storage_name, name, resource_group_name, n handle_raw_exception(e) +def setup_core_dns(cmd, distro=None, kube_config=None, kube_context=None, skip_ssl_verification=False): + # Checking the connection to kubernetes cluster. + check_kube_connection(kube_config, kube_context, skip_ssl_verification) + + # create a local path to store the original and the changed deployment and core dns configmap. + time_stamp = time.strftime("%Y-%m-%d-%H.%M.%S") + + parent_folder, folder_status, error = create_folder("setup-core-dns", time_stamp) + if not folder_status: + raise ValidationError(error) + + original_folder, folder_status, error = create_sub_folder(parent_folder, "original") + if not folder_status: + raise ValidationError(error) + + kube_client = create_kube_client(kube_config, kube_context, skip_ssl_verification) + + # backup original deployment and configmap + logger.info("Backup existing coredns deployment and configmap") + original_coredns_deployment = get_core_dns_deployment(kube_client, original_folder) + coredns_deployment = copy.deepcopy(original_coredns_deployment) + + original_coredns_configmap = get_core_dns_configmap(kube_client, original_folder) + coredns_configmap = copy.deepcopy(original_coredns_configmap) + + volumes = coredns_deployment.spec.template.spec.volumes + if volumes is None: + raise ValidationError('Unexpected Volumes in coredns deployment, Volumes not found') + + volume_mounts = coredns_deployment.spec.template.spec.containers[0].volume_mounts + if volume_mounts is None: + raise ValidationError('Unexpected Volume mounts in coredns deployment, VolumeMounts not found') + + coredns_configmap_volume_set = False + custom_coredns_configmap_volume_set = False + custom_coredns_configmap_volume_mounted = False + + for volume in volumes: + if volume.config_map is not None: + if volume.config_map.name == CORE_DNS: + for mount in volume_mounts: + if mount.name is not None and mount.name == volume.name: + coredns_configmap_volume_set = True + break + elif volume.config_map.name == CUSTOM_CORE_DNS: + custom_coredns_configmap_volume_set = True + for mount in volume_mounts: + if mount.name is not None and mount.name == volume.name: + custom_coredns_configmap_volume_mounted = True + break + + if not coredns_configmap_volume_set: + raise ValidationError("Cannot find volume and volume mounts for core dns config map") + + original_custom_core_dns_configmap = backup_custom_core_dns_configmap(kube_client, original_folder) + + new_filepath_with_timestamp, folder_status, error = create_sub_folder(parent_folder, "new") + if not folder_status: + raise ValidationError(error) + + try: + patch_coredns(kube_client, coredns_configmap, coredns_deployment, new_filepath_with_timestamp, + original_custom_core_dns_configmap is not None, not custom_coredns_configmap_volume_set, not custom_coredns_configmap_volume_mounted) + except Exception as e: + logger.error(f"Failed to setup custom coredns. {e}") + logger.info("Start to reverted coredns") + replace_succeeded = False + retry_count = 0 + while not replace_succeeded and retry_count < 10: + logger.info(f"Retry the revert operation with retry count {retry_count}") + + try: + logger.info("Start to reverted coredns configmap") + latest_core_dns_configmap = get_core_dns_configmap(kube_client) + latest_core_dns_configmap.data = original_coredns_configmap.data + + replace_configmap(CORE_DNS, KUBE_SYSTEM, kube_client, latest_core_dns_configmap) + logger.info("Reverted coredns configmap successfully") + + logger.info("Start to reverted coredns deployment") + latest_core_dns_deployment = get_core_dns_deployment(kube_client) + latest_core_dns_deployment.spec.template.spec = original_coredns_deployment.spec.template.spec + + replace_deployment(CORE_DNS, KUBE_SYSTEM, kube_client, latest_core_dns_deployment) + logger.info("Reverted coredns deployment successfully") + + if original_custom_core_dns_configmap is None: + delete_configmap(CUSTOM_CORE_DNS, KUBE_SYSTEM, kube_client) + replace_succeeded = True + except Exception as revertEx: + logger.warning(f"Failed to revert coredns configmap or deployment {revertEx}") + retry_count = retry_count + 1 + time.sleep(2) + + if not replace_succeeded: + logger.error(f"Failed to revert the deployment and configuration. " + f"You can get the original coredns config and deployment from {original_folder}") + + def init_dapr_components(cmd, resource_group_name, environment_name, statestore="redis", pubsub="redis"): _validate_subscription_registered(cmd, CONTAINER_APPS_RP) diff --git a/src/containerapp/azext_containerapp/tests/latest/custom_preparers.py b/src/containerapp/azext_containerapp/tests/latest/custom_preparers.py index 25632a81f74..ab0e49f8a4d 100644 --- a/src/containerapp/azext_containerapp/tests/latest/custom_preparers.py +++ b/src/containerapp/azext_containerapp/tests/latest/custom_preparers.py @@ -14,9 +14,10 @@ # pylint: disable=too-many-instance-attributes + class ConnectedClusterPreparer(NoTrafficRecordingPreparer, SingleValueReplacer): def __init__(self, name_prefix='aks', location='eastus2euap', aks_name='my-aks-cluster', connected_cluster_name='my-connected-cluster', - resource_group_parameter_name='resource_group', skip_delete=False): + resource_group_parameter_name='resource_group', skip_delete=False, skip_connected_cluster=False): super(ConnectedClusterPreparer, self).__init__(name_prefix, 15) self.cli_ctx = get_dummy_cli() self.location = location @@ -24,6 +25,7 @@ def __init__(self, name_prefix='aks', location='eastus2euap', aks_name='my-aks-c self.connected_cluster_name = connected_cluster_name self.resource_group_parameter_name = resource_group_parameter_name self.skip_delete = skip_delete + self.skip_connected_cluster = skip_connected_cluster def create_resource(self, name, **kwargs): group = self._get_resource_group(**kwargs) @@ -37,11 +39,12 @@ def create_resource(self, name, **kwargs): self.live_only_execute(self.cli_ctx, f'az aks create --resource-group {group} --name {self.infra_cluster} --enable-aad --generate-ssh-keys --enable-cluster-autoscaler --min-count 4 --max-count 10 --node-count 4 --location {aks_location}') self.live_only_execute(self.cli_ctx, f'az aks get-credentials --resource-group {group} --name {self.infra_cluster} --overwrite-existing --admin') - self.live_only_execute(self.cli_ctx, f'az connectedk8s connect --resource-group {group} --name {self.connected_cluster_name} --location {arc_location}') - connected_cluster = self.live_only_execute(self.cli_ctx, f'az connectedk8s show --resource-group {group} --name {self.connected_cluster_name}').get_output_in_json() - while connected_cluster is not None and connected_cluster["connectivityStatus"] == "Connecting": - time.sleep(5) + if not self.skip_connected_cluster: + self.live_only_execute(self.cli_ctx, f'az connectedk8s connect --resource-group {group} --name {self.connected_cluster_name} --location {arc_location}') connected_cluster = self.live_only_execute(self.cli_ctx, f'az connectedk8s show --resource-group {group} --name {self.connected_cluster_name}').get_output_in_json() + while connected_cluster is not None and connected_cluster["connectivityStatus"] == "Connecting": + time.sleep(5) + connected_cluster = self.live_only_execute(self.cli_ctx, f'az connectedk8s show --resource-group {group} --name {self.connected_cluster_name}').get_output_in_json() except AttributeError: # live only execute returns None if playing from record pass diff --git a/src/containerapp/azext_containerapp/tests/latest/recordings/test_containerapp_arc_setup_core_dns_e2e.yaml b/src/containerapp/azext_containerapp/tests/latest/recordings/test_containerapp_arc_setup_core_dns_e2e.yaml new file mode 100644 index 00000000000..d5741e37850 --- /dev/null +++ b/src/containerapp/azext_containerapp/tests/latest/recordings/test_containerapp_arc_setup_core_dns_e2e.yaml @@ -0,0 +1,419 @@ +interactions: +- request: + body: null + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAPI-Generator/24.2.0/python + method: GET + uri: https://my-aks-clu-clitestrga5j5z-df477a-5jnybmvi.hcp.southcentralus.azmk8s.io/version/ + response: + body: + string: "{\n \"major\": \"1\",\n \"minor\": \"30\",\n \"gitVersion\": \"v1.30.7\",\n + \ \"gitCommit\": \"0c76c645d5a665cfeb736719b1cc47354193dc9a\",\n \"gitTreeState\": + \"clean\",\n \"buildDate\": \"2024-12-02T23:17:46Z\",\n \"goVersion\": \"go1.22.8\",\n + \ \"compiler\": \"gc\",\n \"platform\": \"linux/amd64\"\n}" + headers: + audit-id: + - 8b666f40-5d9c-4c52-8bf5-ddc6b0849255 + cache-control: + - no-cache, private + content-length: + - '263' + content-type: + - application/json + date: + - Tue, 21 Jan 2025 10:08:51 GMT + x-kubernetes-pf-flowschema-uid: + - e5f124d7-149c-4db0-a5eb-7fe0d90bf88a + x-kubernetes-pf-prioritylevel-uid: + - 7c2bb6c8-a20e-4d61-b768-b779c7ee405e + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAPI-Generator/24.2.0/python + method: GET + uri: https://my-aks-clu-clitestrga5j5z-df477a-5jnybmvi.hcp.southcentralus.azmk8s.io/apis/apps/v1/namespaces/kube-system/deployments/mock-deployment?pretty=True + response: + body: + string: "{\n \"kind\": \"Deployment\",\n \"apiVersion\": \"apps/v1\",\n \"metadata\": + {\n \"name\": \"coredns\",\n \"namespace\": \"kube-system\",\n \"uid\": + \"4ef6e1e1-e393-454c-bc0d-0ef830279b87\",\n \"resourceVersion\": \"1315\",\n + \ \"generation\": 2,\n \"creationTimestamp\": \"2025-01-21T10:05:49Z\",\n + \ \"labels\": {\n \"addonmanager.kubernetes.io/mode\": \"Reconcile\",\n + \ \"k8s-app\": \"kube-dns\",\n \"kubernetes.azure.com/managedby\": + \"aks\",\n \"kubernetes.io/cluster-service\": \"true\",\n \"kubernetes.io/name\": + \"CoreDNS\",\n \"version\": \"v20\"\n },\n \"annotations\": {\n + \ \"deployment.kubernetes.io/revision\": \"1\",\n \"kubectl.kubernetes.io/last-applied-configuration\": + \"{\\\"apiVersion\\\":\\\"apps/v1\\\",\\\"kind\\\":\\\"Deployment\\\",\\\"metadata\\\":{\\\"annotations\\\":{},\\\"labels\\\":{\\\"addonmanager.kubernetes.io/mode\\\":\\\"Reconcile\\\",\\\"k8s-app\\\":\\\"kube-dns\\\",\\\"kubernetes.azure.com/managedby\\\":\\\"aks\\\",\\\"kubernetes.io/cluster-service\\\":\\\"true\\\",\\\"kubernetes.io/name\\\":\\\"CoreDNS\\\",\\\"version\\\":\\\"v20\\\"},\\\"name\\\":\\\"coredns\\\",\\\"namespace\\\":\\\"kube-system\\\"},\\\"spec\\\":{\\\"paused\\\":false,\\\"revisionHistoryLimit\\\":2,\\\"selector\\\":{\\\"matchLabels\\\":{\\\"k8s-app\\\":\\\"kube-dns\\\",\\\"version\\\":\\\"v20\\\"}},\\\"strategy\\\":{\\\"rollingUpdate\\\":{\\\"maxUnavailable\\\":1},\\\"type\\\":\\\"RollingUpdate\\\"},\\\"template\\\":{\\\"metadata\\\":{\\\"annotations\\\":{\\\"prometheus.io/port\\\":\\\"9153\\\"},\\\"labels\\\":{\\\"k8s-app\\\":\\\"kube-dns\\\",\\\"kubernetes.azure.com/managedby\\\":\\\"aks\\\",\\\"kubernetes.io/cluster-service\\\":\\\"true\\\",\\\"version\\\":\\\"v20\\\"}},\\\"spec\\\":{\\\"affinity\\\":{\\\"nodeAffinity\\\":{\\\"preferredDuringSchedulingIgnoredDuringExecution\\\":[{\\\"preference\\\":{\\\"matchExpressions\\\":[{\\\"key\\\":\\\"kubernetes.azure.com/mode\\\",\\\"operator\\\":\\\"In\\\",\\\"values\\\":[\\\"system\\\"]}]},\\\"weight\\\":100}],\\\"requiredDuringSchedulingIgnoredDuringExecution\\\":{\\\"nodeSelectorTerms\\\":[{\\\"matchExpressions\\\":[{\\\"key\\\":\\\"kubernetes.azure.com/cluster\\\",\\\"operator\\\":\\\"Exists\\\"},{\\\"key\\\":\\\"type\\\",\\\"operator\\\":\\\"NotIn\\\",\\\"values\\\":[\\\"virtual-kubelet\\\"]},{\\\"key\\\":\\\"kubernetes.io/os\\\",\\\"operator\\\":\\\"In\\\",\\\"values\\\":[\\\"linux\\\"]}]}]}},\\\"podAntiAffinity\\\":{\\\"preferredDuringSchedulingIgnoredDuringExecution\\\":[{\\\"podAffinityTerm\\\":{\\\"labelSelector\\\":{\\\"matchExpressions\\\":[{\\\"key\\\":\\\"k8s-app\\\",\\\"operator\\\":\\\"In\\\",\\\"values\\\":[\\\"kube-dns\\\"]}]},\\\"topologyKey\\\":\\\"topology.kubernetes.io/zone\\\"},\\\"weight\\\":10},{\\\"podAffinityTerm\\\":{\\\"labelSelector\\\":{\\\"matchExpressions\\\":[{\\\"key\\\":\\\"k8s-app\\\",\\\"operator\\\":\\\"In\\\",\\\"values\\\":[\\\"kube-dns\\\"]}]},\\\"topologyKey\\\":\\\"kubernetes.io/hostname\\\"},\\\"weight\\\":5}]}},\\\"containers\\\":[{\\\"args\\\":[\\\"-conf\\\",\\\"/etc/coredns/Corefile\\\"],\\\"env\\\":[{\\\"name\\\":\\\"GOMAXPROCS\\\",\\\"valueFrom\\\":{\\\"resourceFieldRef\\\":{\\\"resource\\\":\\\"limits.cpu\\\"}}}],\\\"image\\\":\\\"mcr.microsoft.com/oss/kubernetes/coredns:v1.9.4-hotfix.20240704\\\",\\\"imagePullPolicy\\\":\\\"IfNotPresent\\\",\\\"livenessProbe\\\":{\\\"failureThreshold\\\":5,\\\"httpGet\\\":{\\\"path\\\":\\\"/health\\\",\\\"port\\\":8080,\\\"scheme\\\":\\\"HTTP\\\"},\\\"initialDelaySeconds\\\":60,\\\"successThreshold\\\":1,\\\"timeoutSeconds\\\":5},\\\"name\\\":\\\"coredns\\\",\\\"ports\\\":[{\\\"containerPort\\\":53,\\\"name\\\":\\\"dns\\\",\\\"protocol\\\":\\\"UDP\\\"},{\\\"containerPort\\\":53,\\\"name\\\":\\\"dns-tcp\\\",\\\"protocol\\\":\\\"TCP\\\"},{\\\"containerPort\\\":9153,\\\"name\\\":\\\"metrics\\\",\\\"protocol\\\":\\\"TCP\\\"}],\\\"readinessProbe\\\":{\\\"httpGet\\\":{\\\"path\\\":\\\"/ready\\\",\\\"port\\\":8181,\\\"scheme\\\":\\\"HTTP\\\"}},\\\"resources\\\":{\\\"limits\\\":{\\\"cpu\\\":3,\\\"memory\\\":\\\"500Mi\\\"},\\\"requests\\\":{\\\"cpu\\\":\\\"100m\\\",\\\"memory\\\":\\\"70Mi\\\"}},\\\"securityContext\\\":{\\\"allowPrivilegeEscalation\\\":false,\\\"capabilities\\\":{\\\"add\\\":[\\\"NET_BIND_SERVICE\\\"],\\\"drop\\\":[\\\"ALL\\\"]},\\\"procMount\\\":\\\"Default\\\",\\\"readOnlyRootFilesystem\\\":true},\\\"volumeMounts\\\":[{\\\"mountPath\\\":\\\"/etc/coredns\\\",\\\"name\\\":\\\"config-volume\\\",\\\"readOnly\\\":true},{\\\"mountPath\\\":\\\"/etc/coredns/custom\\\",\\\"name\\\":\\\"custom-config-volume\\\",\\\"readOnly\\\":true},{\\\"mountPath\\\":\\\"/tmp\\\",\\\"name\\\":\\\"tmp\\\"}]}],\\\"dnsPolicy\\\":\\\"Default\\\",\\\"priorityClassName\\\":\\\"system-node-critical\\\",\\\"securityContext\\\":{\\\"seccompProfile\\\":{\\\"type\\\":\\\"RuntimeDefault\\\"}},\\\"serviceAccountName\\\":\\\"coredns\\\",\\\"tolerations\\\":[{\\\"effect\\\":\\\"NoSchedule\\\",\\\"key\\\":\\\"node-role.kubernetes.io/master\\\"},{\\\"key\\\":\\\"CriticalAddonsOnly\\\",\\\"operator\\\":\\\"Exists\\\"},{\\\"effect\\\":\\\"NoExecute\\\",\\\"key\\\":\\\"node.kubernetes.io/unreachable\\\",\\\"operator\\\":\\\"Exists\\\",\\\"tolerationSeconds\\\":30},{\\\"effect\\\":\\\"NoExecute\\\",\\\"key\\\":\\\"node.kubernetes.io/not-ready\\\",\\\"operator\\\":\\\"Exists\\\",\\\"tolerationSeconds\\\":30}],\\\"volumes\\\":[{\\\"configMap\\\":{\\\"items\\\":[{\\\"key\\\":\\\"Corefile\\\",\\\"path\\\":\\\"Corefile\\\"}],\\\"name\\\":\\\"coredns\\\"},\\\"name\\\":\\\"config-volume\\\"},{\\\"configMap\\\":{\\\"name\\\":\\\"coredns-custom\\\",\\\"optional\\\":true},\\\"name\\\":\\\"custom-config-volume\\\"},{\\\"emptyDir\\\":{},\\\"name\\\":\\\"tmp\\\"}]}}}}\\n\"\n + \ },\n \"managedFields\": [\n {\n \"manager\": \"cluster-proportional-autoscaler\",\n + \ \"operation\": \"Update\",\n \"apiVersion\": \"apps/v1\",\n + \ \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": {\n \"f:spec\": + {\n \"f:replicas\": {}\n }\n },\n \"subresource\": + \"scale\"\n },\n {\n \"manager\": \"kubectl-client-side-apply\",\n + \ \"operation\": \"Update\",\n \"apiVersion\": \"apps/v1\",\n + \ \"time\": \"2025-01-21T10:05:49Z\",\n \"fieldsType\": \"FieldsV1\",\n + \ \"fieldsV1\": {\n \"f:metadata\": {\n \"f:annotations\": + {\n \".\": {},\n \"f:kubectl.kubernetes.io/last-applied-configuration\": + {}\n },\n \"f:labels\": {\n \".\": {},\n + \ \"f:addonmanager.kubernetes.io/mode\": {},\n \"f:k8s-app\": + {},\n \"f:kubernetes.azure.com/managedby\": {},\n \"f:kubernetes.io/cluster-service\": + {},\n \"f:kubernetes.io/name\": {},\n \"f:version\": + {}\n }\n },\n \"f:spec\": {\n \"f:progressDeadlineSeconds\": + {},\n \"f:revisionHistoryLimit\": {},\n \"f:selector\": + {},\n \"f:strategy\": {\n \"f:rollingUpdate\": {\n + \ \".\": {},\n \"f:maxSurge\": {},\n \"f:maxUnavailable\": + {}\n },\n \"f:type\": {}\n },\n \"f:template\": + {\n \"f:metadata\": {\n \"f:annotations\": {\n + \ \".\": {},\n \"f:prometheus.io/port\": + {}\n },\n \"f:labels\": {\n \".\": + {},\n \"f:k8s-app\": {},\n \"f:kubernetes.azure.com/managedby\": + {},\n \"f:kubernetes.io/cluster-service\": {},\n \"f:version\": + {}\n }\n },\n \"f:spec\": {\n \"f:affinity\": + {\n \".\": {},\n \"f:nodeAffinity\": {\n + \ \".\": {},\n \"f:preferredDuringSchedulingIgnoredDuringExecution\": + {},\n \"f:requiredDuringSchedulingIgnoredDuringExecution\": + {}\n },\n \"f:podAntiAffinity\": {\n \".\": + {},\n \"f:preferredDuringSchedulingIgnoredDuringExecution\": + {}\n }\n },\n \"f:containers\": + {\n \"k:{\\\"name\\\":\\\"coredns\\\"}\": {\n \".\": + {},\n \"f:args\": {},\n \"f:env\": {\n + \ \".\": {},\n \"k:{\\\"name\\\":\\\"GOMAXPROCS\\\"}\": + {\n \".\": {},\n \"f:name\": + {},\n \"f:valueFrom\": {\n \".\": + {},\n \"f:resourceFieldRef\": {}\n }\n + \ }\n },\n \"f:image\": + {},\n \"f:imagePullPolicy\": {},\n \"f:livenessProbe\": + {\n \".\": {},\n \"f:failureThreshold\": + {},\n \"f:httpGet\": {\n \".\": + {},\n \"f:path\": {},\n \"f:port\": + {},\n \"f:scheme\": {}\n },\n + \ \"f:initialDelaySeconds\": {},\n \"f:periodSeconds\": + {},\n \"f:successThreshold\": {},\n \"f:timeoutSeconds\": + {}\n },\n \"f:name\": {},\n \"f:ports\": + {\n \".\": {},\n \"k:{\\\"containerPort\\\":53,\\\"protocol\\\":\\\"TCP\\\"}\": + {\n \".\": {},\n \"f:containerPort\": + {},\n \"f:name\": {},\n \"f:protocol\": + {}\n },\n \"k:{\\\"containerPort\\\":53,\\\"protocol\\\":\\\"UDP\\\"}\": + {\n \".\": {},\n \"f:containerPort\": + {},\n \"f:name\": {},\n \"f:protocol\": + {}\n },\n \"k:{\\\"containerPort\\\":9153,\\\"protocol\\\":\\\"TCP\\\"}\": + {\n \".\": {},\n \"f:containerPort\": + {},\n \"f:name\": {},\n \"f:protocol\": + {}\n }\n },\n \"f:readinessProbe\": + {\n \".\": {},\n \"f:failureThreshold\": + {},\n \"f:httpGet\": {\n \".\": + {},\n \"f:path\": {},\n \"f:port\": + {},\n \"f:scheme\": {}\n },\n + \ \"f:periodSeconds\": {},\n \"f:successThreshold\": + {},\n \"f:timeoutSeconds\": {}\n },\n + \ \"f:resources\": {\n \".\": {},\n + \ \"f:limits\": {\n \".\": {},\n + \ \"f:cpu\": {},\n \"f:memory\": + {}\n },\n \"f:requests\": {\n \".\": + {},\n \"f:cpu\": {},\n \"f:memory\": + {}\n }\n },\n \"f:securityContext\": + {\n \".\": {},\n \"f:allowPrivilegeEscalation\": + {},\n \"f:capabilities\": {\n \".\": + {},\n \"f:add\": {},\n \"f:drop\": + {}\n },\n \"f:procMount\": {},\n + \ \"f:readOnlyRootFilesystem\": {}\n },\n + \ \"f:terminationMessagePath\": {},\n \"f:terminationMessagePolicy\": + {},\n \"f:volumeMounts\": {\n \".\": + {},\n \"k:{\\\"mountPath\\\":\\\"/etc/coredns\\\"}\": + {\n \".\": {},\n \"f:mountPath\": + {},\n \"f:name\": {},\n \"f:readOnly\": + {}\n },\n \"k:{\\\"mountPath\\\":\\\"/etc/coredns/custom\\\"}\": + {\n \".\": {},\n \"f:mountPath\": + {},\n \"f:name\": {},\n \"f:readOnly\": + {}\n },\n \"k:{\\\"mountPath\\\":\\\"/tmp\\\"}\": + {\n \".\": {},\n \"f:mountPath\": + {},\n \"f:name\": {}\n }\n }\n + \ }\n },\n \"f:dnsPolicy\": + {},\n \"f:priorityClassName\": {},\n \"f:restartPolicy\": + {},\n \"f:schedulerName\": {},\n \"f:securityContext\": + {\n \".\": {},\n \"f:seccompProfile\": {\n + \ \".\": {},\n \"f:type\": {}\n }\n + \ },\n \"f:serviceAccount\": {},\n \"f:serviceAccountName\": + {},\n \"f:terminationGracePeriodSeconds\": {},\n \"f:tolerations\": + {},\n \"f:volumes\": {\n \".\": {},\n \"k:{\\\"name\\\":\\\"config-volume\\\"}\": + {\n \".\": {},\n \"f:configMap\": {\n + \ \".\": {},\n \"f:defaultMode\": + {},\n \"f:items\": {},\n \"f:name\": + {}\n },\n \"f:name\": {}\n },\n + \ \"k:{\\\"name\\\":\\\"custom-config-volume\\\"}\": {\n \".\": + {},\n \"f:configMap\": {\n \".\": + {},\n \"f:defaultMode\": {},\n \"f:name\": + {},\n \"f:optional\": {}\n },\n \"f:name\": + {}\n },\n \"k:{\\\"name\\\":\\\"tmp\\\"}\": + {\n \".\": {},\n \"f:emptyDir\": {},\n + \ \"f:name\": {}\n }\n }\n + \ }\n }\n }\n }\n },\n {\n + \ \"manager\": \"kube-controller-manager\",\n \"operation\": + \"Update\",\n \"apiVersion\": \"apps/v1\",\n \"time\": \"2025-01-21T10:06:24Z\",\n + \ \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": {\n \"f:metadata\": + {\n \"f:annotations\": {\n \"f:deployment.kubernetes.io/revision\": + {}\n }\n },\n \"f:status\": {\n \"f:availableReplicas\": + {},\n \"f:conditions\": {\n \".\": {},\n \"k:{\\\"type\\\":\\\"Available\\\"}\": + {\n \".\": {},\n \"f:lastTransitionTime\": {},\n + \ \"f:lastUpdateTime\": {},\n \"f:message\": + {},\n \"f:reason\": {},\n \"f:status\": {},\n + \ \"f:type\": {}\n },\n \"k:{\\\"type\\\":\\\"Progressing\\\"}\": + {\n \".\": {},\n \"f:lastTransitionTime\": {},\n + \ \"f:lastUpdateTime\": {},\n \"f:message\": + {},\n \"f:reason\": {},\n \"f:status\": {},\n + \ \"f:type\": {}\n }\n },\n \"f:observedGeneration\": + {},\n \"f:readyReplicas\": {},\n \"f:replicas\": {},\n + \ \"f:updatedReplicas\": {}\n }\n },\n \"subresource\": + \"status\"\n }\n ]\n },\n \"spec\": {\n \"replicas\": 2,\n \"selector\": + {\n \"matchLabels\": {\n \"k8s-app\": \"kube-dns\",\n \"version\": + \"v20\"\n }\n },\n \"template\": {\n \"metadata\": {\n \"creationTimestamp\": + null,\n \"labels\": {\n \"k8s-app\": \"kube-dns\",\n \"kubernetes.azure.com/managedby\": + \"aks\",\n \"kubernetes.io/cluster-service\": \"true\",\n \"version\": + \"v20\"\n },\n \"annotations\": {\n \"prometheus.io/port\": + \"9153\"\n }\n },\n \"spec\": {\n \"volumes\": [\n + \ {\n \"name\": \"config-volume\",\n \"configMap\": + {\n \"name\": \"coredns\",\n \"items\": [\n {\n + \ \"key\": \"Corefile\",\n \"path\": \"Corefile\"\n + \ }\n ],\n \"defaultMode\": 420\n + \ }\n },\n {\n \"name\": \"custom-config-volume\",\n + \ \"configMap\": {\n \"name\": \"coredns-custom\",\n + \ \"defaultMode\": 420,\n \"optional\": true\n }\n + \ },\n {\n \"name\": \"tmp\",\n \"emptyDir\": + {}\n }\n ],\n \"containers\": [\n {\n \"name\": + \"coredns\",\n \"image\": \"mcr.microsoft.com/oss/kubernetes/coredns:v1.9.4-hotfix.20240704\",\n + \ \"args\": [\n \"-conf\",\n \"/etc/coredns/Corefile\"\n + \ ],\n \"ports\": [\n {\n \"name\": + \"dns\",\n \"containerPort\": 53,\n \"protocol\": + \"UDP\"\n },\n {\n \"name\": \"dns-tcp\",\n + \ \"containerPort\": 53,\n \"protocol\": \"TCP\"\n + \ },\n {\n \"name\": \"metrics\",\n + \ \"containerPort\": 9153,\n \"protocol\": \"TCP\"\n + \ }\n ],\n \"env\": [\n {\n + \ \"name\": \"GOMAXPROCS\",\n \"valueFrom\": + {\n \"resourceFieldRef\": {\n \"resource\": + \"limits.cpu\",\n \"divisor\": \"0\"\n }\n + \ }\n }\n ],\n \"resources\": + {\n \"limits\": {\n \"cpu\": \"3\",\n \"memory\": + \"500Mi\"\n },\n \"requests\": {\n \"cpu\": + \"100m\",\n \"memory\": \"70Mi\"\n }\n },\n + \ \"volumeMounts\": [\n {\n \"name\": + \"config-volume\",\n \"readOnly\": true,\n \"mountPath\": + \"/etc/coredns\"\n },\n {\n \"name\": + \"custom-config-volume\",\n \"readOnly\": true,\n \"mountPath\": + \"/etc/coredns/custom\"\n },\n {\n \"name\": + \"tmp\",\n \"mountPath\": \"/tmp\"\n }\n ],\n + \ \"livenessProbe\": {\n \"httpGet\": {\n \"path\": + \"/health\",\n \"port\": 8080,\n \"scheme\": + \"HTTP\"\n },\n \"initialDelaySeconds\": 60,\n \"timeoutSeconds\": + 5,\n \"periodSeconds\": 10,\n \"successThreshold\": + 1,\n \"failureThreshold\": 5\n },\n \"readinessProbe\": + {\n \"httpGet\": {\n \"path\": \"/ready\",\n \"port\": + 8181,\n \"scheme\": \"HTTP\"\n },\n \"timeoutSeconds\": + 1,\n \"periodSeconds\": 10,\n \"successThreshold\": + 1,\n \"failureThreshold\": 3\n },\n \"terminationMessagePath\": + \"/dev/termination-log\",\n \"terminationMessagePolicy\": \"File\",\n + \ \"imagePullPolicy\": \"IfNotPresent\",\n \"securityContext\": + {\n \"capabilities\": {\n \"add\": [\n \"NET_BIND_SERVICE\"\n + \ ],\n \"drop\": [\n \"ALL\"\n + \ ]\n },\n \"readOnlyRootFilesystem\": + true,\n \"allowPrivilegeEscalation\": false,\n \"procMount\": + \"Default\"\n }\n }\n ],\n \"restartPolicy\": + \"Always\",\n \"terminationGracePeriodSeconds\": 30,\n \"dnsPolicy\": + \"Default\",\n \"serviceAccountName\": \"coredns\",\n \"serviceAccount\": + \"coredns\",\n \"securityContext\": {\n \"seccompProfile\": + {\n \"type\": \"RuntimeDefault\"\n }\n },\n \"affinity\": + {\n \"nodeAffinity\": {\n \"requiredDuringSchedulingIgnoredDuringExecution\": + {\n \"nodeSelectorTerms\": [\n {\n \"matchExpressions\": + [\n {\n \"key\": \"kubernetes.azure.com/cluster\",\n + \ \"operator\": \"Exists\"\n },\n {\n + \ \"key\": \"type\",\n \"operator\": + \"NotIn\",\n \"values\": [\n \"virtual-kubelet\"\n + \ ]\n },\n {\n \"key\": + \"kubernetes.io/os\",\n \"operator\": \"In\",\n \"values\": + [\n \"linux\"\n ]\n }\n + \ ]\n }\n ]\n },\n + \ \"preferredDuringSchedulingIgnoredDuringExecution\": [\n {\n + \ \"weight\": 100,\n \"preference\": {\n \"matchExpressions\": + [\n {\n \"key\": \"kubernetes.azure.com/mode\",\n + \ \"operator\": \"In\",\n \"values\": + [\n \"system\"\n ]\n }\n + \ ]\n }\n }\n ]\n },\n + \ \"podAntiAffinity\": {\n \"preferredDuringSchedulingIgnoredDuringExecution\": + [\n {\n \"weight\": 10,\n \"podAffinityTerm\": + {\n \"labelSelector\": {\n \"matchExpressions\": + [\n {\n \"key\": \"k8s-app\",\n + \ \"operator\": \"In\",\n \"values\": + [\n \"kube-dns\"\n ]\n }\n + \ ]\n },\n \"topologyKey\": + \"topology.kubernetes.io/zone\"\n }\n },\n {\n + \ \"weight\": 5,\n \"podAffinityTerm\": {\n \"labelSelector\": + {\n \"matchExpressions\": [\n {\n + \ \"key\": \"k8s-app\",\n \"operator\": + \"In\",\n \"values\": [\n \"kube-dns\"\n + \ ]\n }\n ]\n + \ },\n \"topologyKey\": \"kubernetes.io/hostname\"\n + \ }\n }\n ]\n }\n },\n + \ \"schedulerName\": \"default-scheduler\",\n \"tolerations\": + [\n {\n \"key\": \"node-role.kubernetes.io/master\",\n + \ \"effect\": \"NoSchedule\"\n },\n {\n \"key\": + \"CriticalAddonsOnly\",\n \"operator\": \"Exists\"\n },\n + \ {\n \"key\": \"node.kubernetes.io/unreachable\",\n \"operator\": + \"Exists\",\n \"effect\": \"NoExecute\",\n \"tolerationSeconds\": + 30\n },\n {\n \"key\": \"node.kubernetes.io/not-ready\",\n + \ \"operator\": \"Exists\",\n \"effect\": \"NoExecute\",\n + \ \"tolerationSeconds\": 30\n }\n ],\n \"priorityClassName\": + \"system-node-critical\"\n }\n },\n \"strategy\": {\n \"type\": + \"RollingUpdate\",\n \"rollingUpdate\": {\n \"maxUnavailable\": + 1,\n \"maxSurge\": \"25%\"\n }\n },\n \"revisionHistoryLimit\": + 2,\n \"progressDeadlineSeconds\": 600\n },\n \"status\": {\n \"observedGeneration\": + 2,\n \"replicas\": 2,\n \"updatedReplicas\": 2,\n \"readyReplicas\": + 2,\n \"availableReplicas\": 2,\n \"conditions\": [\n {\n \"type\": + \"Available\",\n \"status\": \"True\",\n \"lastUpdateTime\": + \"2025-01-21T10:06:21Z\",\n \"lastTransitionTime\": \"2025-01-21T10:06:21Z\",\n + \ \"reason\": \"MinimumReplicasAvailable\",\n \"message\": \"Deployment + has minimum availability.\"\n },\n {\n \"type\": \"Progressing\",\n + \ \"status\": \"True\",\n \"lastUpdateTime\": \"2025-01-21T10:06:24Z\",\n + \ \"lastTransitionTime\": \"2025-01-21T10:05:49Z\",\n \"reason\": + \"NewReplicaSetAvailable\",\n \"message\": \"ReplicaSet \\\"coredns-54b69f46b8\\\" + has successfully progressed.\"\n }\n ]\n }\n}" + headers: + audit-id: + - 97c2fbb0-b308-4f81-8da8-8e2afe4870cd + cache-control: + - no-cache, private + content-type: + - application/json + date: + - Tue, 21 Jan 2025 10:08:52 GMT + transfer-encoding: + - chunked + x-kubernetes-pf-flowschema-uid: + - e5f124d7-149c-4db0-a5eb-7fe0d90bf88a + x-kubernetes-pf-prioritylevel-uid: + - 7c2bb6c8-a20e-4d61-b768-b779c7ee405e + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAPI-Generator/24.2.0/python + method: GET + uri: https://my-aks-clu-clitestrga5j5z-df477a-5jnybmvi.hcp.southcentralus.azmk8s.io/api/v1/namespaces/kube-system/configmaps/coredns?pretty=True + response: + body: + string: "{\n \"kind\": \"ConfigMap\",\n \"apiVersion\": \"v1\",\n \"metadata\": + {\n \"name\": \"coredns\",\n \"namespace\": \"kube-system\",\n \"uid\": + \"af905ba0-1853-4243-ba8b-0ac84e68b8f2\",\n \"resourceVersion\": \"516\",\n + \ \"creationTimestamp\": \"2025-01-21T10:05:49Z\",\n \"labels\": {\n + \ \"addonmanager.kubernetes.io/mode\": \"Reconcile\",\n \"k8s-app\": + \"kube-dns\",\n \"kubernetes.io/cluster-service\": \"true\"\n },\n + \ \"annotations\": {\n \"kubectl.kubernetes.io/last-applied-configuration\": + \"{\\\"apiVersion\\\":\\\"v1\\\",\\\"data\\\":{\\\"Corefile\\\":\\\".:53 {\\\\n + \ errors\\\\n ready\\\\n health {\\\\n lameduck 5s\\\\n }\\\\n + \ kubernetes cluster.local in-addr.arpa ip6.arpa {\\\\n pods insecure\\\\n + \ fallthrough in-addr.arpa ip6.arpa\\\\n ttl 30\\\\n }\\\\n prometheus + :9153\\\\n forward . /etc/resolv.conf\\\\n cache 30\\\\n loop\\\\n + \ reload\\\\n loadbalance\\\\n import custom/*.override\\\\n template + ANY ANY internal.cloudapp.net {\\\\n match \\\\\\\"^(?:[^.]+\\\\\\\\.){4,}internal\\\\\\\\.cloudapp\\\\\\\\.net\\\\\\\\.$\\\\\\\"\\\\n + \ rcode NXDOMAIN\\\\n fallthrough\\\\n }\\\\n template ANY + ANY reddog.microsoft.com {\\\\n rcode NXDOMAIN\\\\n }\\\\n}\\\\nimport + custom/*.server\\\\n\\\"},\\\"kind\\\":\\\"ConfigMap\\\",\\\"metadata\\\":{\\\"annotations\\\":{},\\\"labels\\\":{\\\"addonmanager.kubernetes.io/mode\\\":\\\"Reconcile\\\",\\\"k8s-app\\\":\\\"kube-dns\\\",\\\"kubernetes.io/cluster-service\\\":\\\"true\\\"},\\\"name\\\":\\\"coredns\\\",\\\"namespace\\\":\\\"kube-system\\\"}}\\n\"\n + \ },\n \"managedFields\": [\n {\n \"manager\": \"kubectl-client-side-apply\",\n + \ \"operation\": \"Update\",\n \"apiVersion\": \"v1\",\n \"time\": + \"2025-01-21T10:05:49Z\",\n \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": + {\n \"f:data\": {\n \".\": {},\n \"f:Corefile\": + {}\n },\n \"f:metadata\": {\n \"f:annotations\": + {\n \".\": {},\n \"f:kubectl.kubernetes.io/last-applied-configuration\": + {}\n },\n \"f:labels\": {\n \".\": {},\n + \ \"f:addonmanager.kubernetes.io/mode\": {},\n \"f:k8s-app\": + {},\n \"f:kubernetes.io/cluster-service\": {}\n }\n + \ }\n }\n }\n ]\n },\n \"data\": {\n \"Corefile\": + \".:53 {\\n errors\\n ready\\n health {\\n lameduck 5s\\n }\\n + \ kubernetes cluster.local in-addr.arpa ip6.arpa {\\n pods insecure\\n + \ fallthrough in-addr.arpa ip6.arpa\\n ttl 30\\n }\\n prometheus + :9153\\n forward . /etc/resolv.conf\\n cache 30\\n loop\\n reload\\n + \ loadbalance\\n import custom/*.override\\n template ANY ANY internal.cloudapp.net + {\\n match \\\"^(?:[^.]+\\\\.){4,}internal\\\\.cloudapp\\\\.net\\\\.$\\\"\\n + \ rcode NXDOMAIN\\n fallthrough\\n }\\n template ANY ANY reddog.microsoft.com + {\\n rcode NXDOMAIN\\n }\\n}\\nimport custom/*.server\\n\"\n }\n}" + headers: + audit-id: + - a5ec2754-ac7e-4462-a4ae-2a4d2269feee + cache-control: + - no-cache, private + content-type: + - application/json + date: + - Tue, 21 Jan 2025 10:08:54 GMT + transfer-encoding: + - chunked + x-kubernetes-pf-flowschema-uid: + - e5f124d7-149c-4db0-a5eb-7fe0d90bf88a + x-kubernetes-pf-prioritylevel-uid: + - 7c2bb6c8-a20e-4d61-b768-b779c7ee405e + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - application/json + Content-Type: + - application/json + User-Agent: + - OpenAPI-Generator/24.2.0/python + method: GET + uri: https://my-aks-clu-clitestrga5j5z-df477a-5jnybmvi.hcp.southcentralus.azmk8s.io/api/v1/namespaces/kube-system/configmaps/coredns-custom?pretty=True + response: + body: + string: "{\n \"kind\": \"ConfigMap\",\n \"apiVersion\": \"v1\",\n \"metadata\": + {\n \"name\": \"coredns-custom\",\n \"namespace\": \"kube-system\",\n + \ \"uid\": \"fb42657a-6cfb-443a-8a5f-09525eb04eb0\",\n \"resourceVersion\": + \"425\",\n \"creationTimestamp\": \"2025-01-21T10:05:41Z\",\n \"labels\": + {\n \"addonmanager.kubernetes.io/mode\": \"EnsureExists\",\n \"k8s-app\": + \"kube-dns\",\n \"kubernetes.io/cluster-service\": \"true\"\n },\n + \ \"managedFields\": [\n {\n \"manager\": \"kubectl-create\",\n + \ \"operation\": \"Update\",\n \"apiVersion\": \"v1\",\n \"time\": + \"2025-01-21T10:05:41Z\",\n \"fieldsType\": \"FieldsV1\",\n \"fieldsV1\": + {\n \"f:metadata\": {\n \"f:labels\": {\n \".\": + {},\n \"f:addonmanager.kubernetes.io/mode\": {},\n \"f:k8s-app\": + {},\n \"f:kubernetes.io/cluster-service\": {}\n }\n + \ }\n }\n }\n ]\n }\n}" + headers: + audit-id: + - 542881e3-49a7-402f-aee5-816c00732aa9 + cache-control: + - no-cache, private + content-length: + - '901' + content-type: + - application/json + date: + - Tue, 21 Jan 2025 10:08:54 GMT + x-kubernetes-pf-flowschema-uid: + - e5f124d7-149c-4db0-a5eb-7fe0d90bf88a + x-kubernetes-pf-prioritylevel-uid: + - 7c2bb6c8-a20e-4d61-b768-b779c7ee405e + status: + code: 200 + message: OK +version: 1 diff --git a/src/containerapp/azext_containerapp/tests/latest/test_containerapp_arc_commands.py b/src/containerapp/azext_containerapp/tests/latest/test_containerapp_arc_commands.py new file mode 100644 index 00000000000..dccec584694 --- /dev/null +++ b/src/containerapp/azext_containerapp/tests/latest/test_containerapp_arc_commands.py @@ -0,0 +1,31 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azure.cli.testsdk.scenario_tests import AllowLargeResponse +from azure.cli.testsdk import (ScenarioTest, ResourceGroupPreparer, live_only) +from azure.cli.testsdk.decorators import serial_test +from .custom_preparers import ConnectedClusterPreparer + + +class ContainerAppArcTest(ScenarioTest): + def __init__(self, *arg, **kwargs): + super().__init__(*arg, random_config_dir=True, **kwargs) + + @AllowLargeResponse(8192) + def test_containerapp_arc_invalid_command(self): + with self.assertRaises(SystemExit) as cm: + self.cmd('containerapp arc setup-core-dns --yes') + self.assertNotEqual(cm.exception.code, 0) + + with self.assertRaises(SystemExit) as cm: + self.cmd('containerapp arc setup-core-dns --distro Aks-Hci --yes') + self.assertNotEqual(cm.exception.code, 0) + + @serial_test() + @live_only() + @ResourceGroupPreparer(location="southcentralus", random_name_length=15) + @ConnectedClusterPreparer(location="southcentralus", skip_connected_cluster=True) + def test_containerapp_arc_setup_core_dns_e2e(self, resource_group, connected_cluster_name): + self.cmd('containerapp arc setup-core-dns --distro AksAzureLocal --yes', expect_failure=False) \ No newline at end of file diff --git a/src/containerapp/setup.py b/src/containerapp/setup.py index f1bde4f28e0..5771c2465e8 100644 --- a/src/containerapp/setup.py +++ b/src/containerapp/setup.py @@ -47,7 +47,7 @@ ] # TODO: Add any additional SDK dependencies here -DEPENDENCIES = ['pycomposefile>=0.0.29', 'docker'] +DEPENDENCIES = ['pycomposefile>=0.0.29', 'docker', 'kubernetes==24.2.0'] # Install pack CLI to build runnable application images from source _ = get_pack_exec_path()