CRD cleanup + helm client download retires + error classification

src/connectedk8s/HISTORY.rst

@@ -2,6 +2,13 @@
 
 Release History
 
+1.3.18
+++++++
+
+* Checking Outbound connectivity to OBO endpoint 
+* Cleaning up stale CRDs if present during onboarding (even in absence of azure-arc release)
+* Adding retries in Helm client download
+
 1.3.17
 ++++++
 

src/connectedk8s/azext_connectedk8s/_constants.py

@@ -197,7 +197,7 @@
 Outbound_Network_Connectivity_Check = "outbound_network_connectivity_check.txt"
 Events_of_Incomplete_Diagnoser_Job = "diagnoser_failure_events.txt"
 # Connect Precheck Diagnoser constants
-Cluster_Diagnostic_Checks_Job_Registry_Path = "mcr.microsoft.com/azurearck8s/helmchart/stable/clusterdiagnosticchecks:0.1.1"
+Cluster_Diagnostic_Checks_Job_Registry_Path = "connectprecheck.azurecr.io/helm/new-pre-onboarding-inspector:0.1.9-new"
 Cluster_Diagnostic_Checks_Helm_Install_Failed_Fault_Type = "Error while installing cluster diagnostic checks helm release"
 Cluster_Diagnostic_Checks_Execution_Failed_Fault_Type = "Error occured while executing cluster diagnostic checks"
 Cluster_Diagnostic_Checks_Release_Cleanup_Failed = "Error occured while cleaning up the cluster diagnostic checks helm release"
@@ -212,6 +212,9 @@
 Cluster_Diagnostic_Checks_Job_Log_Save_Failed = 'Failed to save cluster diagnostic checks job log'
 # Diagnostic Results Name
 Outbound_Connectivity_Check_Result_String = "Outbound Network Connectivity Result:"
+Outbound_Connectivity_Check_String = "Outbound Network Connectivity"
+Outbound_Connectivity_Check_String_For_OBO_Endpoint = "Outbound Network Connectivity Check for OBO endpoints"
+Outbound_Connectivity_Check_Failed_For_OBO = "Outbound network connectivity check failed for OBO endpoint"
 DNS_Check_Result_String = "DNS Result:"
 AZ_CLI_ADAL_TO_MSAL_MIGRATE_VERSION = '2.30.0'
 CLIENT_PROXY_VERSION = '1.3.022011'

src/connectedk8s/azext_connectedk8s/_precheckutils.py

@@ -59,18 +59,23 @@ def fetch_diagnostic_checks_results(corev1_api_instance, batchv1_api_instance, h
             cluster_diagnostic_checks_container_log_list = cluster_diagnostic_checks_container_log.split("\n")
             cluster_diagnostic_checks_container_log_list.pop(-1)
             dns_check_log = ""
+            outbound_check_log = ""
             counter_container_logs = 1
             # For retrieving only cluster_diagnostic_checks logs from the output
             for outputs in cluster_diagnostic_checks_container_log_list:
-                if consts.Outbound_Connectivity_Check_Result_String in outputs:
+                if consts.Outbound_Connectivity_Check_String in outputs:
                     counter_container_logs = 1
+                    if outbound_check_log == "":
+                        outbound_check_log += outputs
+                    else:
+                        outbound_check_log += "  " + outputs
                 elif consts.DNS_Check_Result_String in outputs:
                     dns_check_log += outputs
                     counter_container_logs = 0
                 elif counter_container_logs == 0:
                     dns_check_log += "  " + outputs
             dns_check, storage_space_available = azext_utils.check_cluster_DNS(dns_check_log, filepath_with_timestamp, storage_space_available, diagnoser_output)
-            outbound_connectivity_check, storage_space_available = azext_utils.check_cluster_outbound_connectivity(cluster_diagnostic_checks_container_log_list[-1], filepath_with_timestamp, storage_space_available, diagnoser_output)
+            outbound_connectivity_check, storage_space_available = azext_utils.check_cluster_outbound_connectivity(outbound_check_log, filepath_with_timestamp, storage_space_available, diagnoser_output)
         else:
             return consts.Diagnostic_Check_Incomplete, storage_space_available
 

src/connectedk8s/azext_connectedk8s/_utils.py

@@ -238,12 +238,26 @@ def check_cluster_outbound_connectivity(outbound_connectivity_check_log, filepat
         outbound_connectivity_response = outbound_connectivity_response[::-1]
         if consts.Outbound_Connectivity_Check_Result_String not in outbound_connectivity_check_log:
             return consts.Diagnostic_Check_Incomplete, storage_space_available
+
+        # extracting the obo endpoint log specifically from outbound connectivity check job logs
+        obo_check_log = outbound_connectivity_check_log.split('  ')[0]
+        # extracting the obo endpoint url
+        obo_enpoint_url = obo_check_log.split(" : ")[1]
+        # extracting the obo endpoint response code from outbound connectivity check
+        obo_check_response_code = obo_check_log.split(" : ")[2]
         # Validating if outbound connectiivty is working or not and displaying proper result
         if(outbound_connectivity_response != "000"):
+            # since connectivity to obo endpoint is required only for cluster connect feature , we only give out a warning if connectivity to obo endpoint is not present
+            if obo_check_response_code == "000":
+                logger.warning("The outbound network connectivity check for has failed for the endpoint - " + obo_enpoint_url + "\nThis will affect the \"cluser-connect\" functionality. If you have configured an outbound proxy in your environment, please ensure that all endpoints are properly whitelisted by referring to the network requirements outlined here: https://learn.microsoft.com/en-us/azure/azure-arc/network-requirements-consolidated?tabs=azure-cloud#azure-arc-enabled-kubernetes-endpoints.\n")
+                telemetry.set_exception(exception='Outbound network connectivity check failed for the OBO endpoint', fault_type=consts.Outbound_Connectivity_Check_Failed_For_OBO, summary="Outbound network connectivity check failed for the OBO endpoint")
             if storage_space_available:
                 outbound_connectivity_check_path = os.path.join(filepath_with_timestamp, consts.Outbound_Network_Connectivity_Check)
                 with open(outbound_connectivity_check_path, 'w+') as outbound:
-                    outbound.write("Response code " + outbound_connectivity_response + "\nOutbound network connectivity check passed successfully.")
+                    if obo_check_response_code == "000":
+                        outbound.write("Response code " + outbound_connectivity_response + "\nOutbound network connectivity check passed successfully." + "\nOutbound connectivity failed for the endpoint:" + obo_enpoint_url + " ,this is an optional endpoint needed for cluster-connect feature.")
+                    else:
+                        outbound.write("Response code " + outbound_connectivity_response + "\nOutbound network connectivity check passed successfully.")
             return consts.Diagnostic_Check_Passed, storage_space_available
         else:
             logger.warning("Error: We found an issue with outbound network connectivity from the cluster.\nPlease ensure to meet the following network requirements 'https://docs.microsoft.com/en-us/azure/azure-arc/kubernetes/quickstart-connect-cluster?tabs=azure-cli#meet-network-requirements' \nIf your cluster is behind an outbound proxy server, please ensure that you have passed proxy parameters during the onboarding of your cluster.\nFor more details visit 'https://docs.microsoft.com/en-us/azure/azure-arc/kubernetes/quickstart-connect-cluster?tabs=azure-cli#connect-using-an-outbound-proxy-server' \n")

src/connectedk8s/azext_connectedk8s/custom.py

@@ -100,6 +100,9 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, correlat
         account = Profile().get_subscription(subscription_id)
         onboarding_tenant_id = account['homeTenantId']
 
+    resource_id = f'/subscriptions/{subscription_id}/resourcegroups/{resource_group_name}/providers/Microsoft.Kubernetes/connectedClusters/{cluster_name}/location/{location}'
+    telemetry.add_extension_event('connectedk8s', {'Context.Default.AzureCLI.resourceid': resource_id})
+
     # Send cloud information to telemetry
     azure_cloud = send_cloud_telemetry(cmd)
 
@@ -243,9 +246,6 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, correlat
     }
     telemetry.add_extension_event('connectedk8s', kubernetes_properties)
 
-    resource_id = f'/subscriptions/{subscription_id}/resourcegroups/{resource_group_name}/providers/Microsoft.Kubernetes/connectedClusters/{cluster_name}/location/{location}'
-    telemetry.add_extension_event('connectedk8s', {'Context.Default.AzureCLI.resourceid': resource_id})
-
     # Checking if it is an AKS cluster
     is_aks_cluster = check_aks_cluster(kube_config, kube_context)
     if is_aks_cluster:
@@ -331,6 +331,9 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, correlat
                                      "in the resource group {} ".format(resource_group_name) +
                                      "and corresponds to a different Kubernetes cluster.", recommendation="To onboard this Kubernetes cluster " +
                                      "to Azure, specify different resource name or resource group name.")
+        else:
+            # cleanup of stuck CRD if release namespace is not present/deleted
+            crd_cleanup_force_delete(kubectl_client_location, kube_config, kube_context)
 
     # Resource group Creation
     if resource_group_exists(cmd.cli_ctx, resource_group_name, subscription_id) is False:
@@ -509,12 +512,18 @@ def install_helm_client():
 
         # Downloading compressed helm client executable
         logger.warning("Downloading helm client for first time. This can take few minutes...")
-        try:
-            response = urllib.request.urlopen(requestUri)
-        except Exception as e:
-            telemetry.set_exception(exception=e, fault_type=consts.Download_Helm_Fault_Type,
-                                    summary='Unable to download helm client.')
-            raise CLIInternalError("Failed to download helm client.", recommendation="Please check your internet connection." + str(e))
+        retry_count = 3
+        retry_delay = 5
+        for i in range(retry_count):
+            try:
+                response = urllib.request.urlopen(requestUri)
+                break
+            except Exception as e:
+                if i == retry_count - 1:
+                    telemetry.set_exception(exception=e, fault_type=consts.Download_Helm_Fault_Type,
+                                            summary='Unable to download helm client.')
+                    raise CLIInternalError("Failed to download helm client.", recommendation="Please check your internet connection." + str(e))
+                time.sleep(retry_delay)
 
         responseContent = response.read()
         response.close()

src/connectedk8s/setup.py

@@ -17,7 +17,7 @@
 # TODO: Confirm this is the right version number you want and it matches your
 # HISTORY.rst entry.
 
-VERSION = '1.3.17'
+VERSION = '1.3.18'
 
 # The full list of classifiers is available at
 # https://pypi.python.org/pypi?%3Aaction=list_classifiers