[k8s-extension] add support for provisionedClusters (#144)

* add provisionedClusters support for k8s-extension

* support added in azuredefender and containerInsights

* add support for azuremlkubernetes

* add support for Dapr and OpenServiceMesh

* add support in model

* fix the cli errors

* remove redundant changes for osm

* add description for cmd

* updating the api version

* modify the description of the argument

Co-authored-by: Bavneet Singh <[email protected]>

Author: bavneetsingh16

src/k8s-extension/azext_k8s_extension/_params.py

@@ -29,9 +29,12 @@ def load_arguments(self, _):
                    options_list=['--cluster-name', '-c'],
                    help='Name of the Kubernetes cluster')
         c.argument('cluster_type',
-                   arg_type=get_enum_type(['connectedClusters', 'managedClusters', 'appliances']),
+                   arg_type=get_enum_type(['connectedClusters', 'managedClusters', 'appliances', 'provisionedClusters']),
                    options_list=['--cluster-type', '-t'],
-                   help='Specify Arc clusters or AKS managed clusters or Arc appliances.')
+                   help='Specify Arc clusters or AKS managed clusters or Arc appliances or provisionedClusters.')
+        c.argument('cluster_resource_provider',
+                   options_list=['--cluster-resource-provider', '--cluster-rp'],
+                   help='Cluster Resource Provider name for this clusterType (Required for provisionedClusters)')
         c.argument('scope',
                    arg_type=get_enum_type(['cluster', 'namespace']),
                    help='Specify the extension scope.')

src/k8s-extension/azext_k8s_extension/consts.py

@@ -13,11 +13,14 @@
 CONNECTED_CLUSTER_RP = "Microsoft.Kubernetes"
 MANAGED_CLUSTER_RP = "Microsoft.ContainerService"
 APPLIANCE_RP = "Microsoft.ResourceConnector"
+HYBRIDCONTAINERSERVICE_RP = "microsoft.hybridcontainerservice"
 
 CONNECTED_CLUSTER_TYPE = "connectedclusters"
 MANAGED_CLUSTER_TYPE = "managedclusters"
 APPLIANCE_TYPE = "appliances"
+PROVISIONED_CLUSTER_TYPE = "provisionedclusters"
 
 CONNECTED_CLUSTER_API_VERSION = "2021-10-01"
 MANAGED_CLUSTER_API_VERSION = "2021-10-01"
 APPLIANCE_API_VERSION = "2021-10-31-preview"
+HYBRIDCONTAINERSERVICE_API_VERSION = "2022-05-01-preview"

src/k8s-extension/azext_k8s_extension/custom.py

@@ -53,10 +53,10 @@ def ExtensionFactory(extension_name):
     return extension_map.get(extension_name, DefaultExtension)()
 
 
-def show_k8s_extension(client, resource_group_name, cluster_name, name, cluster_type):
+def show_k8s_extension(client, resource_group_name, cluster_name, name, cluster_type, cluster_resource_provider=None):
     """Get an existing K8s Extension."""
     # Determine ClusterRP
-    cluster_rp, _ = get_cluster_rp_api_version(cluster_type)
+    cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_resource_provider)
 
     try:
         extension = client.get(
@@ -95,6 +95,7 @@ def create_k8s_extension(
     name,
     cluster_type,
     extension_type,
+    cluster_resource_provider=None,
     scope=None,
     auto_upgrade_minor_version=None,
     release_train=None,
@@ -110,7 +111,7 @@ def create_k8s_extension(
     """Create a new Extension Instance."""
 
     extension_type_lower = extension_type.lower()
-    cluster_rp, _ = get_cluster_rp_api_version(cluster_type)
+    cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_resource_provider)
 
     # Configuration Settings & Configuration Protected Settings
     if configuration_settings is not None and configuration_settings_file is not None:
@@ -167,6 +168,7 @@ def create_k8s_extension(
         cluster_name,
         name,
         cluster_type,
+        cluster_rp,
         extension_type_lower,
         scope,
         auto_upgrade_minor_version,
@@ -193,7 +195,7 @@ def create_k8s_extension(
     # We don't create the identity if we are in DF
     if create_identity and not is_dogfood_cluster(cmd):
         identity_object, location = __create_identity(
-            cmd, resource_group_name, cluster_name, cluster_type
+            cmd, resource_group_name, cluster_name, cluster_type, cluster_rp
         )
         if identity_object is not None and location is not None:
             extension_instance.identity, extension_instance.location = (
@@ -214,8 +216,8 @@ def create_k8s_extension(
     )
 
 
-def list_k8s_extension(client, resource_group_name, cluster_name, cluster_type):
-    cluster_rp, _ = get_cluster_rp_api_version(cluster_type)
+def list_k8s_extension(client, resource_group_name, cluster_name, cluster_type, cluster_resource_provider=None):
+    cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_resource_provider)
     return client.list(resource_group_name, cluster_rp, cluster_type, cluster_name)
 
 
@@ -226,6 +228,7 @@ def update_k8s_extension(
     cluster_name,
     name,
     cluster_type,
+    cluster_resource_provider=None,
     auto_upgrade_minor_version=None,
     release_train=None,
     version=None,
@@ -253,22 +256,21 @@ def update_k8s_extension(
         user_confirmation_factory(cmd, yes, msg)
 
     # Determine ClusterRP
-    cluster_rp, _ = get_cluster_rp_api_version(cluster_type)
+    cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_resource_provider)
 
     # We need to determine the ExtensionType to call ExtensionFactory and create Extension class
     extension = show_k8s_extension(
-        client, resource_group_name, cluster_name, name, cluster_type
+        client, resource_group_name, cluster_name, name, cluster_type, cluster_rp
     )
     extension_type_lower = extension.extension_type.lower()
 
-    config_settings = None
-    config_protected_settings = None
+    config_settings = {}
+    config_protected_settings = {}
     # Get Configuration Settings from file
     if configuration_settings_file is not None:
         config_settings = read_config_settings_file(configuration_settings_file)
 
     if configuration_settings is not None:
-        config_settings = {}
         for dicts in configuration_settings:
             for key, value in dicts.items():
                 config_settings[key] = value
@@ -280,7 +282,6 @@ def update_k8s_extension(
         )
 
     if configuration_protected_settings is not None:
-        config_protected_settings = {}
         for dicts in configuration_protected_settings:
             for key, value in dicts.items():
                 config_protected_settings[key] = value
@@ -320,13 +321,14 @@ def delete_k8s_extension(
     cluster_name,
     name,
     cluster_type,
+    cluster_resource_provider=None,
     no_wait=False,
     yes=False,
     force=False,
 ):
     """Delete an existing Kubernetes Extension."""
     # Determine ClusterRP
-    cluster_rp, _ = get_cluster_rp_api_version(cluster_type)
+    cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_resource_provider)
     extension = None
     try:
         extension = client.get(
@@ -343,7 +345,7 @@ def delete_k8s_extension(
 
     # If there is any custom delete logic, this will call the logic
     extension_class.Delete(
-        cmd, client, resource_group_name, cluster_name, name, cluster_type, yes
+        cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes
     )
 
     return sdk_no_wait(
@@ -358,7 +360,7 @@ def delete_k8s_extension(
     )
 
 
-def __create_identity(cmd, resource_group_name, cluster_name, cluster_type):
+def __create_identity(cmd, resource_group_name, cluster_name, cluster_type, cluster_rp):
     subscription_id = get_subscription_id(cmd.cli_ctx)
     resources = cf_resources(cmd.cli_ctx, subscription_id)
 
@@ -369,7 +371,7 @@ def __create_identity(cmd, resource_group_name, cluster_name, cluster_type):
     ):
         return None, None
 
-    cluster_rp, parent_api_version = get_cluster_rp_api_version(cluster_type)
+    cluster_rp, parent_api_version = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp)
 
     cluster_resource_id = (
         "/subscriptions/{0}/resourceGroups/{1}/providers/{2}/{3}/{4}".format(

src/k8s-extension/azext_k8s_extension/partner_extensions/AzureDefender.py

@@ -18,8 +18,8 @@
 
 
 class AzureDefender(DefaultExtension):
-    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, extension_type,
-               scope, auto_upgrade_minor_version, release_train, version, target_namespace,
+    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
+               extension_type, scope, auto_upgrade_minor_version, release_train, version, target_namespace,
                release_namespace, configuration_settings, configuration_protected_settings,
                configuration_settings_file, configuration_protected_settings_file):
 
@@ -43,7 +43,7 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
                        'only supports cluster scope and single instance of this extension.', extension_type)
         logger.warning("Defaulting to extension name '%s' and release-namespace '%s'", name, release_namespace)
 
-        _get_container_insights_settings(cmd, resource_group_name, cluster_name, configuration_settings,
+        _get_container_insights_settings(cmd, resource_group_name, cluster_rp, cluster_type, cluster_name, configuration_settings,
                                          configuration_protected_settings, is_ci_extension_type)
 
         # NOTE-2: Return a valid Extension object, Instance name and flag for Identity

src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py

@@ -108,8 +108,8 @@ def __init__(self):
 
         self.OPEN_SHIFT = 'openshift'
 
-    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, extension_type,
-               scope, auto_upgrade_minor_version, release_train, version, target_namespace,
+    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
+               extension_type, scope, auto_upgrade_minor_version, release_train, version, target_namespace,
                release_namespace, configuration_settings, configuration_protected_settings,
                configuration_settings_file, configuration_protected_settings_file):
         if scope == 'namespace':
@@ -126,7 +126,7 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
 
         # get the arc's location
         subscription_id = get_subscription_id(cmd.cli_ctx)
-        cluster_rp, parent_api_version = get_cluster_rp_api_version(cluster_type)
+        cluster_rp, parent_api_version = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp)
         cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers/{2}' \
             '/{3}/{4}'.format(subscription_id, resource_group_name, cluster_rp, cluster_type, cluster_name)
         cluster_location = ''
@@ -216,7 +216,7 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
         )
         return extension, name, create_identity
 
-    def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, yes):
+    def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes):
         user_confirmation_factory(cmd, yes)
 
     def Update(self, cmd, resource_group_name, cluster_name, auto_upgrade_minor_version, release_train, version, configuration_settings,

src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py

@@ -32,8 +32,8 @@
 
 
 class ContainerInsights(DefaultExtension):
-    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, extension_type,
-               scope, auto_upgrade_minor_version, release_train, version, target_namespace,
+    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
+               extension_type, scope, auto_upgrade_minor_version, release_train, version, target_namespace,
                release_namespace, configuration_settings, configuration_protected_settings,
                configuration_settings_file, configuration_protected_settings_file):
         """ExtensionType 'microsoft.azuremonitor.containers' specific validations & defaults for Create
@@ -56,7 +56,7 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
                        'only supports cluster scope and single instance of this extension.', extension_type)
         logger.warning("Defaulting to extension name '%s' and release-namespace '%s'", name, release_namespace)
 
-        _get_container_insights_settings(cmd, resource_group_name, cluster_name, configuration_settings,
+        _get_container_insights_settings(cmd, resource_group_name, cluster_rp, cluster_type, cluster_name, configuration_settings,
                                          configuration_protected_settings, is_ci_extension_type)
 
         # NOTE-2: Return a valid Extension object, Instance name and flag for Identity
@@ -72,11 +72,11 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
         )
         return extension, name, create_identity
 
-    def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, yes):
+    def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes):
         # Delete DCR-A if it exists incase of MSI Auth
         useAADAuth = False
         isDCRAExists = False
-        cluster_rp, _ = get_cluster_rp_api_version(cluster_type)
+        cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp)
         try:
             extension = client.get(resource_group_name, cluster_rp, cluster_type, cluster_name, name)
         except Exception:
@@ -140,8 +140,8 @@ def _invoke_deployment(cmd, resource_group_name, deployment_name, template, para
     return sdk_no_wait(no_wait, smc.begin_create_or_update, resource_group_name, deployment_name, deployment)
 
 
-def _ensure_default_log_analytics_workspace_for_monitoring(cmd, subscription_id,
-                                                           cluster_resource_group_name, cluster_name):
+def _ensure_default_log_analytics_workspace_for_monitoring(cmd, subscription_id, cluster_resource_group_name,
+                                                           cluster_rp, cluster_type, cluster_name):
     # mapping for azure public cloud
     # log analytics workspaces cannot be created in WCUS region due to capacity limits
     # so mapped to EUS per discussion with log analytics team
@@ -236,8 +236,8 @@ def _ensure_default_log_analytics_workspace_for_monitoring(cmd, subscription_id,
     cluster_location = ''
     resources = cf_resources(cmd.cli_ctx, subscription_id)
 
-    cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Kubernetes' \
-        '/connectedClusters/{2}'.format(subscription_id, cluster_resource_group_name, cluster_name)
+    cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers//{2}/{3}/{4}'.format(
+        subscription_id, cluster_resource_group_name, cluster_rp, cluster_type, cluster_name)
     try:
         resource = resources.get_by_id(cluster_resource_id, '2020-01-01-preview')
         cluster_location = resource.location.lower()
@@ -438,8 +438,8 @@ def _ensure_container_insights_for_monitoring(cmd, workspace_resource_id):
                               validate=False, no_wait=False, subscription_id=subscription_id)
 
 
-def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_name, configuration_settings,
-                                     configuration_protected_settings, is_ci_extension_type):
+def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_rp, cluster_type, cluster_name,
+                                     configuration_settings, configuration_protected_settings, is_ci_extension_type):
 
     subscription_id = get_subscription_id(cmd.cli_ctx)
     workspace_resource_id = ''
@@ -477,15 +477,15 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_n
 
     if not workspace_resource_id:
         workspace_resource_id = _ensure_default_log_analytics_workspace_for_monitoring(
-            cmd, subscription_id, cluster_resource_group_name, cluster_name)
+            cmd, subscription_id, cluster_resource_group_name, cluster_rp, cluster_type, cluster_name)
     else:
         if not is_valid_resource_id(workspace_resource_id):
             raise InvalidArgumentValueError('{} is not a valid Azure resource ID.'.format(workspace_resource_id))
 
     if is_ci_extension_type:
         if useAADAuth:
             logger.info("creating data collection rule and association")
-            _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_resource_group_name, cluster_name, workspace_resource_id)
+            _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_resource_group_name, cluster_rp, cluster_type, cluster_name, workspace_resource_id)
         elif not _is_container_insights_solution_exists(cmd, workspace_resource_id):
             logger.info("Creating ContainerInsights solution resource, since it doesn't exist and it is using legacy authentication")
             _ensure_container_insights_for_monitoring(cmd, workspace_resource_id).result()
@@ -545,13 +545,13 @@ def get_existing_container_insights_extension_dcr_tags(cmd, dcr_url):
     return tags
 
 
-def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_resource_group_name, cluster_name, workspace_resource_id):
+def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_resource_group_name, cluster_rp, cluster_type, cluster_name, workspace_resource_id):
     from azure.core.exceptions import HttpResponseError
 
     cluster_region = ''
     resources = cf_resources(cmd.cli_ctx, subscription_id)
-    cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers/Microsoft.Kubernetes' \
-        '/connectedClusters/{2}'.format(subscription_id, cluster_resource_group_name, cluster_name)
+    cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers/{2}/{3}/{4}'.format(
+        subscription_id, cluster_resource_group_name, cluster_rp, cluster_type, cluster_name)
     try:
         resource = resources.get_by_id(cluster_resource_id, '2020-01-01-preview')
         cluster_region = resource.location.lower()

src/k8s-extension/azext_k8s_extension/partner_extensions/Dapr.py

@@ -19,8 +19,8 @@ def __init__(self):
         # constants for configuration settings.
         self.CLUSTER_TYPE = 'global.clusterType'
 
-    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, extension_type,
-               scope, auto_upgrade_minor_version, release_train, version, target_namespace,
+    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
+               extension_type, scope, auto_upgrade_minor_version, release_train, version, target_namespace,
                release_namespace, configuration_settings, configuration_protected_settings,
                configuration_settings_file, configuration_protected_settings_file):
 

src/k8s-extension/azext_k8s_extension/partner_extensions/DefaultExtension.py

@@ -25,6 +25,7 @@ def Create(
         cluster_name,
         name,
         cluster_type,
+        cluster_rp,
         extension_type,
         scope,
         auto_upgrade_minor_version,
@@ -87,7 +88,7 @@ def Update(
         )
 
     def Delete(
-        self, cmd, client, resource_group_name, cluster_name, name, cluster_type, yes
+        self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes
     ):
         user_confirmation_factory(cmd, yes)
 

src/k8s-extension/azext_k8s_extension/partner_extensions/OpenServiceMesh.py

@@ -32,8 +32,8 @@
 
 class OpenServiceMesh(DefaultExtension):
 
-    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, extension_type,
-               scope, auto_upgrade_minor_version, release_train, version, target_namespace,
+    def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp,
+               extension_type, scope, auto_upgrade_minor_version, release_train, version, target_namespace,
                release_namespace, configuration_settings, configuration_protected_settings,
                configuration_settings_file, configuration_protected_settings_file):
         """ExtensionType 'microsoft.openservicemesh' specific validations & defaults for Create

src/k8s-extension/azext_k8s_extension/partner_extensions/PartnerExtensionModel.py

@@ -18,6 +18,7 @@ def Create(
         cluster_name: str,
         name: str,
         cluster_type: str,
+        cluster_rp: str,
         extension_type: str,
         scope: str,
         auto_upgrade_minor_version: bool,
@@ -57,6 +58,7 @@ def Delete(
         cluster_name: str,
         name: str,
         cluster_type: str,
+        cluster_rp: str,
         yes: bool,
     ):
         pass