AzExt - Adding support for the v2021-09-01-preview API

docs/disk-encryption-set.md

@@ -0,0 +1,84 @@
+# Using a custom Disk Encryption Set
+
+## What is the Disk Encryption Set used for?
+
+In summary, it allows a customer to control the keys that are used to encrypt/decrypt VM disks.
+See [deploy-a-vm-with-customer-managed-keys](https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal#deploy-a-vm-with-customer-managed-keys) for more information.
+
+## How to deploy?
+First, install and use the AzureCLI extension with
+```bash
+make az
+```
+
+>You can check if the extension is in use by running:
+```bash
+az extension list
+[
+  {
+    "experimental": false,
+    "extensionType": "dev",
+    "name": "aro",
+    "path": "<path to go SRC>/github.com/Azure/ARO-RP/python/az/aro",
+    "preview": true,
+    "version": "1.0.1"
+  }
+]
+```
+
+Follow [tutorial-create-cluster](https://docs.microsoft.com/en-us/azure/openshift/tutorial-create-cluster) but don't run the `az aro create` command, instead proceed as follows:
+
+  - set additional env variables
+```bash
+export KEYVAULT_NAME=$USER-enckv
+export KEYVAULT_KEY_NAME=$USER-key
+export DISK_ENCRYPTION_SET_NAME=$USER-des
+```
+  - create the KeyVault and Key
+```bash
+az keyvault create -n $KEYVAULT_NAME \
+                   -g $RESOURCEGROUP \
+                   -l $LOCATION \
+                   --enable-purge-protection true \
+                   --enable-soft-delete true
+
+az keyvault key create --vault-name $KEYVAULT_NAME \
+                       -n $KEYVAULT_KEY_NAME \
+                       --protection software
+
+KEYVAULT_ID=$(az keyvault show --name $KEYVAULT_NAME --query "[id]" -o tsv)
+
+KEYVAULT_KEY_URL=$(az keyvault key show --vault-name $KEYVAULT_NAME \
+                                        --name $KEYVAULT_KEY_NAME \
+                                        --query "[key.kid]" -o tsv)
+```
+  - create the DES and add permissions to use the KeyVault
+```bash
+az disk-encryption-set create -n $DISK_ENCRYPTION_SET_NAME \
+                              -l $LOCATION \
+                              -g $RESOURCEGROUP \
+                              --source-vault $KEYVAULT_ID \
+                              --key-url $KEYVAULT_KEY_URL
+
+DES_IDENTITY=$(az disk-encryption-set show -n $DISK_ENCRYPTION_SET_NAME \
+                                           -g $RESOURCEGROUP \
+                                           --query "[identity.principalId]" \
+                                           -o tsv)
+
+az keyvault set-policy -n $KEYVAULT_NAME \
+                       -g $RESOURCEGROUP \
+                       --object-id $DES_IDENTITY \
+                       --key-permissions wrapkey unwrapkey get
+```
+  - run the az aro create command
+```bash
+az aro create --resource-group $RESOURCEGROUP \
+              --name $CLUSTER  \
+              --vnet aro-vnet  \
+              --master-subnet master-subnet \
+              --worker-subnet worker-subnet \
+              --disk-encryption-set $DES_ID
+```
+
+After creating the cluster all VMs should have the customer controlled Disk Encryption Set.
+>Remember to delete the disk-encryption-set and keyvault when done.

python/az/aro/HISTORY.rst

@@ -25,3 +25,7 @@ Release History
 1.0.0
 ++++++
 * Remove preview flag.
+
+1.0.1
+++++++
+* Switch to new preview API

python/az/aro/azext_aro/_client_factory.py

@@ -4,7 +4,7 @@
 import urllib3
 
 from azext_aro.custom import rp_mode_development
-from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30 import AzureRedHatOpenShiftClient
+from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview import AzureRedHatOpenShiftClient
 from azure.cli.core.commands.client_factory import get_mgmt_service_client
 
 

python/az/aro/azext_aro/_params.py

@@ -4,8 +4,10 @@
 from azext_aro._validators import validate_cidr
 from azext_aro._validators import validate_client_id
 from azext_aro._validators import validate_cluster_resource_group
+from azext_aro._validators import validate_disk_encryption_set
 from azext_aro._validators import validate_domain
 from azext_aro._validators import validate_pull_secret
+from azext_aro._validators import validate_sdn
 from azext_aro._validators import validate_subnet
 from azext_aro._validators import validate_client_secret
 from azext_aro._validators import validate_visibility
@@ -54,10 +56,23 @@ def load_arguments(self, _):
         c.argument('service_cidr',
                    help='CIDR of service network. Must be a minimum of /18 or larger.',
                    validator=validate_cidr('service_cidr'))
+        c.argument('software_defined_network', arg_type=get_enum_type(['OVNKubernetes', 'OpenShiftSDN']),
+                   options_list=['--software-defined-network-type', '--sdn-type'],
+                   help='SDN type either "OpenShiftSDN" (default) or "OVNKubernetes"',
+                   validator=validate_sdn)
 
+        c.argument('disk_encryption_set',
+                   help='ResourceID of the DiskEncryptionSet to be used for master and worker VMs.',
+                   validator=validate_disk_encryption_set)
+        c.argument('master_encryption_at_host', arg_type=get_three_state_flag(),
+                   options_list=['--master-encryption-at-host', '--master-enc-host'],
+                   help='Encryption at host flag for master VMs.')
         c.argument('master_vm_size',
                    help='Size of master VMs.')
 
+        c.argument('worker_encryption_at_host', arg_type=get_three_state_flag(),
+                   options_list=['--worker-encryption-at-host', '--worker-enc-host'],
+                   help='Encryption at host flag for worker VMs.')
         c.argument('worker_vm_size',
                    help='Size of worker VMs.')
         c.argument('worker_vm_disk_size_gb',

python/az/aro/azext_aro/_rbac.py

@@ -11,7 +11,8 @@
 from msrest.exceptions import ValidationError
 from msrestazure.tools import resource_id
 
-NETWORK_CONTRIBUTOR = '4d97b98b-1d4f-4787-a291-c67834d212e7'
+ROLE_NETWORK_CONTRIBUTOR = '4d97b98b-1d4f-4787-a291-c67834d212e7'
+ROLE_READER = 'acdd72a7-3385-48ef-bd42-f606fba81ae7'
 
 logger = get_logger(__name__)
 
@@ -34,7 +35,7 @@ def _create_role_assignment(auth_client, resource, params):
             logger.warning("%s; retry %d of %d", ex, retries, max_retries)
 
 
-def assign_network_contributor_to_resource(cli_ctx, resource, object_id):
+def assign_role_to_resource(cli_ctx, resource, object_id, role_name):
     auth_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_AUTHORIZATION)
 
     RoleAssignmentCreateParameters = get_sdk(cli_ctx, ResourceType.MGMT_AUTHORIZATION,
@@ -45,7 +46,7 @@ def assign_network_contributor_to_resource(cli_ctx, resource, object_id):
         subscription=get_subscription_id(cli_ctx),
         namespace='Microsoft.Authorization',
         type='roleDefinitions',
-        name=NETWORK_CONTRIBUTOR,
+        name=role_name,
     )
 
     _create_role_assignment(auth_client, resource, RoleAssignmentCreateParameters(
@@ -55,14 +56,14 @@ def assign_network_contributor_to_resource(cli_ctx, resource, object_id):
     ))
 
 
-def has_network_contributor_on_resource(cli_ctx, resource, object_id):
+def has_role_assignment_on_resource(cli_ctx, resource, object_id, role_name):
     auth_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_AUTHORIZATION)
 
     role_definition_id = resource_id(
         subscription=get_subscription_id(cli_ctx),
         namespace='Microsoft.Authorization',
         type='roleDefinitions',
-        name=NETWORK_CONTRIBUTOR,
+        name=role_name,
     )
 
     for assignment in auth_client.role_assignments.list_for_scope(resource):

python/az/aro/azext_aro/_validators.py

@@ -9,7 +9,7 @@
 from azure.cli.core.commands.client_factory import get_mgmt_service_client
 from azure.cli.core.commands.client_factory import get_subscription_id
 from azure.cli.core.profiles import ResourceType
-from azure.cli.core.azclierror import CLIInternalError, InvalidArgumentValueError, \
+from azure.cli.core.azclierror import InvalidArgumentValueError, \
     RequiredArgumentMissingError
 from knack.log import get_logger
 from msrestazure.azure_exceptions import CloudError
@@ -64,6 +64,23 @@ def validate_cluster_resource_group(cmd, namespace):
                 namespace.cluster_resource_group)
 
 
+def validate_disk_encryption_set(cmd, namespace):
+    if namespace.disk_encryption_set is not None:
+        if not is_valid_resource_id(namespace.disk_encryption_set):
+            raise InvalidArgumentValueError(
+                "Invalid --disk-encryption-set '%s', has to be a resource ID." %
+                namespace.disk_encryption_set)
+
+        desid = parse_resource_id(namespace.disk_encryption_set)
+        compute_client = get_mgmt_service_client(cmd.cli_ctx, ResourceType.MGMT_COMPUTE)
+        try:
+            compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'],
+                                                    disk_encryption_set_name=desid['name'])
+        except CloudError as err:
+            raise InvalidArgumentValueError("Invald --disc-encryption-set, error when getting '%s': %s" %
+                                            (namespace.disk_encryption_set, err.message)) from err
+
+
 def validate_domain(namespace):
     if namespace.domain is not None:
         if not re.match(r'^' +
@@ -90,6 +107,13 @@ def validate_pull_secret(namespace):
             raise InvalidArgumentValueError("Invalid --pull-secret.") from e
 
 
+def validate_sdn(namespace):
+    if namespace.software_defined_network is not None:
+        if namespace.software_defined_network not in ['OVNKubernetes', 'OpenshiftSDN']:
+            raise InvalidArgumentValueError("Invalid --software-defined-network '%s'." %
+                                            namespace.software_defined_network)
+
+
 def validate_subnet(key):
     def _validate_subnet(cmd, namespace):
         subnet = getattr(namespace, key)
@@ -136,7 +160,8 @@ def _validate_subnet(cmd, namespace):
             client.subnets.get(parts['resource_group'],
                                parts['name'], parts['child_name_1'])
         except CloudError as err:
-            raise CLIInternalError(err.message) from err
+            raise InvalidArgumentValueError("Invald --%s, error when getting '%s': %s" %
+                                            (key.replace('_', '-'), subnet, err.message)) from err
 
     return _validate_subnet
 

python/az/aro/azext_aro/commands.py

@@ -10,7 +10,7 @@
 
 def load_command_table(self, _):
     aro_sdk = CliCommandType(
-        operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30.operations#OpenShiftClustersOperations.{}',  # pylint: disable=line-too-long
+        operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.operations#OpenShiftClustersOperations.{}',  # pylint: disable=line-too-long
         client_factory=cf_aro)
 
     with self.command_group('aro', aro_sdk, client_factory=cf_aro) as g:

python/az/aro/azext_aro/custom.py

@@ -15,10 +15,11 @@
 from msrest.exceptions import HttpOperationError
 from knack.log import get_logger
 
-import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30.models as openshiftcluster
+import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.models as openshiftcluster
 
 from azext_aro._aad import AADManager
-from azext_aro._rbac import assign_network_contributor_to_resource, has_network_contributor_on_resource
+from azext_aro._rbac import assign_role_to_resource, has_role_assignment_on_resource
+from azext_aro._rbac import ROLE_NETWORK_CONTRIBUTOR, ROLE_READER
 from azext_aro._validators import validate_subnets
 
 logger = get_logger(__name__)
@@ -42,7 +43,11 @@ def aro_create(cmd,  # pylint: disable=too-many-locals
                client_secret=None,
                pod_cidr=None,
                service_cidr=None,
+               software_defined_network=None,
+               disk_encryption_set=None,
+               master_encryption_at_host=False,
                master_vm_size=None,
+               worker_encryption_at_host=False,
                worker_vm_size=None,
                worker_vm_disk_size_gb=None,
                worker_count=None,
@@ -104,10 +109,13 @@ def aro_create(cmd,  # pylint: disable=too-many-locals
         network_profile=openshiftcluster.NetworkProfile(
             pod_cidr=pod_cidr or '10.128.0.0/14',
             service_cidr=service_cidr or '172.30.0.0/16',
+            software_defined_network=software_defined_network or 'OpenShiftSDN'
         ),
         master_profile=openshiftcluster.MasterProfile(
             vm_size=master_vm_size or 'Standard_D8s_v3',
             subnet_id=master_subnet,
+            encryption_at_host='Enabled' if master_encryption_at_host else 'Disabled',
+            disk_encryption_set_id=disk_encryption_set,
         ),
         worker_profiles=[
             openshiftcluster.WorkerProfile(
@@ -116,6 +124,8 @@ def aro_create(cmd,  # pylint: disable=too-many-locals
                 disk_size_gb=worker_vm_disk_size_gb or 128,
                 subnet_id=worker_subnet,
                 count=worker_count or 3,
+                encryption_at_host='Enabled' if worker_encryption_at_host else 'Disabled',
+                disk_encryption_set_id=disk_encryption_set,
             )
         ],
         apiserver_profile=openshiftcluster.APIServerProfile(
@@ -279,6 +289,13 @@ def get_network_resources(cli_ctx, subnets, vnet):
     return resources
 
 
+def get_disk_encryption_resources(oc):
+    disk_encryption_set = oc.master_profile.disk_encryption_set_id
+    resources = set()
+    resources.add(disk_encryption_set)
+    return resources
+
+
 # cluster_application_update manages cluster application & service principal update
 # If called without parameters it should be best-effort
 # If called with parameters it fails if something is not possible
@@ -363,8 +380,9 @@ def resolve_rp_client_id():
 
 def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids):
     try:
-        # Get cluster resources we need to assign network contributor on
-        resources = get_cluster_network_resources(cli_ctx, oc)
+        # Get cluster resources we need to assign permissions on, sort to ensure the same order of operations
+        resources = {ROLE_NETWORK_CONTRIBUTOR: sorted(get_cluster_network_resources(cli_ctx, oc)),
+                     ROLE_READER: sorted(get_disk_encryption_resources(oc))}
     except (CloudError, HttpOperationError) as e:
         if fail:
             logger.error(e.message)
@@ -373,18 +391,19 @@ def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids):
         return
 
     for sp_id in sp_obj_ids:
-        for resource in sorted(resources):
-            # Create the role assignment if it doesn't exist
-            # Assume that the role assignment exists if we fail to look it up
-            resource_contributor_exists = True
-
-            try:
-                resource_contributor_exists = has_network_contributor_on_resource(cli_ctx, resource, sp_id)
-            except CloudError as e:
-                if fail:
-                    logger.error(e.message)
-                    raise
-                logger.info(e.message)
-
-            if not resource_contributor_exists:
-                assign_network_contributor_to_resource(cli_ctx, resource, sp_id)
+        for role in sorted(resources):
+            for resource in resources[role]:
+                # Create the role assignment if it doesn't exist
+                # Assume that the role assignment exists if we fail to look it up
+                resource_contributor_exists = True
+
+                try:
+                    resource_contributor_exists = has_role_assignment_on_resource(cli_ctx, resource, sp_id, role)
+                except CloudError as e:
+                    if fail:
+                        logger.error(e.message)
+                        raise
+                    logger.info(e.message)
+
+                if not resource_contributor_exists:
+                    assign_role_to_resource(cli_ctx, resource, sp_id, role)