diff --git a/docs/disk-encryption-set.md b/docs/disk-encryption-set.md new file mode 100644 index 00000000000..e4ae331d481 --- /dev/null +++ b/docs/disk-encryption-set.md @@ -0,0 +1,84 @@ +# Using a custom Disk Encryption Set + +## What is the Disk Encryption Set used for? + +In summary, it allows a customer to control the keys that are used to encrypt/decrypt VM disks. +See [deploy-a-vm-with-customer-managed-keys](https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal#deploy-a-vm-with-customer-managed-keys) for more information. + +## How to deploy? +First, install and use the AzureCLI extension with +```bash +make az +``` + +>You can check if the extension is in use by running: +```bash +az extension list +[ + { + "experimental": false, + "extensionType": "dev", + "name": "aro", + "path": "/github.com/Azure/ARO-RP/python/az/aro", + "preview": true, + "version": "1.0.1" + } +] +``` + +Follow [tutorial-create-cluster](https://docs.microsoft.com/en-us/azure/openshift/tutorial-create-cluster) but don't run the `az aro create` command, instead proceed as follows: + + - set additional env variables +```bash +export KEYVAULT_NAME=$USER-enckv +export KEYVAULT_KEY_NAME=$USER-key +export DISK_ENCRYPTION_SET_NAME=$USER-des +``` + - create the KeyVault and Key +```bash +az keyvault create -n $KEYVAULT_NAME \ + -g $RESOURCEGROUP \ + -l $LOCATION \ + --enable-purge-protection true \ + --enable-soft-delete true + +az keyvault key create --vault-name $KEYVAULT_NAME \ + -n $KEYVAULT_KEY_NAME \ + --protection software + +KEYVAULT_ID=$(az keyvault show --name $KEYVAULT_NAME --query "[id]" -o tsv) + +KEYVAULT_KEY_URL=$(az keyvault key show --vault-name $KEYVAULT_NAME \ + --name $KEYVAULT_KEY_NAME \ + --query "[key.kid]" -o tsv) +``` + - create the DES and add permissions to use the KeyVault +```bash +az disk-encryption-set create -n $DISK_ENCRYPTION_SET_NAME \ + -l $LOCATION \ + -g $RESOURCEGROUP \ + --source-vault $KEYVAULT_ID \ + --key-url $KEYVAULT_KEY_URL + +DES_IDENTITY=$(az disk-encryption-set show -n $DISK_ENCRYPTION_SET_NAME \ + -g $RESOURCEGROUP \ + --query "[identity.principalId]" \ + -o tsv) + +az keyvault set-policy -n $KEYVAULT_NAME \ + -g $RESOURCEGROUP \ + --object-id $DES_IDENTITY \ + --key-permissions wrapkey unwrapkey get +``` + - run the az aro create command +```bash +az aro create --resource-group $RESOURCEGROUP \ + --name $CLUSTER \ + --vnet aro-vnet \ + --master-subnet master-subnet \ + --worker-subnet worker-subnet \ + --disk-encryption-set $DES_ID +``` + +After creating the cluster all VMs should have the customer controlled Disk Encryption Set. +>Remember to delete the disk-encryption-set and keyvault when done. diff --git a/python/az/aro/HISTORY.rst b/python/az/aro/HISTORY.rst index 8a86c83c09c..f7f5057c81b 100644 --- a/python/az/aro/HISTORY.rst +++ b/python/az/aro/HISTORY.rst @@ -25,3 +25,7 @@ Release History 1.0.0 ++++++ * Remove preview flag. + +1.0.1 +++++++ +* Switch to new preview API diff --git a/python/az/aro/azext_aro/_client_factory.py b/python/az/aro/azext_aro/_client_factory.py index e03b5a16c46..9963abc2161 100644 --- a/python/az/aro/azext_aro/_client_factory.py +++ b/python/az/aro/azext_aro/_client_factory.py @@ -4,7 +4,7 @@ import urllib3 from azext_aro.custom import rp_mode_development -from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30 import AzureRedHatOpenShiftClient +from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview import AzureRedHatOpenShiftClient from azure.cli.core.commands.client_factory import get_mgmt_service_client diff --git a/python/az/aro/azext_aro/_params.py b/python/az/aro/azext_aro/_params.py index db41c9fab90..9b6e331718f 100644 --- a/python/az/aro/azext_aro/_params.py +++ b/python/az/aro/azext_aro/_params.py @@ -4,8 +4,10 @@ from azext_aro._validators import validate_cidr from azext_aro._validators import validate_client_id from azext_aro._validators import validate_cluster_resource_group +from azext_aro._validators import validate_disk_encryption_set from azext_aro._validators import validate_domain from azext_aro._validators import validate_pull_secret +from azext_aro._validators import validate_sdn from azext_aro._validators import validate_subnet from azext_aro._validators import validate_client_secret from azext_aro._validators import validate_visibility @@ -54,10 +56,23 @@ def load_arguments(self, _): c.argument('service_cidr', help='CIDR of service network. Must be a minimum of /18 or larger.', validator=validate_cidr('service_cidr')) + c.argument('software_defined_network', arg_type=get_enum_type(['OVNKubernetes', 'OpenShiftSDN']), + options_list=['--software-defined-network-type', '--sdn-type'], + help='SDN type either "OpenShiftSDN" (default) or "OVNKubernetes"', + validator=validate_sdn) + c.argument('disk_encryption_set', + help='ResourceID of the DiskEncryptionSet to be used for master and worker VMs.', + validator=validate_disk_encryption_set) + c.argument('master_encryption_at_host', arg_type=get_three_state_flag(), + options_list=['--master-encryption-at-host', '--master-enc-host'], + help='Encryption at host flag for master VMs.') c.argument('master_vm_size', help='Size of master VMs.') + c.argument('worker_encryption_at_host', arg_type=get_three_state_flag(), + options_list=['--worker-encryption-at-host', '--worker-enc-host'], + help='Encryption at host flag for worker VMs.') c.argument('worker_vm_size', help='Size of worker VMs.') c.argument('worker_vm_disk_size_gb', diff --git a/python/az/aro/azext_aro/_rbac.py b/python/az/aro/azext_aro/_rbac.py index adcd7ffec8e..f096004d7b8 100644 --- a/python/az/aro/azext_aro/_rbac.py +++ b/python/az/aro/azext_aro/_rbac.py @@ -11,7 +11,8 @@ from msrest.exceptions import ValidationError from msrestazure.tools import resource_id -NETWORK_CONTRIBUTOR = '4d97b98b-1d4f-4787-a291-c67834d212e7' +ROLE_NETWORK_CONTRIBUTOR = '4d97b98b-1d4f-4787-a291-c67834d212e7' +ROLE_READER = 'acdd72a7-3385-48ef-bd42-f606fba81ae7' logger = get_logger(__name__) @@ -34,7 +35,7 @@ def _create_role_assignment(auth_client, resource, params): logger.warning("%s; retry %d of %d", ex, retries, max_retries) -def assign_network_contributor_to_resource(cli_ctx, resource, object_id): +def assign_role_to_resource(cli_ctx, resource, object_id, role_name): auth_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_AUTHORIZATION) RoleAssignmentCreateParameters = get_sdk(cli_ctx, ResourceType.MGMT_AUTHORIZATION, @@ -45,7 +46,7 @@ def assign_network_contributor_to_resource(cli_ctx, resource, object_id): subscription=get_subscription_id(cli_ctx), namespace='Microsoft.Authorization', type='roleDefinitions', - name=NETWORK_CONTRIBUTOR, + name=role_name, ) _create_role_assignment(auth_client, resource, RoleAssignmentCreateParameters( @@ -55,14 +56,14 @@ def assign_network_contributor_to_resource(cli_ctx, resource, object_id): )) -def has_network_contributor_on_resource(cli_ctx, resource, object_id): +def has_role_assignment_on_resource(cli_ctx, resource, object_id, role_name): auth_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_AUTHORIZATION) role_definition_id = resource_id( subscription=get_subscription_id(cli_ctx), namespace='Microsoft.Authorization', type='roleDefinitions', - name=NETWORK_CONTRIBUTOR, + name=role_name, ) for assignment in auth_client.role_assignments.list_for_scope(resource): diff --git a/python/az/aro/azext_aro/_validators.py b/python/az/aro/azext_aro/_validators.py index e0a58082495..07a1c24e524 100644 --- a/python/az/aro/azext_aro/_validators.py +++ b/python/az/aro/azext_aro/_validators.py @@ -9,7 +9,7 @@ from azure.cli.core.commands.client_factory import get_mgmt_service_client from azure.cli.core.commands.client_factory import get_subscription_id from azure.cli.core.profiles import ResourceType -from azure.cli.core.azclierror import CLIInternalError, InvalidArgumentValueError, \ +from azure.cli.core.azclierror import InvalidArgumentValueError, \ RequiredArgumentMissingError from knack.log import get_logger from msrestazure.azure_exceptions import CloudError @@ -64,6 +64,23 @@ def validate_cluster_resource_group(cmd, namespace): namespace.cluster_resource_group) +def validate_disk_encryption_set(cmd, namespace): + if namespace.disk_encryption_set is not None: + if not is_valid_resource_id(namespace.disk_encryption_set): + raise InvalidArgumentValueError( + "Invalid --disk-encryption-set '%s', has to be a resource ID." % + namespace.disk_encryption_set) + + desid = parse_resource_id(namespace.disk_encryption_set) + compute_client = get_mgmt_service_client(cmd.cli_ctx, ResourceType.MGMT_COMPUTE) + try: + compute_client.disk_encryption_sets.get(resource_group_name=desid['resource_group'], + disk_encryption_set_name=desid['name']) + except CloudError as err: + raise InvalidArgumentValueError("Invald --disc-encryption-set, error when getting '%s': %s" % + (namespace.disk_encryption_set, err.message)) from err + + def validate_domain(namespace): if namespace.domain is not None: if not re.match(r'^' + @@ -90,6 +107,13 @@ def validate_pull_secret(namespace): raise InvalidArgumentValueError("Invalid --pull-secret.") from e +def validate_sdn(namespace): + if namespace.software_defined_network is not None: + if namespace.software_defined_network not in ['OVNKubernetes', 'OpenshiftSDN']: + raise InvalidArgumentValueError("Invalid --software-defined-network '%s'." % + namespace.software_defined_network) + + def validate_subnet(key): def _validate_subnet(cmd, namespace): subnet = getattr(namespace, key) @@ -136,7 +160,8 @@ def _validate_subnet(cmd, namespace): client.subnets.get(parts['resource_group'], parts['name'], parts['child_name_1']) except CloudError as err: - raise CLIInternalError(err.message) from err + raise InvalidArgumentValueError("Invald --%s, error when getting '%s': %s" % + (key.replace('_', '-'), subnet, err.message)) from err return _validate_subnet diff --git a/python/az/aro/azext_aro/commands.py b/python/az/aro/azext_aro/commands.py index 29665ba945e..f0ef86ae764 100644 --- a/python/az/aro/azext_aro/commands.py +++ b/python/az/aro/azext_aro/commands.py @@ -10,7 +10,7 @@ def load_command_table(self, _): aro_sdk = CliCommandType( - operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30.operations#OpenShiftClustersOperations.{}', # pylint: disable=line-too-long + operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.operations#OpenShiftClustersOperations.{}', # pylint: disable=line-too-long client_factory=cf_aro) with self.command_group('aro', aro_sdk, client_factory=cf_aro) as g: diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py index fcdf60379c1..f557fc37d9f 100644 --- a/python/az/aro/azext_aro/custom.py +++ b/python/az/aro/azext_aro/custom.py @@ -15,10 +15,11 @@ from msrest.exceptions import HttpOperationError from knack.log import get_logger -import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2020_04_30.models as openshiftcluster +import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2021_09_01_preview.models as openshiftcluster from azext_aro._aad import AADManager -from azext_aro._rbac import assign_network_contributor_to_resource, has_network_contributor_on_resource +from azext_aro._rbac import assign_role_to_resource, has_role_assignment_on_resource +from azext_aro._rbac import ROLE_NETWORK_CONTRIBUTOR, ROLE_READER from azext_aro._validators import validate_subnets logger = get_logger(__name__) @@ -42,7 +43,11 @@ def aro_create(cmd, # pylint: disable=too-many-locals client_secret=None, pod_cidr=None, service_cidr=None, + software_defined_network=None, + disk_encryption_set=None, + master_encryption_at_host=False, master_vm_size=None, + worker_encryption_at_host=False, worker_vm_size=None, worker_vm_disk_size_gb=None, worker_count=None, @@ -104,10 +109,13 @@ def aro_create(cmd, # pylint: disable=too-many-locals network_profile=openshiftcluster.NetworkProfile( pod_cidr=pod_cidr or '10.128.0.0/14', service_cidr=service_cidr or '172.30.0.0/16', + software_defined_network=software_defined_network or 'OpenShiftSDN' ), master_profile=openshiftcluster.MasterProfile( vm_size=master_vm_size or 'Standard_D8s_v3', subnet_id=master_subnet, + encryption_at_host='Enabled' if master_encryption_at_host else 'Disabled', + disk_encryption_set_id=disk_encryption_set, ), worker_profiles=[ openshiftcluster.WorkerProfile( @@ -116,6 +124,8 @@ def aro_create(cmd, # pylint: disable=too-many-locals disk_size_gb=worker_vm_disk_size_gb or 128, subnet_id=worker_subnet, count=worker_count or 3, + encryption_at_host='Enabled' if worker_encryption_at_host else 'Disabled', + disk_encryption_set_id=disk_encryption_set, ) ], apiserver_profile=openshiftcluster.APIServerProfile( @@ -279,6 +289,13 @@ def get_network_resources(cli_ctx, subnets, vnet): return resources +def get_disk_encryption_resources(oc): + disk_encryption_set = oc.master_profile.disk_encryption_set_id + resources = set() + resources.add(disk_encryption_set) + return resources + + # cluster_application_update manages cluster application & service principal update # If called without parameters it should be best-effort # If called with parameters it fails if something is not possible @@ -363,8 +380,9 @@ def resolve_rp_client_id(): def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids): try: - # Get cluster resources we need to assign network contributor on - resources = get_cluster_network_resources(cli_ctx, oc) + # Get cluster resources we need to assign permissions on, sort to ensure the same order of operations + resources = {ROLE_NETWORK_CONTRIBUTOR: sorted(get_cluster_network_resources(cli_ctx, oc)), + ROLE_READER: sorted(get_disk_encryption_resources(oc))} except (CloudError, HttpOperationError) as e: if fail: logger.error(e.message) @@ -373,18 +391,19 @@ def ensure_resource_permissions(cli_ctx, oc, fail, sp_obj_ids): return for sp_id in sp_obj_ids: - for resource in sorted(resources): - # Create the role assignment if it doesn't exist - # Assume that the role assignment exists if we fail to look it up - resource_contributor_exists = True - - try: - resource_contributor_exists = has_network_contributor_on_resource(cli_ctx, resource, sp_id) - except CloudError as e: - if fail: - logger.error(e.message) - raise - logger.info(e.message) - - if not resource_contributor_exists: - assign_network_contributor_to_resource(cli_ctx, resource, sp_id) + for role in sorted(resources): + for resource in resources[role]: + # Create the role assignment if it doesn't exist + # Assume that the role assignment exists if we fail to look it up + resource_contributor_exists = True + + try: + resource_contributor_exists = has_role_assignment_on_resource(cli_ctx, resource, sp_id, role) + except CloudError as e: + if fail: + logger.error(e.message) + raise + logger.info(e.message) + + if not resource_contributor_exists: + assign_role_to_resource(cli_ctx, resource, sp_id, role)