From 5b63c3d556325952f52ea3b7b8e6335c912e62b3 Mon Sep 17 00:00:00 2001 From: Roland Kunkel Date: Mon, 1 Nov 2021 10:17:08 +0100 Subject: [PATCH 1/2] CI cluster creation with disk encryption + E2E tests --- deploy/cluster-development-predeploy.json | 185 ++++++++++++++++++ deploy/cluster-predeploy.json | 91 --------- hack/validate-imports/validate-imports.go | 2 + pkg/deploy/bindata.go | 20 +- pkg/deploy/generator/const.go | 2 +- .../generator/resources_disk_encryption.go | 109 +++++++++++ pkg/deploy/generator/templates_cluster.go | 6 + pkg/util/azureclient/mgmt/compute/disks.go | 3 + pkg/util/azureclient/mgmt/keyvault/vaults.go | 32 +++ pkg/util/cluster/cluster.go | 99 +++++++--- .../mocks/azureclient/mgmt/compute/compute.go | 15 ++ test/e2e/adminapi_resources.go | 5 + test/e2e/disk_encryption.go | 80 ++++++++ test/e2e/setup.go | 20 +- 14 files changed, 528 insertions(+), 141 deletions(-) create mode 100644 deploy/cluster-development-predeploy.json delete mode 100644 deploy/cluster-predeploy.json create mode 100644 pkg/deploy/generator/resources_disk_encryption.go create mode 100644 pkg/util/azureclient/mgmt/keyvault/vaults.go create mode 100644 test/e2e/disk_encryption.go diff --git a/deploy/cluster-development-predeploy.json b/deploy/cluster-development-predeploy.json new file mode 100644 index 00000000000..c6e9637e8bc --- /dev/null +++ b/deploy/cluster-development-predeploy.json @@ -0,0 +1,185 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ci": { + "type": "bool", + "defaultValue": false + }, + "clusterName": { + "type": "string" + }, + "clusterServicePrincipalId": { + "type": "string" + }, + "diskEncryptionSetName": { + "type": "string" + }, + "fpServicePrincipalId": { + "type": "string" + }, + "kvName": { + "type": "string" + }, + "masterAddressPrefix": { + "type": "string" + }, + "routes": { + "type": "array", + "defaultValue": [] + }, + "vnetAddressPrefix": { + "type": "string" + }, + "workerAddressPrefix": { + "type": "string" + } + }, + "resources": [ + { + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[parameters('vnetAddressPrefix')]" + ] + } + }, + "name": "dev-vnet", + "type": "Microsoft.Network/virtualNetworks", + "location": "[resourceGroup().location]", + "condition": "[parameters('ci')]", + "apiVersion": "2020-08-01" + }, + { + "properties": { + "routes": "[parameters('routes')]" + }, + "name": "[concat(parameters('clusterName'), '-rt')]", + "type": "Microsoft.Network/routeTables", + "location": "[resourceGroup().location]", + "apiVersion": "2020-08-01" + }, + { + "properties": { + "addressPrefix": "[parameters('masterAddressPrefix')]", + "routeTable": { + "id": "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]", + "tags": null + } + }, + "name": "[concat('dev-vnet/', parameters('clusterName'), '-master')]", + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2020-08-01", + "dependsOn": [ + "[resourceid('Microsoft.Network/virtualNetworks', 'dev-vnet')]", + "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]" + ], + "location": "[resourceGroup().location]" + }, + { + "properties": { + "addressPrefix": "[parameters('workerAddressPrefix')]", + "routeTable": { + "id": "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]", + "tags": null + } + }, + "name": "[concat('dev-vnet/', parameters('clusterName'), '-worker')]", + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2020-08-01", + "dependsOn": [ + "[resourceid('Microsoft.Network/virtualNetworks/subnets', 'dev-vnet', concat(parameters('clusterName'), '-master'))]", + "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]" + ], + "location": "[resourceGroup().location]" + }, + { + "name": "[parameters('kvName')]", + "type": "Microsoft.KeyVault/vaults", + "location": "[resourceGroup().location]", + "properties": { + "tenantId": "[subscription().tenantId]", + "sku": { + "family": "A", + "name": "standard" + }, + "accessPolicies": [ + { + "tenantId": "[subscription().tenantId]", + "objectId": "[parameters('clusterServicePrincipalId')]", + "permissions": { + "keys": [ + "create", + "get", + "delete" + ] + } + } + ], + "enabledForDiskEncryption": true, + "enablePurgeProtection": true + }, + "condition": "[parameters('ci')]", + "apiVersion": "2019-09-01" + }, + { + "properties": { + "kty": "RSA", + "keySize": 4096 + }, + "name": "[concat(parameters('kvName'), '/', concat(resourceGroup().name, '-', 'disk-encryption-key'))]", + "type": "Microsoft.KeyVault/vaults/keys", + "location": "[resourceGroup().location]", + "condition": "[parameters('ci')]", + "apiVersion": "2019-09-01", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]" + ] + }, + { + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "activeKey": { + "sourceVault": { + "id": "[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]" + }, + "keyUrl": "[reference(resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-', 'disk-encryption-key')), '2019-09-01', 'Full').properties.keyUriWithVersion]" + } + }, + "name": "[parameters('diskEncryptionSetName')]", + "type": "Microsoft.Compute/diskEncryptionSets", + "location": "[resourceGroup().location]", + "condition": "[parameters('ci')]", + "apiVersion": "2020-12-01", + "dependsOn": [ + "[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-', 'disk-encryption-key'))]" + ] + }, + { + "name": "[concat(parameters('kvName'), '/add')]", + "type": "Microsoft.KeyVault/vaults/accessPolicies", + "location": "[resourceGroup().location]", + "properties": { + "accessPolicies": [ + { + "tenantId": "[subscription().tenantId]", + "objectId": "[reference(resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName')), '2021-04-01', 'Full').identity.PrincipalId]", + "permissions": { + "keys": [ + "get", + "wrapKey", + "unwrapKey" + ] + } + } + ] + }, + "apiVersion": "2019-09-01", + "dependsOn": [ + "[resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName'))]" + ] + } + ] +} diff --git a/deploy/cluster-predeploy.json b/deploy/cluster-predeploy.json deleted file mode 100644 index 1e2ffbd4e8a..00000000000 --- a/deploy/cluster-predeploy.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "ci": { - "type": "bool", - "defaultValue": false - }, - "clusterName": { - "type": "string" - }, - "clusterServicePrincipalId": { - "type": "string" - }, - "fpServicePrincipalId": { - "type": "string" - }, - "masterAddressPrefix": { - "type": "string" - }, - "routes": { - "type": "array", - "defaultValue": [] - }, - "vnetAddressPrefix": { - "type": "string" - }, - "workerAddressPrefix": { - "type": "string" - } - }, - "resources": [ - { - "properties": { - "addressSpace": { - "addressPrefixes": [ - "[parameters('vnetAddressPrefix')]" - ] - } - }, - "name": "dev-vnet", - "type": "Microsoft.Network/virtualNetworks", - "location": "[resourceGroup().location]", - "condition": "[parameters('ci')]", - "apiVersion": "2020-08-01" - }, - { - "properties": { - "routes": "[parameters('routes')]" - }, - "name": "[concat(parameters('clusterName'), '-rt')]", - "type": "Microsoft.Network/routeTables", - "location": "[resourceGroup().location]", - "apiVersion": "2020-08-01" - }, - { - "properties": { - "addressPrefix": "[parameters('masterAddressPrefix')]", - "routeTable": { - "id": "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]", - "tags": null - } - }, - "name": "[concat('dev-vnet/', parameters('clusterName'), '-master')]", - "type": "Microsoft.Network/virtualNetworks/subnets", - "apiVersion": "2020-08-01", - "dependsOn": [ - "[resourceid('Microsoft.Network/virtualNetworks', 'dev-vnet')]", - "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]" - ], - "location": "[resourceGroup().location]" - }, - { - "properties": { - "addressPrefix": "[parameters('workerAddressPrefix')]", - "routeTable": { - "id": "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]", - "tags": null - } - }, - "name": "[concat('dev-vnet/', parameters('clusterName'), '-worker')]", - "type": "Microsoft.Network/virtualNetworks/subnets", - "apiVersion": "2020-08-01", - "dependsOn": [ - "[resourceid('Microsoft.Network/virtualNetworks/subnets', 'dev-vnet', concat(parameters('clusterName'), '-master'))]", - "[resourceid('Microsoft.Network/routeTables', concat(parameters('clusterName'), '-rt'))]" - ], - "location": "[resourceGroup().location]" - } - ] -} diff --git a/hack/validate-imports/validate-imports.go b/hack/validate-imports/validate-imports.go index e5b69f853b8..9b67d75be92 100644 --- a/hack/validate-imports/validate-imports.go +++ b/hack/validate-imports/validate-imports.go @@ -126,6 +126,8 @@ func acceptableNames(path string) []string { return []string{"", "utilnamespace"} case "github.com/Azure/ARO-RP/pkg/util/recover": return []string{"", "utilrecover"} + case "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/keyvault": + return []string{"", "keyvaultclient"} case "github.com/Azure/ARO-RP/test/database": return []string{"testdatabase"} case "github.com/Azure/ARO-RP/test/util/dynamichelper": diff --git a/pkg/deploy/bindata.go b/pkg/deploy/bindata.go index f7451aadb28..7428331af6b 100644 --- a/pkg/deploy/bindata.go +++ b/pkg/deploy/bindata.go @@ -1,6 +1,6 @@ // Code generated for package deploy by go-bindata DO NOT EDIT. (@generated) // sources: -// cluster-predeploy.json +// cluster-development-predeploy.json // databases-development.json // env-development.json // gateway-production-managed-identity.json @@ -96,22 +96,22 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _clusterPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x55\x4d\x8f\xda\x30\x10\xbd\xf3\x2b\xa2\xb4\x52\x40\xca\x17\x48\x95\x2a\x6e\x3d\x55\x3d\x74\xbb\xd2\xae\xf6\x82\x72\x18\x9c\x81\x75\xeb\xd8\x96\xed\xd0\xd2\x8a\xff\x5e\x25\x21\x81\x38\x09\xec\x22\x38\xb4\x5a\xed\x65\x63\xcf\x8c\xdf\xbc\x37\x6f\xf8\x33\x72\x1c\xc7\x71\xdf\x6b\xf2\x8c\x19\xb8\x73\xc7\x7d\x36\x46\xea\x79\x14\x55\x27\x61\x06\x1c\xd6\x98\x21\x37\x21\xfc\xce\x15\x86\x44\x64\xfb\x3b\x1d\xcd\xe2\xe9\x87\x20\x9e\x06\xf1\x34\x4a\x51\x32\xb1\x2d\xe2\x1e\x31\x93\x0c\x0c\x86\xdf\xb5\xe0\xef\x5c\xbf\x7a\x81\x08\x6e\x90\x9b\x27\x54\x9a\x0a\x5e\x3c\x34\x0d\xe3\xe2\xaf\x0e\x90\xa0\x20\x43\x83\x4a\xbb\x73\xa7\x82\x55\x25\xd2\xd6\x77\x79\x66\xb6\x12\x8b\x12\x4b\x21\xd8\x3e\xbf\xb9\x4b\x71\x05\x39\x33\x4f\xc0\xf2\x22\x66\x05\x4c\x63\x13\xb1\xf3\x8f\x0a\xb3\x5c\x1b\x54\x77\x90\xe1\xf0\x0b\xda\x28\xca\xd7\xee\xa9\x02\x0f\xa8\x36\x94\xe0\xbd\xa2\x9c\x50\x09\xec\x4b\x7a\x59\xb9\x95\xbc\x56\xa5\x0c\x0a\x5c\x9f\xd2\x54\xa1\xd6\xf7\x0a\x57\xf4\xd7\x65\x85\x94\xc8\x0d\xea\xe1\x5c\x50\x0a\xb6\x67\x04\x58\x24\xbd\xa5\x37\x1c\xcd\x15\x10\xfe\x14\xea\xc7\xe5\xad\x8e\x8e\xca\xb9\x0a\xb5\xc8\x15\x29\xfb\x5d\x34\x31\x56\x29\xa9\x84\x44\x65\x68\x0f\x2b\xe5\x3d\x54\x48\x1e\x24\x90\xee\x54\xd9\x51\x15\x5e\xeb\xc1\x4e\xf0\xe2\xe0\x8c\xb1\xd7\xa1\xcd\x9b\x24\x6e\x6f\x6e\xd2\x39\xdd\xb5\x4e\x76\x96\x6c\xbc\xf2\x81\x9b\xe2\x26\x28\x5e\xb1\x65\xad\x49\xfc\x4a\x89\x12\x5a\xac\x4c\x78\x87\xa6\xa0\x3f\xda\x50\x65\x72\x60\xfb\x4f\x6d\x27\x32\x41\xc0\xec\x3d\xbf\xa8\x49\xfe\xac\x44\x2e\xc7\x93\xb0\xbe\x4c\xec\x2c\x22\x78\x4a\x9b\xb4\x63\x0a\x08\x2d\x7a\xb6\xc2\x41\xd2\xa3\xd5\x32\x8b\x67\x71\x10\x7f\x0c\xe2\x69\xef\xd4\xbc\x52\xd3\xc6\x05\x6d\x1c\xd5\x71\x87\xff\x21\x5e\x17\x44\x70\x02\x66\xdc\x6a\xe5\xb0\x82\xbc\x89\xef\x78\x81\x32\x3d\xbd\x0d\x33\x5f\x42\x78\x84\x25\xc3\x2b\xb1\x7e\x4b\x1a\xc1\x32\x69\x9b\xcd\x9e\x9d\xd5\xa5\xe2\x20\x47\xd9\xf4\xb0\xc1\x68\xda\xea\x9b\xa6\x63\xef\x24\x7b\x9e\xef\xbc\x54\x9e\x5e\x50\x95\x4e\xb0\x2e\x5a\xe7\x39\x63\x17\x7a\xaf\x9e\x11\xaf\x36\x61\xe4\xf9\xce\x49\x44\x15\x6d\xaf\x1a\x1a\xcb\xae\x91\xce\x97\x1c\x4d\x67\x80\x06\x47\xa1\xb3\xee\x25\xf2\x54\x7f\xe3\xbd\x5b\xec\x9c\x06\x16\x18\xcf\x77\x9a\xde\x07\xe4\xbf\x95\xa8\xad\x97\x92\xcb\xdc\x74\x73\x97\xf4\xfc\xdc\xbd\xb9\xe4\xbc\x4b\x2a\xda\xfe\x1b\x97\xd4\x60\x5a\x6e\x79\x99\x36\xf5\xbe\xf8\x97\xbd\x55\xfe\x97\x8c\x76\xa3\xbf\x01\x00\x00\xff\xff\x9c\xba\xe7\x92\xbd\x0c\x00\x00") +var _clusterDevelopmentPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x58\x5d\x8f\xa3\x36\x14\x7d\x9f\x5f\x81\x68\x25\x12\x29\x7c\x24\xda\x56\x9d\x79\x1b\xb5\xdd\x6a\xb4\xea\x76\xd4\x6c\xa7\x0f\x51\x1e\x1c\x73\x93\xf1\xc6\xd8\xc8\x36\xd9\xb2\x55\xfe\x7b\x65\x08\x24\x80\x21\x2c\x93\xb4\xa3\x55\xb3\xd2\x4a\xc0\xf5\xf5\xfd\x3a\xc7\xc7\xf3\xf7\x8d\x65\x59\x96\xfd\xad\xc4\xcf\x10\x21\xfb\xce\xb2\x9f\x95\x8a\xe5\x9d\xef\xe7\x6f\xbc\x08\x31\xb4\x81\x08\x98\xf2\xd0\xe7\x44\x80\x87\x79\x74\xf8\x26\xfd\x59\x30\xfd\xce\x0d\xa6\x6e\x30\xf5\x43\x88\x29\x4f\xb5\xdd\x07\x88\x62\x8a\x14\x78\x1f\x25\x67\xdf\xd8\x93\x7c\x07\xcc\x99\x02\xa6\x9e\x40\x48\xc2\x99\xde\x68\xea\x05\xfa\x5f\x61\x10\x23\x81\x22\x50\x20\xa4\x7d\x67\xe5\x61\xe5\x0b\x49\xe5\x39\x7b\xa7\xd2\x18\xb4\x8b\x15\xe7\xf4\xb0\xbe\xfc\x16\xc2\x1a\x25\x54\x3d\x21\x9a\x68\x9b\x35\xa2\x12\x4a\x8b\xfd\xe4\xc4\x31\x4d\xa4\x02\xf1\x1e\x45\xd0\xbe\x83\x54\x82\xb0\x8d\xdd\xe5\x60\x0e\x62\x47\x30\x3c\x0a\xc2\x30\x89\x11\x7d\x08\x87\xb9\x0b\x89\xdc\xfe\xcc\xb0\x48\x63\x45\x38\x9b\x83\x1a\x1e\xd9\x3a\xbe\x54\x50\xdb\xdd\xf0\x28\x22\xa4\xcb\x73\x1f\x86\x02\xa4\x7c\x14\xb0\x26\x7f\x0d\x73\x24\x78\xa2\x40\xb6\xaf\x45\x42\xa0\xf4\xcc\x1c\x2c\x96\x46\xd7\x3b\x06\xea\x02\x11\x7e\xe2\x62\x3b\x3c\xd5\x9b\x13\x77\xb6\x00\xc9\x13\x81\xb3\x7c\x17\xa5\x4d\xcd\x55\x2c\x78\x0c\x42\x11\x43\x55\xb2\xef\x28\x8f\x64\x1e\x23\xdc\x6c\x5e\xdd\x2a\x8f\xb7\xb6\x61\xc3\x78\x71\x04\xe8\xc8\x69\x94\xcd\x19\x2f\x6d\xe3\xda\x65\xe3\xed\xbe\xf2\x66\x5f\x6b\x1b\xcb\xc7\xcd\x0e\x61\xe7\xea\x5d\xea\x6d\x2d\x8a\xf8\x2b\xc1\x82\x4b\xbe\x56\xde\x7b\x50\xba\xfc\xfe\x8e\x08\x95\x20\x7a\x78\x94\xf5\x85\x94\x63\xa4\x0e\xd4\xb3\x28\x8a\xfc\x8b\xe0\x49\x3c\x1a\x7b\xc5\xc7\x65\x7d\x15\xe6\x2c\x24\xe5\xb2\xd3\x12\x60\xa2\x73\xae\x99\xa3\x98\x9c\x30\xdc\x2c\x98\x05\x6e\xf0\x83\x1b\x4c\x8d\x53\xf3\x85\x3d\x2d\x51\x50\x8d\x23\x7f\xdd\xa8\x7f\x5b\x5d\x17\x98\x33\x8c\xd4\xa8\x92\xca\x91\x09\x9d\xf1\xc4\x72\x5c\xa1\x0c\xb9\xb5\x57\x3e\x0b\xe1\x03\x5a\x51\xb8\x50\xd5\xaf\x59\x46\x54\x03\x69\xb5\x9a\x06\xce\x6a\x96\xe2\xd8\x8e\x2c\xe9\x76\x80\x91\xb0\x92\x37\x09\x47\x4e\x67\xf5\x9c\x89\xd5\xb7\x3d\xc6\xa0\xf2\x3e\xa1\x8d\x4e\x9d\x25\x94\x0e\xc4\x5e\x31\x23\x4e\x01\x42\xdf\x99\x58\x9d\x11\xe5\x65\xfb\xa2\xa1\xa9\xc1\xd5\x97\xc9\x8a\x81\x6a\x0c\x50\xeb\x28\x34\xe8\x3e\x06\x16\xca\xdf\x98\x91\xc5\xce\xf5\xa0\x16\x8c\x33\xb1\xca\xdc\x5b\xda\x7f\xad\xa6\x56\x76\x5a\x0e\x43\xd3\xd5\x51\x62\x38\xee\xfe\x47\xc9\x79\x94\xe4\x65\xfb\x6a\x50\x52\x04\x53\x41\x4b\xbf\xde\x14\x7c\xf1\x55\x62\xab\x1c\x90\xd3\x98\x72\x21\xdd\xab\xf7\xef\x20\x7d\xd2\xd2\xd5\xdf\xe9\xff\x2f\x74\xa4\x9e\x03\xbc\x02\x86\x98\x7a\xc8\xa1\x28\x93\x95\xc4\x82\x64\x17\x91\xd1\xd8\x2b\xbe\x19\x9b\x25\xb7\x49\x3b\xb4\xd7\x28\x22\x34\xd5\x3e\xef\xdb\x80\x58\x54\x4b\x2a\xc4\x42\x24\xc2\xa6\x8c\xdc\x1b\x76\x45\x18\x6b\xea\xe1\x94\x60\xd2\xa1\x5c\xcd\x61\xbd\x28\xe1\xd2\x01\x5f\x7d\x04\x5c\x38\x30\x8c\x5f\xf3\x0a\x66\xe6\xc8\xd2\x61\x0c\x22\x22\x52\x83\xd8\xdc\xa3\x8a\xf1\x16\xd2\x6e\xc1\x5e\x5a\x62\x01\x48\x41\xc7\xc6\xa5\xe5\xa6\x21\xb7\x8d\x66\x21\x50\x50\x60\x96\xfb\xc5\xaf\x29\xfb\x8b\xdf\xde\xf8\xa5\xf9\x76\x69\xe8\x3a\x30\x8d\xfa\xf0\x2d\x17\x3f\x55\x2e\xcb\xf6\x9d\xa5\x44\x02\xad\x2b\x1e\x13\xb1\x81\x47\xc1\x15\xe0\x13\xf3\x4e\x96\x7f\xa1\xf0\x9f\xde\xba\xc1\xed\x85\x14\xeb\x56\x65\x08\xfa\x7d\x6e\xc2\x90\x1e\x84\x39\xf9\xac\x21\xf4\x26\xb8\xfd\x7e\xf0\x1d\xa0\xe0\xa7\x89\xe5\xf8\x47\x56\xad\xd3\x8b\xf6\xa0\x09\x35\xa3\x7c\x22\xb7\x2e\x94\x3d\x70\xb7\x90\x1a\x08\xfd\x3c\xbf\xf9\xd9\x28\xbf\x86\xdb\x5a\xd9\xb4\x81\xa7\xe6\x43\xe5\xb8\xaa\xa5\x59\x13\x07\x45\xbd\x1b\xc7\x52\x8f\x81\x21\x21\x30\x45\xb2\xa9\x30\x31\xf9\xa1\xe0\xf3\x54\x2a\x88\xee\xa5\x24\x1b\x06\x61\xf7\xed\xf0\xac\x1c\xc4\x8a\xec\xe0\x1d\x98\xb7\xcc\x4c\xf2\x12\x64\x19\x77\xb2\x57\x5d\xee\x5d\xa2\x68\x2d\x69\x95\x5b\x6e\x21\xfd\x43\xd0\xc3\xb6\x6b\x10\xc0\x30\x8c\x7a\x05\x90\x0d\x67\x4b\x14\x2f\x40\xc9\xc4\x72\x8e\xe3\xa6\xcd\xde\x26\x94\x3a\x63\xef\xd8\x07\x2f\x8b\x99\xfc\x49\xd4\xf3\x61\x46\x0d\x39\xf7\x14\xaa\xa7\xc1\x1b\xff\xca\xd8\x4b\x96\xfc\xc8\xa3\x38\x51\xe0\x37\x3c\xbc\x0e\xf0\xce\x02\x77\x3a\xbb\x16\x78\xaf\x34\x06\x03\xb0\xdf\x97\xc3\x51\x68\x90\x1b\x3d\xc8\xb8\x26\xac\xfe\x15\xed\xf9\xda\xc4\xdc\x39\x8e\x68\x47\x42\x6d\x44\x5a\xc0\x96\xc3\x7f\x36\x75\x83\x37\x55\xf8\x17\xc4\xee\x9d\x88\xc6\xff\x42\x32\xf6\x14\x82\x9f\x04\x8a\xf5\x99\xd0\xc3\x34\x61\x85\xf1\xb5\x65\x63\x27\x25\x5e\xf7\xc4\x7f\xf1\x5c\xb4\xf3\xc1\x4d\xfe\xbc\xbf\xf9\x27\x00\x00\xff\xff\x87\xc4\xa9\xe3\xb1\x1b\x00\x00") -func clusterPredeployJsonBytes() ([]byte, error) { +func clusterDevelopmentPredeployJsonBytes() ([]byte, error) { return bindataRead( - _clusterPredeployJson, - "cluster-predeploy.json", + _clusterDevelopmentPredeployJson, + "cluster-development-predeploy.json", ) } -func clusterPredeployJson() (*asset, error) { - bytes, err := clusterPredeployJsonBytes() +func clusterDevelopmentPredeployJson() (*asset, error) { + bytes, err := clusterDevelopmentPredeployJsonBytes() if err != nil { return nil, err } - info := bindataFileInfo{name: "cluster-predeploy.json", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + info := bindataFileInfo{name: "cluster-development-predeploy.json", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -548,7 +548,7 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "cluster-predeploy.json": clusterPredeployJson, + "cluster-development-predeploy.json": clusterDevelopmentPredeployJson, "databases-development.json": databasesDevelopmentJson, "env-development.json": envDevelopmentJson, "gateway-production-managed-identity.json": gatewayProductionManagedIdentityJson, @@ -611,7 +611,7 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "cluster-predeploy.json": {clusterPredeployJson, map[string]*bintree{}}, + "cluster-development-predeploy.json": {clusterDevelopmentPredeployJson, map[string]*bintree{}}, "databases-development.json": {databasesDevelopmentJson, map[string]*bintree{}}, "env-development.json": {envDevelopmentJson, map[string]*bintree{}}, "gateway-production-managed-identity.json": {gatewayProductionManagedIdentityJson, map[string]*bintree{}}, diff --git a/pkg/deploy/generator/const.go b/pkg/deploy/generator/const.go index 4127b139cef..80864cec72e 100644 --- a/pkg/deploy/generator/const.go +++ b/pkg/deploy/generator/const.go @@ -20,7 +20,7 @@ const ( FileGatewayProduction = "gateway-production.json" FileGatewayProductionParameters = "gateway-production-parameters.json" - FileClusterPredeploy = "cluster-predeploy.json" + FileClusterPredeploy = "cluster-development-predeploy.json" fileEnvDevelopment = "env-development.json" fileDatabaseDevelopment = "databases-development.json" fileRPDevelopmentPredeploy = "rp-development-predeploy.json" diff --git a/pkg/deploy/generator/resources_disk_encryption.go b/pkg/deploy/generator/resources_disk_encryption.go new file mode 100644 index 00000000000..83f95e4ffdc --- /dev/null +++ b/pkg/deploy/generator/resources_disk_encryption.go @@ -0,0 +1,109 @@ +package generator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "fmt" + + mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" + mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault" + "github.com/Azure/go-autorest/autorest/to" + + "github.com/Azure/ARO-RP/pkg/util/arm" + "github.com/Azure/ARO-RP/pkg/util/azureclient" +) + +func (g *generator) diskEncryptionKeyVault() *arm.Resource { + vaultResource := g.keyVault("[parameters('kvName')]", &[]mgmtkeyvault.AccessPolicyEntry{ + { + ObjectID: to.StringPtr("[parameters('clusterServicePrincipalId')]"), + Permissions: &mgmtkeyvault.Permissions{ + Keys: &[]mgmtkeyvault.KeyPermissions{mgmtkeyvault.KeyPermissionsCreate, mgmtkeyvault.KeyPermissionsGet, mgmtkeyvault.KeyPermissionsDelete}, + }, + // is later replaced by "[subscription().tenantId]" + TenantID: &tenantUUIDHack, + }, + }) + + vaultResource.Condition = to.StringPtr("[parameters('ci')]") + + return vaultResource +} + +const diskEncryptionKeyName string = "concat(resourceGroup().name, '-', 'disk-encryption-key')" + +func (g *generator) diskEncryptionKey() *arm.Resource { + key := &mgmtkeyvault.Key{ + KeyProperties: &mgmtkeyvault.KeyProperties{ + Kty: mgmtkeyvault.RSA, + KeySize: to.Int32Ptr(4096), + }, + + Name: to.StringPtr(fmt.Sprintf("[concat(parameters('kvName'), '/', %s)]", diskEncryptionKeyName)), + Type: to.StringPtr("Microsoft.KeyVault/vaults/keys"), + Location: to.StringPtr("[resourceGroup().location]"), + } + + return &arm.Resource{ + Resource: key, + APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), + DependsOn: []string{"[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"}, + Condition: to.StringPtr("[parameters('ci')]"), + } +} + +func (g *generator) diskEncryptionSet() *arm.Resource { + diskEncryptionSet := &mgmtcompute.DiskEncryptionSet{ + EncryptionSetProperties: &mgmtcompute.EncryptionSetProperties{ + ActiveKey: &mgmtcompute.KeyVaultAndKeyReference{ + KeyURL: to.StringPtr(fmt.Sprintf("[reference(resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s), '%s', 'Full').properties.keyUriWithVersion]", diskEncryptionKeyName, azureclient.APIVersion("Microsoft.KeyVault"))), + SourceVault: &mgmtcompute.SourceVault{ + ID: to.StringPtr("[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"), + }, + }, + }, + + Name: to.StringPtr("[parameters('diskEncryptionSetName')]"), + Type: to.StringPtr("Microsoft.Compute/diskEncryptionSets"), + Location: to.StringPtr("[resourceGroup().location]"), + Identity: &mgmtcompute.EncryptionSetIdentity{Type: mgmtcompute.SystemAssigned}, + } + + return &arm.Resource{ + Resource: diskEncryptionSet, + APIVersion: azureclient.APIVersion("Microsoft.Compute"), + Condition: to.StringPtr("[parameters('ci')]"), + DependsOn: []string{fmt.Sprintf("[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s)]", diskEncryptionKeyName)}, + } +} + +func (g *generator) diskEncryptionKeyVaultAccessPolicy() *arm.Resource { + accessPolicy := &mgmtkeyvault.VaultAccessPolicyParameters{ + Properties: &mgmtkeyvault.VaultAccessPolicyProperties{ + AccessPolicies: &[]mgmtkeyvault.AccessPolicyEntry{ + { + TenantID: &tenantUUIDHack, + ObjectID: to.StringPtr(fmt.Sprintf("[reference(resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName')), '%s', 'Full').identity.PrincipalId]", azureclient.APIVersion("Microsoft.Compute/diskEncryptionSets"))), + Permissions: &mgmtkeyvault.Permissions{ + Keys: &[]mgmtkeyvault.KeyPermissions{ + mgmtkeyvault.KeyPermissionsGet, + mgmtkeyvault.KeyPermissionsWrapKey, + mgmtkeyvault.KeyPermissionsUnwrapKey, + }, + }, + }, + }, + }, + + Name: to.StringPtr("[concat(parameters('kvName'), '/add')]"), + Type: to.StringPtr("Microsoft.KeyVault/vaults/accessPolicies"), + Location: to.StringPtr("[resourceGroup().location]"), + } + + return &arm.Resource{ + Resource: accessPolicy, + APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), + DependsOn: []string{"[resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName'))]"}, + } +} diff --git a/pkg/deploy/generator/templates_cluster.go b/pkg/deploy/generator/templates_cluster.go index b03275839f6..58bc67cbd4e 100644 --- a/pkg/deploy/generator/templates_cluster.go +++ b/pkg/deploy/generator/templates_cluster.go @@ -19,6 +19,8 @@ func (g *generator) clusterPredeploy() *arm.Template { "vnetAddressPrefix", "masterAddressPrefix", "workerAddressPrefix", + "diskEncryptionSetName", + "kvName", } for _, param := range params { @@ -39,6 +41,10 @@ func (g *generator) clusterPredeploy() *arm.Template { g.clusterRouteTable(), g.clusterMasterSubnet(), g.clusterWorkerSubnet(), + g.diskEncryptionKeyVault(), + g.diskEncryptionKey(), + g.diskEncryptionSet(), + g.diskEncryptionKeyVaultAccessPolicy(), ) return t diff --git a/pkg/util/azureclient/mgmt/compute/disks.go b/pkg/util/azureclient/mgmt/compute/disks.go index 31f95cab0f4..4f550adb790 100644 --- a/pkg/util/azureclient/mgmt/compute/disks.go +++ b/pkg/util/azureclient/mgmt/compute/disks.go @@ -4,6 +4,8 @@ package compute // Licensed under the Apache License 2.0. import ( + "context" + mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" "github.com/Azure/go-autorest/autorest" @@ -12,6 +14,7 @@ import ( // DisksClient is a minimal interface for azure DisksClient type DisksClient interface { + Get(ctx context.Context, resourceGroupName string, diskName string) (result mgmtcompute.Disk, err error) DisksClientAddons } diff --git a/pkg/util/azureclient/mgmt/keyvault/vaults.go b/pkg/util/azureclient/mgmt/keyvault/vaults.go new file mode 100644 index 00000000000..f7b9b9e7b41 --- /dev/null +++ b/pkg/util/azureclient/mgmt/keyvault/vaults.go @@ -0,0 +1,32 @@ +package keyvault + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault" + "github.com/Azure/go-autorest/autorest" +) + +// VaultsClient is a minimal interface for azure VaultsClient +type VaultsClient interface { + CheckNameAvailability(ctx context.Context, vaultName mgmtkeyvault.VaultCheckNameAvailabilityParameters) (result mgmtkeyvault.CheckNameAvailabilityResult, err error) +} + +type vaultsClient struct { + mgmtkeyvault.VaultsClient +} + +var _ VaultsClient = &vaultsClient{} + +// NewDisksClient creates a new DisksClient +func NewVaultsClient(subscriptionID string, authorizer autorest.Authorizer) VaultsClient { + client := mgmtkeyvault.NewVaultsClient(subscriptionID) + client.Authorizer = authorizer + + return &vaultsClient{ + VaultsClient: client, + } +} diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go index 6c84b10d62b..a315ebf46d1 100644 --- a/pkg/util/cluster/cluster.go +++ b/pkg/util/cluster/cluster.go @@ -15,6 +15,7 @@ import ( "strings" "time" + mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" mgmtauthorization "github.com/Azure/azure-sdk-for-go/services/preview/authorization/mgmt/2018-09-01-preview/authorization" mgmtfeatures "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features" @@ -37,12 +38,18 @@ import ( "github.com/Azure/ARO-RP/pkg/util/azureclient/graphrbac" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/authorization" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/features" + keyvaultclient "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/keyvault" "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/network" redhatopenshift20200430 "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2020-04-30/redhatopenshift" redhatopenshift20210901preview "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/redhatopenshift/2021-09-01-preview/redhatopenshift" "github.com/Azure/ARO-RP/pkg/util/rbac" ) +const ( + sharedKeyVaultNameSuffix string = "-e2eKV" + sharedDiskEncryptionSetNameSuffix string = "-disk-encryption-set" +) + type Cluster struct { log *logrus.Entry env env.Core @@ -59,6 +66,7 @@ type Cluster struct { routetables network.RouteTablesClient roleassignments authorization.RoleAssignmentsClient peerings network.VirtualNetworkPeeringsClient + vaultsClient keyvaultclient.VaultsClient } type errors []error @@ -74,7 +82,7 @@ func (errs errors) Error() string { return sb.String() } -func New(log *logrus.Entry, env env.Core, ci bool) (*Cluster, error) { +func New(log *logrus.Entry, environment env.Core, ci bool) (*Cluster, error) { if env.IsLocalDevelopmentMode() { for _, key := range []string{ "AZURE_FP_CLIENT_ID", @@ -90,27 +98,28 @@ func New(log *logrus.Entry, env env.Core, ci bool) (*Cluster, error) { return nil, err } - graphAuthorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(env.Environment().GraphEndpoint) + graphAuthorizer, err := auth.NewAuthorizerFromEnvironmentWithResource(environment.Environment().GraphEndpoint) if err != nil { return nil, err } c := &Cluster{ log: log, - env: env, + env: environment, ci: ci, - deployments: features.NewDeploymentsClient(env.Environment(), env.SubscriptionID(), authorizer), - groups: features.NewResourceGroupsClient(env.Environment(), env.SubscriptionID(), authorizer), - openshiftclustersv20200430: redhatopenshift20200430.NewOpenShiftClustersClient(env.Environment(), env.SubscriptionID(), authorizer), - openshiftclustersv20210901preview: redhatopenshift20210901preview.NewOpenShiftClustersClient(env.Environment(), env.SubscriptionID(), authorizer), - applications: graphrbac.NewApplicationsClient(env.Environment(), env.TenantID(), graphAuthorizer), - serviceprincipals: graphrbac.NewServicePrincipalClient(env.Environment(), env.TenantID(), graphAuthorizer), - securitygroups: network.NewSecurityGroupsClient(env.Environment(), env.SubscriptionID(), authorizer), - subnets: network.NewSubnetsClient(env.Environment(), env.SubscriptionID(), authorizer), - routetables: network.NewRouteTablesClient(env.Environment(), env.SubscriptionID(), authorizer), - roleassignments: authorization.NewRoleAssignmentsClient(env.Environment(), env.SubscriptionID(), authorizer), - peerings: network.NewVirtualNetworkPeeringsClient(env.Environment(), env.SubscriptionID(), authorizer), + deployments: features.NewDeploymentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + groups: features.NewResourceGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + openshiftclustersv20200430: redhatopenshift20200430.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), + openshiftclustersv20210901preview: redhatopenshift20210901preview.NewOpenShiftClustersClient(environment.Environment(), environment.SubscriptionID(), authorizer), + applications: graphrbac.NewApplicationsClient(environment.Environment(), environment.TenantID(), graphAuthorizer), + serviceprincipals: graphrbac.NewServicePrincipalClient(environment.Environment(), environment.TenantID(), graphAuthorizer), + securitygroups: network.NewSecurityGroupsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + subnets: network.NewSubnetsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + routetables: network.NewRouteTablesClient(environment.Environment(), environment.SubscriptionID(), authorizer), + roleassignments: authorization.NewRoleAssignmentsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + peerings: network.NewVirtualNetworkPeeringsClient(environment.Environment(), environment.SubscriptionID(), authorizer), + vaultsClient: keyvaultclient.NewVaultsClient(environment.SubscriptionID(), authorizer), } return c, nil @@ -126,7 +135,7 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str fpSPID := os.Getenv("AZURE_FP_SERVICE_PRINCIPAL_ID") if fpSPID == "" { - return fmt.Errorf("service principal id is not found") + return fmt.Errorf("fp service principal id is not found") } c.log.Infof("creating AAD application") @@ -172,6 +181,26 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str return err } + // use the shared one by default and overwrite, if we need to generate one because we are in CI + kvName := fmt.Sprintf("%s%s", vnetResourceGroup, sharedKeyVaultNameSuffix) + + diskEncryptionSetName := fmt.Sprintf("%s%s", vnetResourceGroup, sharedDiskEncryptionSetNameSuffix) + + if c.ci { + // name is limited to 24 characters, but must be globally unique, so we generate one and try if it is available + kvName = "kv-" + uuid.Must(uuid.NewV4()).String()[:21] + result, err := c.vaultsClient.CheckNameAvailability(ctx, mgmtkeyvault.VaultCheckNameAvailabilityParameters{Name: &kvName, Type: to.StringPtr("Microsoft.KeyVault/vaults")}) + if err != nil { + return err + } + + if result.NameAvailable != nil && !*result.NameAvailable { + return fmt.Errorf("Could not generate unique key vault name: %v", result.Reason) + } + + diskEncryptionSetName = clusterName + "-disk-encryption-set" + } + parameters := map[string]*arm.ParametersParameter{ "clusterName": {Value: clusterName}, "ci": {Value: c.ci}, @@ -180,6 +209,8 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str "vnetAddressPrefix": {Value: addressPrefix}, "masterAddressPrefix": {Value: masterSubnet}, "workerAddressPrefix": {Value: workerSubnet}, + "kvName": {Value: kvName}, + "diskEncryptionSetName": {Value: diskEncryptionSetName}, } // TODO: ick @@ -212,20 +243,23 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str return err } + diskEncryptionSetID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/diskEncryptionSets/%s", c.env.SubscriptionID(), vnetResourceGroup, diskEncryptionSetName) + c.log.Info("creating role assignments") - for _, scope := range []string{ - "/subscriptions/" + c.env.SubscriptionID() + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/dev-vnet", - "/subscriptions/" + c.env.SubscriptionID() + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/routeTables/" + clusterName + "-rt", + for _, scope := range []struct{ resource, role string }{ + {"/subscriptions/" + c.env.SubscriptionID() + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/dev-vnet", rbac.RoleNetworkContributor}, + {"/subscriptions/" + c.env.SubscriptionID() + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/routeTables/" + clusterName + "-rt", rbac.RoleNetworkContributor}, + {diskEncryptionSetID, rbac.RoleReader}, } { for _, principalID := range []string{spID, fpSPID} { for i := 0; i < 5; i++ { _, err = c.roleassignments.Create( ctx, - scope, + scope.resource, uuid.Must(uuid.NewV4()).String(), mgmtauthorization.RoleAssignmentCreateParameters{ RoleAssignmentProperties: &mgmtauthorization.RoleAssignmentProperties{ - RoleDefinitionID: to.StringPtr("/subscriptions/" + c.env.SubscriptionID() + "/providers/Microsoft.Authorization/roleDefinitions/" + rbac.RoleNetworkContributor), + RoleDefinitionID: to.StringPtr("/subscriptions/" + c.env.SubscriptionID() + "/providers/Microsoft.Authorization/roleDefinitions/" + scope.role), PrincipalID: &principalID, PrincipalType: mgmtauthorization.ServicePrincipal, }, @@ -256,7 +290,8 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str } c.log.Info("creating cluster") - err = c.createCluster(ctx, vnetResourceGroup, clusterName, appID, appSecret, visibility) + err = c.createCluster(ctx, vnetResourceGroup, clusterName, appID, appSecret, diskEncryptionSetID, visibility) + if err != nil { return err } @@ -357,7 +392,7 @@ func (c *Cluster) Delete(ctx context.Context, vnetResourceGroup, clusterName str // createCluster created new clusters, based on where it is running. // development - using preview api // production - using stable GA api -func (c *Cluster) createCluster(ctx context.Context, vnetResourceGroup, clusterName, clientID, clientSecret string, visibility api.Visibility) error { +func (c *Cluster) createCluster(ctx context.Context, vnetResourceGroup, clusterName, clientID, clientSecret, diskEncryptionSetID string, visibility api.Visibility) error { // using internal representation for "singe source" of options oc := api.OpenShiftCluster{ Properties: api.OpenShiftClusterProperties{ @@ -375,18 +410,20 @@ func (c *Cluster) createCluster(ctx context.Context, vnetResourceGroup, clusterN SoftwareDefinedNetwork: api.SoftwareDefinedNetworkOpenShiftSDN, }, MasterProfile: api.MasterProfile{ - VMSize: api.VMSizeStandardD8sV3, - EncryptionAtHost: api.EncryptionAtHostDisabled, - SubnetID: fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet/subnets/%s-master", c.env.SubscriptionID(), vnetResourceGroup, clusterName), + VMSize: api.VMSizeStandardD8sV3, + SubnetID: fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet/subnets/%s-master", c.env.SubscriptionID(), vnetResourceGroup, clusterName), + EncryptionAtHost: api.EncryptionAtHostEnabled, + DiskEncryptionSetID: diskEncryptionSetID, }, WorkerProfiles: []api.WorkerProfile{ { - Name: "worker", - VMSize: api.VMSizeStandardD4sV3, - EncryptionAtHost: api.EncryptionAtHostDisabled, - DiskSizeGB: 128, - SubnetID: fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet/subnets/%s-worker", c.env.SubscriptionID(), vnetResourceGroup, clusterName), - Count: 3, + Name: "worker", + VMSize: api.VMSizeStandardD4sV3, + DiskSizeGB: 128, + SubnetID: fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/dev-vnet/subnets/%s-worker", c.env.SubscriptionID(), vnetResourceGroup, clusterName), + Count: 3, + EncryptionAtHost: api.EncryptionAtHostEnabled, + DiskEncryptionSetID: diskEncryptionSetID, }, }, APIServerProfile: api.APIServerProfile{ diff --git a/pkg/util/mocks/azureclient/mgmt/compute/compute.go b/pkg/util/mocks/azureclient/mgmt/compute/compute.go index 3f2be02958b..f06be48d1f8 100644 --- a/pkg/util/mocks/azureclient/mgmt/compute/compute.go +++ b/pkg/util/mocks/azureclient/mgmt/compute/compute.go @@ -49,6 +49,21 @@ func (mr *MockDisksClientMockRecorder) DeleteAndWait(arg0, arg1, arg2 interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAndWait", reflect.TypeOf((*MockDisksClient)(nil).DeleteAndWait), arg0, arg1, arg2) } +// Get mocks base method. +func (m *MockDisksClient) Get(arg0 context.Context, arg1, arg2 string) (compute.Disk, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret0, _ := ret[0].(compute.Disk) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDisksClientMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDisksClient)(nil).Get), arg0, arg1, arg2) +} + // MockResourceSkusClient is a mock of ResourceSkusClient interface. type MockResourceSkusClient struct { ctrl *gomock.Controller diff --git a/test/e2e/adminapi_resources.go b/test/e2e/adminapi_resources.go index 7d906ed7c6e..08aaaa94584 100644 --- a/test/e2e/adminapi_resources.go +++ b/test/e2e/adminapi_resources.go @@ -5,6 +5,7 @@ package e2e import ( "context" + "fmt" "net/http" "strings" @@ -58,6 +59,10 @@ var _ = Describe("[Admin API] List Azure resources action", func() { vnet, err := clients.VirtualNetworks.Get(ctx, r.ResourceGroup, r.ResourceName, "") Expect(err).NotTo(HaveOccurred()) + diskEncryptionSet, err := clients.DiskEncryptionSets.Get(ctx, vnetResourceGroup, fmt.Sprintf("%s-disk-encryption-set", vnetResourceGroup)) + Expect(err).NotTo(HaveOccurred()) + expectedResourceIDs = append(expectedResourceIDs, strings.ToLower(*diskEncryptionSet.ID)) + for _, subnet := range *vnet.Subnets { if _, ok := subnets[strings.ToLower(*subnet.ID)]; !ok { continue diff --git a/test/e2e/disk_encryption.go b/test/e2e/disk_encryption.go new file mode 100644 index 00000000000..b2a6ab4e3eb --- /dev/null +++ b/test/e2e/disk_encryption.go @@ -0,0 +1,80 @@ +package e2e + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/util/stringutils" +) + +var _ = Describe("Encryption at host should be enabled", func() { + BeforeEach(skipIfNotInDevelopmentEnv) + + Specify("each VM should have encryption at host enabled", func() { + ctx := context.Background() + + By("getting the resource group where the VM instances live in") + oc, err := clients.OpenshiftClustersv20200430.Get(ctx, vnetResourceGroup, clusterName) + Expect(err).NotTo(HaveOccurred()) + clusterResourceGroup := stringutils.LastTokenByte(*oc.OpenShiftClusterProperties.ClusterProfile.ResourceGroupID, '/') + + By("listing all VMs") + vms, err := clients.VirtualMachines.List(ctx, clusterResourceGroup) + Expect(err).NotTo(HaveOccurred()) + Expect(vms).NotTo(HaveLen(0)) + + By("checking the encryption property on each VM") + for _, vm := range vms { + log.Println(vm.Name) + Expect(vm.SecurityProfile).To(Not(BeNil())) + Expect(vm.SecurityProfile.EncryptionAtHost).To(Not(BeNil())) + Expect(*vm.SecurityProfile.EncryptionAtHost).To(Equal(true)) + } + + }) +}) + +var _ = Describe("Disk encryption at rest should be enabled with customer managed key", func() { + BeforeEach(skipIfNotInDevelopmentEnv) + + // We have to get the disks by VM, because when getting all disks by resource group, we do not get recently created disks, see https://github.com/Azure/azure-cli/issues/17123 + Specify("each disk should have encryption at rest with customer managed key enabled", func() { + ctx := context.Background() + + By("getting the resource group where the VM instances live in") + oc, err := clients.OpenshiftClustersv20200430.Get(ctx, vnetResourceGroup, clusterName) + Expect(err).NotTo(HaveOccurred()) + clusterResourceGroup := stringutils.LastTokenByte(*oc.OpenShiftClusterProperties.ClusterProfile.ResourceGroupID, '/') + + By("listing all VMs") + vms, err := clients.VirtualMachines.List(ctx, clusterResourceGroup) + Expect(err).NotTo(HaveOccurred()) + Expect(vms).NotTo(HaveLen(0)) + + By("checking the encryption property on each OS disk of each VM") + for _, vm := range vms { + osDisk, err := clients.Disks.Get(ctx, clusterResourceGroup, *vm.StorageProfile.OsDisk.Name) + Expect(err).NotTo(HaveOccurred()) + Expect(osDisk.Encryption.Type).To(Equal(mgmtcompute.EncryptionAtRestWithCustomerKey)) + } + + By("checking if the encrypted storage class is default") + sc, err := clients.Kubernetes.StorageV1().StorageClasses().Get(ctx, "managed-premium-encrypted-cmk", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(sc).NotTo(BeNil()) + + Expect(sc.Annotations).NotTo(BeNil()) + Expect(sc.Annotations["storageclass.kubernetes.io/is-default-class"]).To(Equal("true")) + + Expect(sc.Parameters).NotTo(BeNil()) + Expect(sc.Parameters["diskEncryptionSetID"]).NotTo(BeEmpty()) + }) +}) diff --git a/test/e2e/setup.go b/test/e2e/setup.go index cd4510bbcbc..1d2637fb2a7 100644 --- a/test/e2e/setup.go +++ b/test/e2e/setup.go @@ -41,10 +41,12 @@ type clientSet struct { OpenshiftClustersv20210901preview redhatopenshift20210901preview.OpenShiftClustersClient Operationsv20210901preview redhatopenshift20210901preview.OperationsClient - VirtualMachines compute.VirtualMachinesClient - Resources features.ResourcesClient - ActivityLogs insights.ActivityLogsClient - VirtualNetworks network.VirtualNetworksClient + VirtualMachines compute.VirtualMachinesClient + Resources features.ResourcesClient + ActivityLogs insights.ActivityLogsClient + VirtualNetworks network.VirtualNetworksClient + DiskEncryptionSets compute.DiskEncryptionSetsClient + Disks compute.DisksClient RestConfig *rest.Config Kubernetes kubernetes.Interface @@ -129,10 +131,12 @@ func newClientSet(ctx context.Context) (*clientSet, error) { OpenshiftClustersv20210901preview: redhatopenshift20210901preview.NewOpenShiftClustersClient(_env.Environment(), _env.SubscriptionID(), authorizer), Operationsv20210901preview: redhatopenshift20210901preview.NewOperationsClient(_env.Environment(), _env.SubscriptionID(), authorizer), - VirtualMachines: compute.NewVirtualMachinesClient(_env.Environment(), _env.SubscriptionID(), authorizer), - Resources: features.NewResourcesClient(_env.Environment(), _env.SubscriptionID(), authorizer), - ActivityLogs: insights.NewActivityLogsClient(_env.Environment(), _env.SubscriptionID(), authorizer), - VirtualNetworks: network.NewVirtualNetworksClient(_env.Environment(), _env.SubscriptionID(), authorizer), + VirtualMachines: compute.NewVirtualMachinesClient(_env.Environment(), _env.SubscriptionID(), authorizer), + Resources: features.NewResourcesClient(_env.Environment(), _env.SubscriptionID(), authorizer), + ActivityLogs: insights.NewActivityLogsClient(_env.Environment(), _env.SubscriptionID(), authorizer), + VirtualNetworks: network.NewVirtualNetworksClient(_env.Environment(), _env.SubscriptionID(), authorizer), + Disks: compute.NewDisksClient(_env.Environment(), _env.SubscriptionID(), authorizer), + DiskEncryptionSets: compute.NewDiskEncryptionSetsClient(_env.Environment(), _env.SubscriptionID(), authorizer), RestConfig: restconfig, Kubernetes: cli, From 8499c9726e26853a523e80c7795f00d85a5d4587 Mon Sep 17 00:00:00 2001 From: Roland Kunkel Date: Tue, 9 Nov 2021 17:30:21 +0100 Subject: [PATCH 2/2] review comments --- deploy/cluster-development-predeploy.json | 30 ++--- pkg/deploy/bindata.go | 2 +- pkg/deploy/generator/resources.go | 4 +- pkg/deploy/generator/resources_cluster.go | 91 +++++++++++++++ pkg/deploy/generator/resources_dev.go | 9 +- .../generator/resources_disk_encryption.go | 109 ------------------ pkg/deploy/generator/templates_cluster.go | 1 - pkg/util/cluster/cluster.go | 28 ++--- test/e2e/disk_encryption.go | 8 +- 9 files changed, 126 insertions(+), 156 deletions(-) delete mode 100644 pkg/deploy/generator/resources_disk_encryption.go diff --git a/deploy/cluster-development-predeploy.json b/deploy/cluster-development-predeploy.json index c6e9637e8bc..71c04a049ea 100644 --- a/deploy/cluster-development-predeploy.json +++ b/deploy/cluster-development-predeploy.json @@ -12,9 +12,6 @@ "clusterServicePrincipalId": { "type": "string" }, - "diskEncryptionSetName": { - "type": "string" - }, "fpServicePrincipalId": { "type": "string" }, @@ -103,19 +100,7 @@ "family": "A", "name": "standard" }, - "accessPolicies": [ - { - "tenantId": "[subscription().tenantId]", - "objectId": "[parameters('clusterServicePrincipalId')]", - "permissions": { - "keys": [ - "create", - "get", - "delete" - ] - } - } - ], + "accessPolicies": [], "enabledForDiskEncryption": true, "enablePurgeProtection": true }, @@ -127,7 +112,7 @@ "kty": "RSA", "keySize": 4096 }, - "name": "[concat(parameters('kvName'), '/', concat(resourceGroup().name, '-', 'disk-encryption-key'))]", + "name": "[concat(parameters('kvName'), '/', concat(resourceGroup().name, '-disk-encryption-key'))]", "type": "Microsoft.KeyVault/vaults/keys", "location": "[resourceGroup().location]", "condition": "[parameters('ci')]", @@ -145,16 +130,16 @@ "sourceVault": { "id": "[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]" }, - "keyUrl": "[reference(resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-', 'disk-encryption-key')), '2019-09-01', 'Full').properties.keyUriWithVersion]" + "keyUrl": "[reference(resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-disk-encryption-key')), '2019-09-01', 'Full').properties.keyUriWithVersion]" } }, - "name": "[parameters('diskEncryptionSetName')]", + "name": "[concat(resourceGroup().name, '-disk-encryption-set')]", "type": "Microsoft.Compute/diskEncryptionSets", "location": "[resourceGroup().location]", "condition": "[parameters('ci')]", "apiVersion": "2020-12-01", "dependsOn": [ - "[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-', 'disk-encryption-key'))]" + "[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), concat(resourceGroup().name, '-disk-encryption-key'))]" ] }, { @@ -165,7 +150,7 @@ "accessPolicies": [ { "tenantId": "[subscription().tenantId]", - "objectId": "[reference(resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName')), '2021-04-01', 'Full').identity.PrincipalId]", + "objectId": "[reference(resourceId('Microsoft.Compute/diskEncryptionSets', concat(resourceGroup().name, '-disk-encryption-set')), '2021-04-01', 'Full').identity.PrincipalId]", "permissions": { "keys": [ "get", @@ -176,9 +161,10 @@ } ] }, + "condition": "[parameters('ci')]", "apiVersion": "2019-09-01", "dependsOn": [ - "[resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName'))]" + "[resourceId('Microsoft.Compute/diskEncryptionSets', concat(resourceGroup().name, '-disk-encryption-set'))]" ] } ] diff --git a/pkg/deploy/bindata.go b/pkg/deploy/bindata.go index 7428331af6b..fbe4e73d8c3 100644 --- a/pkg/deploy/bindata.go +++ b/pkg/deploy/bindata.go @@ -96,7 +96,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _clusterDevelopmentPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x58\x5d\x8f\xa3\x36\x14\x7d\x9f\x5f\x81\x68\x25\x12\x29\x7c\x24\xda\x56\x9d\x79\x1b\xb5\xdd\x6a\xb4\xea\x76\xd4\x6c\xa7\x0f\x51\x1e\x1c\x73\x93\xf1\xc6\xd8\xc8\x36\xd9\xb2\x55\xfe\x7b\x65\x08\x24\x80\x21\x2c\x93\xb4\xa3\x55\xb3\xd2\x4a\xc0\xf5\xf5\xfd\x3a\xc7\xc7\xf3\xf7\x8d\x65\x59\x96\xfd\xad\xc4\xcf\x10\x21\xfb\xce\xb2\x9f\x95\x8a\xe5\x9d\xef\xe7\x6f\xbc\x08\x31\xb4\x81\x08\x98\xf2\xd0\xe7\x44\x80\x87\x79\x74\xf8\x26\xfd\x59\x30\xfd\xce\x0d\xa6\x6e\x30\xf5\x43\x88\x29\x4f\xb5\xdd\x07\x88\x62\x8a\x14\x78\x1f\x25\x67\xdf\xd8\x93\x7c\x07\xcc\x99\x02\xa6\x9e\x40\x48\xc2\x99\xde\x68\xea\x05\xfa\x5f\x61\x10\x23\x81\x22\x50\x20\xa4\x7d\x67\xe5\x61\xe5\x0b\x49\xe5\x39\x7b\xa7\xd2\x18\xb4\x8b\x15\xe7\xf4\xb0\xbe\xfc\x16\xc2\x1a\x25\x54\x3d\x21\x9a\x68\x9b\x35\xa2\x12\x4a\x8b\xfd\xe4\xc4\x31\x4d\xa4\x02\xf1\x1e\x45\xd0\xbe\x83\x54\x82\xb0\x8d\xdd\xe5\x60\x0e\x62\x47\x30\x3c\x0a\xc2\x30\x89\x11\x7d\x08\x87\xb9\x0b\x89\xdc\xfe\xcc\xb0\x48\x63\x45\x38\x9b\x83\x1a\x1e\xd9\x3a\xbe\x54\x50\xdb\xdd\xf0\x28\x22\xa4\xcb\x73\x1f\x86\x02\xa4\x7c\x14\xb0\x26\x7f\x0d\x73\x24\x78\xa2\x40\xb6\xaf\x45\x42\xa0\xf4\xcc\x1c\x2c\x96\x46\xd7\x3b\x06\xea\x02\x11\x7e\xe2\x62\x3b\x3c\xd5\x9b\x13\x77\xb6\x00\xc9\x13\x81\xb3\x7c\x17\xa5\x4d\xcd\x55\x2c\x78\x0c\x42\x11\x43\x55\xb2\xef\x28\x8f\x64\x1e\x23\xdc\x6c\x5e\xdd\x2a\x8f\xb7\xb6\x61\xc3\x78\x71\x04\xe8\xc8\x69\x94\xcd\x19\x2f\x6d\xe3\xda\x65\xe3\xed\xbe\xf2\x66\x5f\x6b\x1b\xcb\xc7\xcd\x0e\x61\xe7\xea\x5d\xea\x6d\x2d\x8a\xf8\x2b\xc1\x82\x4b\xbe\x56\xde\x7b\x50\xba\xfc\xfe\x8e\x08\x95\x20\x7a\x78\x94\xf5\x85\x94\x63\xa4\x0e\xd4\xb3\x28\x8a\xfc\x8b\xe0\x49\x3c\x1a\x7b\xc5\xc7\x65\x7d\x15\xe6\x2c\x24\xe5\xb2\xd3\x12\x60\xa2\x73\xae\x99\xa3\x98\x9c\x30\xdc\x2c\x98\x05\x6e\xf0\x83\x1b\x4c\x8d\x53\xf3\x85\x3d\x2d\x51\x50\x8d\x23\x7f\xdd\xa8\x7f\x5b\x5d\x17\x98\x33\x8c\xd4\xa8\x92\xca\x91\x09\x9d\xf1\xc4\x72\x5c\xa1\x0c\xb9\xb5\x57\x3e\x0b\xe1\x03\x5a\x51\xb8\x50\xd5\xaf\x59\x46\x54\x03\x69\xb5\x9a\x06\xce\x6a\x96\xe2\xd8\x8e\x2c\xe9\x76\x80\x91\xb0\x92\x37\x09\x47\x4e\x67\xf5\x9c\x89\xd5\xb7\x3d\xc6\xa0\xf2\x3e\xa1\x8d\x4e\x9d\x25\x94\x0e\xc4\x5e\x31\x23\x4e\x01\x42\xdf\x99\x58\x9d\x11\xe5\x65\xfb\xa2\xa1\xa9\xc1\xd5\x97\xc9\x8a\x81\x6a\x0c\x50\xeb\x28\x34\xe8\x3e\x06\x16\xca\xdf\x98\x91\xc5\xce\xf5\xa0\x16\x8c\x33\xb1\xca\xdc\x5b\xda\x7f\xad\xa6\x56\x76\x5a\x0e\x43\xd3\xd5\x51\x62\x38\xee\xfe\x47\xc9\x79\x94\xe4\x65\xfb\x6a\x50\x52\x04\x53\x41\x4b\xbf\xde\x14\x7c\xf1\x55\x62\xab\x1c\x90\xd3\x98\x72\x21\xdd\xab\xf7\xef\x20\x7d\xd2\xd2\xd5\xdf\xe9\xff\x2f\x74\xa4\x9e\x03\xbc\x02\x86\x98\x7a\xc8\xa1\x28\x93\x95\xc4\x82\x64\x17\x91\xd1\xd8\x2b\xbe\x19\x9b\x25\xb7\x49\x3b\xb4\xd7\x28\x22\x34\xd5\x3e\xef\xdb\x80\x58\x54\x4b\x2a\xc4\x42\x24\xc2\xa6\x8c\xdc\x1b\x76\x45\x18\x6b\xea\xe1\x94\x60\xd2\xa1\x5c\xcd\x61\xbd\x28\xe1\xd2\x01\x5f\x7d\x04\x5c\x38\x30\x8c\x5f\xf3\x0a\x66\xe6\xc8\xd2\x61\x0c\x22\x22\x52\x83\xd8\xdc\xa3\x8a\xf1\x16\xd2\x6e\xc1\x5e\x5a\x62\x01\x48\x41\xc7\xc6\xa5\xe5\xa6\x21\xb7\x8d\x66\x21\x50\x50\x60\x96\xfb\xc5\xaf\x29\xfb\x8b\xdf\xde\xf8\xa5\xf9\x76\x69\xe8\x3a\x30\x8d\xfa\xf0\x2d\x17\x3f\x55\x2e\xcb\xf6\x9d\xa5\x44\x02\xad\x2b\x1e\x13\xb1\x81\x47\xc1\x15\xe0\x13\xf3\x4e\x96\x7f\xa1\xf0\x9f\xde\xba\xc1\xed\x85\x14\xeb\x56\x65\x08\xfa\x7d\x6e\xc2\x90\x1e\x84\x39\xf9\xac\x21\xf4\x26\xb8\xfd\x7e\xf0\x1d\xa0\xe0\xa7\x89\xe5\xf8\x47\x56\xad\xd3\x8b\xf6\xa0\x09\x35\xa3\x7c\x22\xb7\x2e\x94\x3d\x70\xb7\x90\x1a\x08\xfd\x3c\xbf\xf9\xd9\x28\xbf\x86\xdb\x5a\xd9\xb4\x81\xa7\xe6\x43\xe5\xb8\xaa\xa5\x59\x13\x07\x45\xbd\x1b\xc7\x52\x8f\x81\x21\x21\x30\x45\xb2\xa9\x30\x31\xf9\xa1\xe0\xf3\x54\x2a\x88\xee\xa5\x24\x1b\x06\x61\xf7\xed\xf0\xac\x1c\xc4\x8a\xec\xe0\x1d\x98\xb7\xcc\x4c\xf2\x12\x64\x19\x77\xb2\x57\x5d\xee\x5d\xa2\x68\x2d\x69\x95\x5b\x6e\x21\xfd\x43\xd0\xc3\xb6\x6b\x10\xc0\x30\x8c\x7a\x05\x90\x0d\x67\x4b\x14\x2f\x40\xc9\xc4\x72\x8e\xe3\xa6\xcd\xde\x26\x94\x3a\x63\xef\xd8\x07\x2f\x8b\x99\xfc\x49\xd4\xf3\x61\x46\x0d\x39\xf7\x14\xaa\xa7\xc1\x1b\xff\xca\xd8\x4b\x96\xfc\xc8\xa3\x38\x51\xe0\x37\x3c\xbc\x0e\xf0\xce\x02\x77\x3a\xbb\x16\x78\xaf\x34\x06\x03\xb0\xdf\x97\xc3\x51\x68\x90\x1b\x3d\xc8\xb8\x26\xac\xfe\x15\xed\xf9\xda\xc4\xdc\x39\x8e\x68\x47\x42\x6d\x44\x5a\xc0\x96\xc3\x7f\x36\x75\x83\x37\x55\xf8\x17\xc4\xee\x9d\x88\xc6\xff\x42\x32\xf6\x14\x82\x9f\x04\x8a\xf5\x99\xd0\xc3\x34\x61\x85\xf1\xb5\x65\x63\x27\x25\x5e\xf7\xc4\x7f\xf1\x5c\xb4\xf3\xc1\x4d\xfe\xbc\xbf\xf9\x27\x00\x00\xff\xff\x87\xc4\xa9\xe3\xb1\x1b\x00\x00") +var _clusterDevelopmentPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x58\xdf\x8f\xa3\x36\x10\x7e\xdf\xbf\x02\xd1\x4a\x24\x52\xf8\x91\xe8\x5a\x75\xf7\x6d\xd5\xf6\xaa\xd5\xa9\xd7\x55\x73\xdd\x3e\x44\x79\x70\xec\x49\xd6\x07\xd8\xc8\x36\xb9\x72\x55\xfe\xf7\xca\x10\x48\x00\x43\xb2\x34\xb9\x56\xab\xbb\x93\x56\xc2\x8c\xc7\x33\xdf\xcc\xf7\x79\xc2\xdf\x37\x96\x65\x59\xf6\xb7\x12\x3f\x43\x8c\xec\x3b\xcb\x7e\x56\x2a\x91\x77\xbe\x5f\xac\x78\x31\x62\x68\x03\x31\x30\xe5\xa1\xcf\xa9\x00\x0f\xf3\x78\xff\x4e\xfa\xb3\x60\xfa\x9d\x1b\x4c\xdd\x60\xea\x13\x48\x22\x9e\x69\xbb\x0f\x10\x27\x11\x52\xe0\x7d\x94\x9c\x7d\x63\x4f\x8a\x13\x30\x67\x0a\x98\x7a\x02\x21\x29\x67\xfa\xa0\xa9\x17\xe8\xff\xa5\x41\x82\x04\x8a\x41\x81\x90\xf6\x9d\x55\x84\x55\x6c\xa4\xb5\xe7\x7c\x4d\x65\x09\x68\x17\x2b\xce\xa3\xfd\xfe\xea\x1d\x81\x35\x4a\x23\xf5\x84\xa2\x54\xdb\xac\x51\x24\xa1\xb2\xd8\x4d\x8e\x1c\x47\xa9\x54\x20\xde\xa3\x18\xba\x4f\x90\x4a\x50\xb6\xb1\xfb\x1c\xcc\x41\x6c\x29\x86\x47\x41\x19\xa6\x09\x8a\x1e\xc8\x30\x77\xeb\xe4\x52\x9e\xc2\xed\xf0\xa4\x62\xa4\x73\xba\x27\x44\x80\x94\x8f\x02\xd6\xf4\xaf\x61\x8e\x04\x4f\x15\xc8\xee\xbd\x48\x08\x94\x9d\x28\xde\x62\x69\x74\xbd\x65\xa0\x2e\x10\xe1\x27\x2e\xc2\xe1\xa9\xde\x1c\xb9\xb3\x05\x48\x9e\x0a\x9c\xe7\xbb\xa8\x6c\x1a\xae\x12\xc1\x13\x10\x8a\x1a\x50\xc9\xdf\xa3\x22\x92\x79\x82\x70\xbb\x78\x4d\xab\x22\xde\xc6\x81\x2d\xe3\xc5\x81\x55\x23\xa7\x05\x9b\x33\x5e\xda\xc6\xbd\xcb\xd6\xea\xae\xb6\xb2\x6b\x94\x8d\x15\xed\x66\x13\xd8\xba\xfa\x94\x66\x59\x4b\x10\x7f\xa5\x58\x70\xc9\xd7\xca\x7b\x0f\x4a\xc3\xef\x6f\xa9\x50\x29\x8a\xf6\x8f\xb2\xb9\x31\xe2\x18\xa9\xbd\x5e\x2c\x4a\x90\x7f\x11\x3c\x4d\x46\x63\xaf\x7c\xb9\x6c\xee\xc2\x9c\x11\x5a\x6d\x3b\x86\x00\x53\x9d\x73\xc3\x1c\x25\xf4\x48\x96\x66\xc1\x2c\x70\x83\x1f\xdc\x60\x6a\xec\x9a\x17\xd6\xb4\x62\x41\x3d\x8e\x62\xb9\x85\x7f\x17\xae\x0b\xcc\x19\x46\x6a\x54\x4b\xe5\x20\x5f\xce\x78\x62\x39\xae\x50\x86\xdc\xba\x91\xcf\x43\xf8\x80\x56\x11\x5c\x08\xf5\x6b\xc2\x88\x1a\x24\xad\xa3\x69\xd0\xac\x36\x14\x87\x72\xe4\x49\x77\x13\x8c\x92\x5a\xde\x94\x8c\x9c\x5e\xf4\x9c\x89\x75\x6e\x79\x8c\x41\x15\x75\x42\x1b\x9d\x3a\x4b\xa3\x68\x20\xf7\xca\x1e\x71\x4a\x12\xfa\xce\xc4\xea\x8d\xa8\x80\xed\x45\x4d\xd3\xa0\xab\x2f\xd3\x15\x03\xd5\x6a\xa0\xce\x56\x68\xc9\x7d\x02\x8c\xc8\xdf\x98\x51\xc5\x4e\xd5\xa0\x11\x8c\x33\xb1\xaa\xdc\x3b\xca\x7f\xad\xa2\xd6\x4e\x5a\x0e\x63\xd3\xd5\x59\x62\xb8\xee\xbe\xb2\xe4\x34\x4b\x0a\xd8\x5e\x0d\x4b\xca\x60\x6a\x6c\x39\xaf\x36\xa5\x5e\xbc\x4a\x6e\x55\x0d\x72\x1c\x53\x31\x48\x9f\x55\xfb\x77\x90\x3d\xe9\xd1\xd5\xdf\xea\xbf\x17\xba\x52\x4f\x11\x5e\x01\x43\x4c\x3d\x14\x54\x94\xe9\x4a\x62\x41\x13\xed\x69\x34\xf6\xca\x77\xc6\x62\xc9\x30\xed\xa6\xf6\x1a\xc5\x34\xca\xb4\xcf\xfb\x2e\x22\x96\x68\x49\x85\x18\x41\x82\xb4\xc7\xc8\x9d\xe1\x54\x84\xb1\x96\x1e\x1e\x51\x5c\xa4\xb4\x58\x1a\xac\x80\xe9\x2e\x21\x6f\xb9\xf8\x89\xca\xf0\x67\x86\x45\x96\xec\xa1\x53\x22\x85\xce\x1d\x8f\xa9\xd8\xc0\xa3\xe0\x0a\xf0\x91\x79\xaf\x2a\xfc\xcb\x41\x71\x7a\xeb\x06\xb7\x17\x9a\x70\x42\x95\x23\xfe\xfb\xdc\x84\xb9\x1d\x42\x36\xa7\x9f\x35\xe4\x6f\x82\xdb\xef\x07\xcf\x8c\x65\x3f\x4f\x2c\xc7\x3f\xb0\xb0\xd9\x8e\xda\x83\x26\x20\xa1\x32\x74\xa1\xc2\xdf\x0d\x21\x33\x90\xff\x34\x17\xfc\x10\xb2\xff\xc7\x64\x5f\x15\x6c\xa0\xc2\x3e\xd4\xa4\xad\x91\x66\xe3\x22\x29\xb1\x6e\x49\xd8\x19\xcd\x42\x09\x30\x45\xf3\x8e\x30\xb1\x7e\x0f\xf8\x3c\x93\x0a\xe2\x7b\x29\xe9\x86\x01\xe9\xff\x25\x71\x72\x74\xc0\x8a\x6e\xe1\x1d\x98\x8f\xcc\x4d\x0a\x08\xf2\x8c\x3b\x8d\x2c\xc3\x68\x70\x09\xd0\x3a\xd2\xaa\x8e\x0c\x21\xfb\x43\x44\xfb\x63\xd7\x20\x80\x61\x18\x9d\x15\x40\xde\x9c\x1d\x51\x0c\x64\xc8\xc4\x72\x0e\xad\xa6\xef\xd9\xb7\x69\x14\x39\x63\xef\x50\x03\x2f\x8f\x97\xfe\x49\xd5\xf3\xbe\x3f\x0d\xf9\xbe\x6c\xa0\x39\x37\x44\x69\x1a\x8e\x0d\x1c\xfe\x91\xc7\x49\xaa\xc0\x27\x35\x0d\x9e\x1b\xc6\x98\xff\xea\x37\xfa\x74\x76\x2d\x26\x5f\xa1\x27\x06\x88\xc0\xb9\x42\x8e\x08\x19\x32\xa1\xf8\x8d\xdb\xf8\x8b\x0c\x2c\xed\x09\xc0\xc8\xe7\x1e\x71\x19\x3a\xf2\x54\x0e\xf8\xea\x23\xe0\xd2\xc1\x29\xb1\xe8\x66\xc1\xcb\xaf\xcf\x9c\x79\x85\x38\xcc\xa6\x6e\xf0\xa6\x2e\x0e\xa5\xe4\x7b\x47\x5f\x7f\x7b\xf3\x48\x40\xc4\x54\x6a\x46\x98\xb1\xae\x19\xe7\x37\x70\xdf\x97\xc2\xca\x72\xd3\xfa\x7a\x67\x34\xfb\x24\x50\xa2\x6f\x8b\x33\x4c\x53\x56\x1a\xf7\xda\xb6\xbf\x39\x96\xff\x76\xe6\xab\xa0\xb5\xba\xfc\x22\xb3\xde\x45\x04\xe7\xf2\x7d\xd5\x2d\x30\x37\xc5\xf3\xee\xe6\x9f\x00\x00\x00\xff\xff\x60\x27\x3a\x8d\xec\x19\x00\x00") func clusterDevelopmentPredeployJsonBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/deploy/generator/resources.go b/pkg/deploy/generator/resources.go index 8d10b42bc70..74ffbe8af17 100644 --- a/pkg/deploy/generator/resources.go +++ b/pkg/deploy/generator/resources.go @@ -133,7 +133,7 @@ func (g *generator) virtualNetworkPeering(name, vnetB string) *arm.Resource { } } -func (g *generator) keyVault(name string, accessPolicies *[]mgmtkeyvault.AccessPolicyEntry) *arm.Resource { +func (g *generator) keyVault(name string, accessPolicies *[]mgmtkeyvault.AccessPolicyEntry, condition interface{}, dependsOn []string) *arm.Resource { return &arm.Resource{ Resource: &mgmtkeyvault.Vault{ Properties: &mgmtkeyvault.VaultProperties{ @@ -152,5 +152,7 @@ func (g *generator) keyVault(name string, accessPolicies *[]mgmtkeyvault.AccessP Location: to.StringPtr("[resourceGroup().location]"), }, APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), + Condition: condition, + DependsOn: dependsOn, } } diff --git a/pkg/deploy/generator/resources_cluster.go b/pkg/deploy/generator/resources_cluster.go index a561628f438..3297fc92bc6 100644 --- a/pkg/deploy/generator/resources_cluster.go +++ b/pkg/deploy/generator/resources_cluster.go @@ -4,6 +4,10 @@ package generator // Licensed under the Apache License 2.0. import ( + "fmt" + + mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" + mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault" mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -11,6 +15,11 @@ import ( "github.com/Azure/ARO-RP/pkg/util/azureclient" ) +const ( + diskEncryptionKeyName = "concat(resourceGroup().name, '-disk-encryption-key')" + diskEncryptionSetName = "concat(resourceGroup().name, '-disk-encryption-set')" +) + func (g *generator) clusterVnet() *arm.Resource { return g.virtualNetwork("dev-vnet", "[parameters('vnetAddressPrefix')]", nil, "[parameters('ci')]", nil) } @@ -72,3 +81,85 @@ func (g *generator) clusterWorkerSubnet() *arm.Resource { }, } } + +func (g *generator) diskEncryptionKeyVault() *arm.Resource { + vaultResource := g.keyVault("[parameters('kvName')]", &[]mgmtkeyvault.AccessPolicyEntry{}, "[parameters('ci')]", nil) + + return vaultResource +} + +func (g *generator) diskEncryptionKey() *arm.Resource { + key := &mgmtkeyvault.Key{ + KeyProperties: &mgmtkeyvault.KeyProperties{ + Kty: mgmtkeyvault.RSA, + KeySize: to.Int32Ptr(4096), + }, + + Name: to.StringPtr(fmt.Sprintf("[concat(parameters('kvName'), '/', %s)]", diskEncryptionKeyName)), + Type: to.StringPtr("Microsoft.KeyVault/vaults/keys"), + Location: to.StringPtr("[resourceGroup().location]"), + } + + return &arm.Resource{ + Resource: key, + APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), + DependsOn: []string{"[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"}, + Condition: to.StringPtr("[parameters('ci')]"), + } +} + +func (g *generator) diskEncryptionSet() *arm.Resource { + diskEncryptionSet := &mgmtcompute.DiskEncryptionSet{ + EncryptionSetProperties: &mgmtcompute.EncryptionSetProperties{ + ActiveKey: &mgmtcompute.KeyVaultAndKeyReference{ + KeyURL: to.StringPtr(fmt.Sprintf("[reference(resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s), '%s', 'Full').properties.keyUriWithVersion]", diskEncryptionKeyName, azureclient.APIVersion("Microsoft.KeyVault"))), + SourceVault: &mgmtcompute.SourceVault{ + ID: to.StringPtr("[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"), + }, + }, + }, + + Name: to.StringPtr(fmt.Sprintf("[%s]", diskEncryptionSetName)), + Type: to.StringPtr("Microsoft.Compute/diskEncryptionSets"), + Location: to.StringPtr("[resourceGroup().location]"), + Identity: &mgmtcompute.EncryptionSetIdentity{Type: mgmtcompute.SystemAssigned}, + } + + return &arm.Resource{ + Resource: diskEncryptionSet, + APIVersion: azureclient.APIVersion("Microsoft.Compute"), + Condition: to.StringPtr("[parameters('ci')]"), + DependsOn: []string{fmt.Sprintf("[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s)]", diskEncryptionKeyName)}, + } +} + +func (g *generator) diskEncryptionKeyVaultAccessPolicy() *arm.Resource { + accessPolicy := &mgmtkeyvault.VaultAccessPolicyParameters{ + Properties: &mgmtkeyvault.VaultAccessPolicyProperties{ + AccessPolicies: &[]mgmtkeyvault.AccessPolicyEntry{ + { + TenantID: &tenantUUIDHack, + ObjectID: to.StringPtr(fmt.Sprintf("[reference(resourceId('Microsoft.Compute/diskEncryptionSets', %s), '%s', 'Full').identity.PrincipalId]", diskEncryptionSetName, azureclient.APIVersion("Microsoft.Compute/diskEncryptionSets"))), + Permissions: &mgmtkeyvault.Permissions{ + Keys: &[]mgmtkeyvault.KeyPermissions{ + mgmtkeyvault.KeyPermissionsGet, + mgmtkeyvault.KeyPermissionsWrapKey, + mgmtkeyvault.KeyPermissionsUnwrapKey, + }, + }, + }, + }, + }, + + Name: to.StringPtr("[concat(parameters('kvName'), '/add')]"), + Type: to.StringPtr("Microsoft.KeyVault/vaults/accessPolicies"), + Location: to.StringPtr("[resourceGroup().location]"), + } + + return &arm.Resource{ + Resource: accessPolicy, + APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), + Condition: to.StringPtr("[parameters('ci')]"), + DependsOn: []string{fmt.Sprintf("[resourceId('Microsoft.Compute/diskEncryptionSets', %s)]", diskEncryptionSetName)}, + } +} diff --git a/pkg/deploy/generator/resources_dev.go b/pkg/deploy/generator/resources_dev.go index 0b10efd7fea..43d791270a3 100644 --- a/pkg/deploy/generator/resources_dev.go +++ b/pkg/deploy/generator/resources_dev.go @@ -461,14 +461,17 @@ chmod +x /etc/cron.hourly/tmpwatch } const ( - sharedKeyVaultName = "concat(take(resourceGroup().name,15), '-sharedKV')" - sharedDiskEncryptionSetName = "concat(resourceGroup().name, '-disk-encryption-set')" + sharedKeyVaultName = "concat(take(resourceGroup().name,15), '" + SharedKeyVaultNameSuffix + "')" + sharedDiskEncryptionSetName = "concat(resourceGroup().name, '" + SharedDiskEncryptionSetNameSuffix + "')" sharedDiskEncryptionKeyName = "concat(resourceGroup().name, '-disk-encryption-key')" + + SharedKeyVaultNameSuffix = "-sharedKV" + SharedDiskEncryptionSetNameSuffix = "-disk-encryption-set" ) // shared keyvault for keys used for disk encryption sets when creating clusters locally func (g *generator) devDiskEncryptionKeyvault() *arm.Resource { - return g.keyVault(fmt.Sprintf("[%s]", sharedKeyVaultName), &[]mgmtkeyvault.AccessPolicyEntry{}) + return g.keyVault(fmt.Sprintf("[%s]", sharedKeyVaultName), &[]mgmtkeyvault.AccessPolicyEntry{}, nil, nil) } func (g *generator) devDiskEncryptionKey() *arm.Resource { diff --git a/pkg/deploy/generator/resources_disk_encryption.go b/pkg/deploy/generator/resources_disk_encryption.go deleted file mode 100644 index 83f95e4ffdc..00000000000 --- a/pkg/deploy/generator/resources_disk_encryption.go +++ /dev/null @@ -1,109 +0,0 @@ -package generator - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "fmt" - - mgmtcompute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute" - mgmtkeyvault "github.com/Azure/azure-sdk-for-go/services/keyvault/mgmt/2019-09-01/keyvault" - "github.com/Azure/go-autorest/autorest/to" - - "github.com/Azure/ARO-RP/pkg/util/arm" - "github.com/Azure/ARO-RP/pkg/util/azureclient" -) - -func (g *generator) diskEncryptionKeyVault() *arm.Resource { - vaultResource := g.keyVault("[parameters('kvName')]", &[]mgmtkeyvault.AccessPolicyEntry{ - { - ObjectID: to.StringPtr("[parameters('clusterServicePrincipalId')]"), - Permissions: &mgmtkeyvault.Permissions{ - Keys: &[]mgmtkeyvault.KeyPermissions{mgmtkeyvault.KeyPermissionsCreate, mgmtkeyvault.KeyPermissionsGet, mgmtkeyvault.KeyPermissionsDelete}, - }, - // is later replaced by "[subscription().tenantId]" - TenantID: &tenantUUIDHack, - }, - }) - - vaultResource.Condition = to.StringPtr("[parameters('ci')]") - - return vaultResource -} - -const diskEncryptionKeyName string = "concat(resourceGroup().name, '-', 'disk-encryption-key')" - -func (g *generator) diskEncryptionKey() *arm.Resource { - key := &mgmtkeyvault.Key{ - KeyProperties: &mgmtkeyvault.KeyProperties{ - Kty: mgmtkeyvault.RSA, - KeySize: to.Int32Ptr(4096), - }, - - Name: to.StringPtr(fmt.Sprintf("[concat(parameters('kvName'), '/', %s)]", diskEncryptionKeyName)), - Type: to.StringPtr("Microsoft.KeyVault/vaults/keys"), - Location: to.StringPtr("[resourceGroup().location]"), - } - - return &arm.Resource{ - Resource: key, - APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), - DependsOn: []string{"[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"}, - Condition: to.StringPtr("[parameters('ci')]"), - } -} - -func (g *generator) diskEncryptionSet() *arm.Resource { - diskEncryptionSet := &mgmtcompute.DiskEncryptionSet{ - EncryptionSetProperties: &mgmtcompute.EncryptionSetProperties{ - ActiveKey: &mgmtcompute.KeyVaultAndKeyReference{ - KeyURL: to.StringPtr(fmt.Sprintf("[reference(resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s), '%s', 'Full').properties.keyUriWithVersion]", diskEncryptionKeyName, azureclient.APIVersion("Microsoft.KeyVault"))), - SourceVault: &mgmtcompute.SourceVault{ - ID: to.StringPtr("[resourceId('Microsoft.KeyVault/vaults', parameters('kvName'))]"), - }, - }, - }, - - Name: to.StringPtr("[parameters('diskEncryptionSetName')]"), - Type: to.StringPtr("Microsoft.Compute/diskEncryptionSets"), - Location: to.StringPtr("[resourceGroup().location]"), - Identity: &mgmtcompute.EncryptionSetIdentity{Type: mgmtcompute.SystemAssigned}, - } - - return &arm.Resource{ - Resource: diskEncryptionSet, - APIVersion: azureclient.APIVersion("Microsoft.Compute"), - Condition: to.StringPtr("[parameters('ci')]"), - DependsOn: []string{fmt.Sprintf("[resourceId('Microsoft.KeyVault/vaults/keys', parameters('kvName'), %s)]", diskEncryptionKeyName)}, - } -} - -func (g *generator) diskEncryptionKeyVaultAccessPolicy() *arm.Resource { - accessPolicy := &mgmtkeyvault.VaultAccessPolicyParameters{ - Properties: &mgmtkeyvault.VaultAccessPolicyProperties{ - AccessPolicies: &[]mgmtkeyvault.AccessPolicyEntry{ - { - TenantID: &tenantUUIDHack, - ObjectID: to.StringPtr(fmt.Sprintf("[reference(resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName')), '%s', 'Full').identity.PrincipalId]", azureclient.APIVersion("Microsoft.Compute/diskEncryptionSets"))), - Permissions: &mgmtkeyvault.Permissions{ - Keys: &[]mgmtkeyvault.KeyPermissions{ - mgmtkeyvault.KeyPermissionsGet, - mgmtkeyvault.KeyPermissionsWrapKey, - mgmtkeyvault.KeyPermissionsUnwrapKey, - }, - }, - }, - }, - }, - - Name: to.StringPtr("[concat(parameters('kvName'), '/add')]"), - Type: to.StringPtr("Microsoft.KeyVault/vaults/accessPolicies"), - Location: to.StringPtr("[resourceGroup().location]"), - } - - return &arm.Resource{ - Resource: accessPolicy, - APIVersion: azureclient.APIVersion("Microsoft.KeyVault"), - DependsOn: []string{"[resourceId('Microsoft.Compute/diskEncryptionSets', parameters('diskEncryptionSetName'))]"}, - } -} diff --git a/pkg/deploy/generator/templates_cluster.go b/pkg/deploy/generator/templates_cluster.go index 58bc67cbd4e..6779436c6bc 100644 --- a/pkg/deploy/generator/templates_cluster.go +++ b/pkg/deploy/generator/templates_cluster.go @@ -19,7 +19,6 @@ func (g *generator) clusterPredeploy() *arm.Template { "vnetAddressPrefix", "masterAddressPrefix", "workerAddressPrefix", - "diskEncryptionSetName", "kvName", } diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go index a315ebf46d1..3c7dc8f843e 100644 --- a/pkg/util/cluster/cluster.go +++ b/pkg/util/cluster/cluster.go @@ -45,11 +45,6 @@ import ( "github.com/Azure/ARO-RP/pkg/util/rbac" ) -const ( - sharedKeyVaultNameSuffix string = "-e2eKV" - sharedDiskEncryptionSetNameSuffix string = "-disk-encryption-set" -) - type Cluster struct { log *logrus.Entry env env.Core @@ -181,10 +176,14 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str return err } - // use the shared one by default and overwrite, if we need to generate one because we are in CI - kvName := fmt.Sprintf("%s%s", vnetResourceGroup, sharedKeyVaultNameSuffix) - - diskEncryptionSetName := fmt.Sprintf("%s%s", vnetResourceGroup, sharedDiskEncryptionSetNameSuffix) + var kvName string + if len(vnetResourceGroup) > 15 { + // keyvault names need to have a maximum length of 24, + // so we need to cut off some chars if the resource group name is too long + kvName = vnetResourceGroup[:14] + generator.SharedKeyVaultNameSuffix + } else { + kvName = vnetResourceGroup + generator.SharedKeyVaultNameSuffix + } if c.ci { // name is limited to 24 characters, but must be globally unique, so we generate one and try if it is available @@ -197,8 +196,6 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str if result.NameAvailable != nil && !*result.NameAvailable { return fmt.Errorf("Could not generate unique key vault name: %v", result.Reason) } - - diskEncryptionSetName = clusterName + "-disk-encryption-set" } parameters := map[string]*arm.ParametersParameter{ @@ -210,7 +207,6 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str "masterAddressPrefix": {Value: masterSubnet}, "workerAddressPrefix": {Value: workerSubnet}, "kvName": {Value: kvName}, - "diskEncryptionSetName": {Value: diskEncryptionSetName}, } // TODO: ick @@ -243,7 +239,13 @@ func (c *Cluster) Create(ctx context.Context, vnetResourceGroup, clusterName str return err } - diskEncryptionSetID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/diskEncryptionSets/%s", c.env.SubscriptionID(), vnetResourceGroup, diskEncryptionSetName) + diskEncryptionSetID := fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/diskEncryptionSets/%s%s", + c.env.SubscriptionID(), + vnetResourceGroup, + vnetResourceGroup, + generator.SharedDiskEncryptionSetNameSuffix, + ) c.log.Info("creating role assignments") for _, scope := range []struct{ resource, role string }{ diff --git a/test/e2e/disk_encryption.go b/test/e2e/disk_encryption.go index b2a6ab4e3eb..13d41804567 100644 --- a/test/e2e/disk_encryption.go +++ b/test/e2e/disk_encryption.go @@ -16,8 +16,6 @@ import ( ) var _ = Describe("Encryption at host should be enabled", func() { - BeforeEach(skipIfNotInDevelopmentEnv) - Specify("each VM should have encryption at host enabled", func() { ctx := context.Background() @@ -33,7 +31,6 @@ var _ = Describe("Encryption at host should be enabled", func() { By("checking the encryption property on each VM") for _, vm := range vms { - log.Println(vm.Name) Expect(vm.SecurityProfile).To(Not(BeNil())) Expect(vm.SecurityProfile.EncryptionAtHost).To(Not(BeNil())) Expect(*vm.SecurityProfile.EncryptionAtHost).To(Equal(true)) @@ -43,9 +40,6 @@ var _ = Describe("Encryption at host should be enabled", func() { }) var _ = Describe("Disk encryption at rest should be enabled with customer managed key", func() { - BeforeEach(skipIfNotInDevelopmentEnv) - - // We have to get the disks by VM, because when getting all disks by resource group, we do not get recently created disks, see https://github.com/Azure/azure-cli/issues/17123 Specify("each disk should have encryption at rest with customer managed key enabled", func() { ctx := context.Background() @@ -59,6 +53,8 @@ var _ = Describe("Disk encryption at rest should be enabled with customer manage Expect(err).NotTo(HaveOccurred()) Expect(vms).NotTo(HaveLen(0)) + // We have to get the disks by VM, because when getting all disks by resource group, + // we do not get recently created disks, see https://github.com/Azure/azure-cli/issues/17123 By("checking the encryption property on each OS disk of each VM") for _, vm := range vms { osDisk, err := clients.Disks.Get(ctx, clusterResourceGroup, *vm.StorageProfile.OsDisk.Name)