Skip to content
This repository was archived by the owner on Oct 24, 2023. It is now read-only.

feat: enable flexible SSD configuration #3872

Closed
jackfrancis wants to merge 17 commits into from
Closed

feat: enable flexible SSD configuration #3872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
674c165
feat: enable flexible SSL configuration
jackfrancis Sep 25, 2020
06e3101
update test api models
jackfrancis Sep 25, 2020
077871f
move to func
jackfrancis Sep 28, 2020
eac0476
defaults
jackfrancis Sep 28, 2020
acb67f2
on the way
jackfrancis Sep 28, 2020
adc3a5e
more defaults
jackfrancis Sep 30, 2020
16950d2
update test configs
jackfrancis Sep 30, 2020
77782b7
JSON formatting
jackfrancis Sep 30, 2020
adaa772
feedback
jackfrancis Oct 1, 2020
cee5a5e
test configs, and rationalize ephemeral
jackfrancis Oct 1, 2020
ae3bfa4
update to osDiskType and dataDiskType
jackfrancis Dec 15, 2020
192a6fa
add isUpgrade to setMasterProfileDefaults
jackfrancis Dec 15, 2020
63fd928
update UT
jackfrancis Dec 15, 2020
a1b434f
additive UT
jackfrancis Dec 15, 2020
450b092
enable no-vnet E2E config as well
jackfrancis Dec 15, 2020
0b5ed2f
update test configs
jackfrancis Dec 16, 2020
58180a9
add docs reference for iops/mbps dictionary
jackfrancis Dec 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions examples/e2e-tests/kubernetes/release/default/definition-no-vnet.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"kubernetesConfig": {
"enableEncryptionWithExternalKms": true,
"useManagedIdentity": true,
"etcdDiskSizeGB": "1024",
"addons": [
{
"name": "cluster-autoscaler",
Expand Down Expand Up @@ -46,26 +47,29 @@
"masterProfile": {
"count": 3,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"vmSize": "Standard_D4s_v3",
"OSDiskSizeGB": 200,
"availabilityZones": [
"1",
"2"
]
],
"ultraSSDEnabled": true
},
"agentPoolProfiles": [
{
"name": "poollinux",
"count": 1,
"vmSize": "Standard_D2_v3",
"vmSize": "Standard_D2s_v3",
"OSDiskSizeGB": 200,
"storageProfile": "ManagedDisks",
"diskSizesGB": [
128
],
"availabilityProfile": "VirtualMachineScaleSets",
"osDiskCachingType": "ReadOnly",
"dataDiskCachingType": "ReadWrite"
"dataDiskCachingType": "ReadWrite",
"osDiskType": "StandardSSD",
"dataDiskType": "PremiumSSD"
},
{
"name": "poolwin",
Expand All @@ -78,9 +82,15 @@
{
"name": "pool1604vhd",
"count": 1,
"vmSize": "Standard_D2_v3",
"vmSize": "Standard_D4s_v3",
"distro": "aks-ubuntu-16.04",
"availabilityProfile": "VirtualMachineScaleSets"
"availabilityProfile": "VirtualMachineScaleSets",
"availabilityZones": [
"1",
"2"
],
"osDiskType": "StandardSSD",
"ultraSSDEnabled": true
},
{
"name": "pool1604",
Expand Down
19 changes: 13 additions & 6 deletions examples/e2e-tests/kubernetes/release/default/definition.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"enableEncryptionWithExternalKms": true,
"useManagedIdentity": true,
"clusterSubnet": "10.239.0.0/16",
"etcdDiskSizeGB": "1024",
"addons": [
{
"name": "kubernetes-dashboard",
Expand Down Expand Up @@ -51,18 +52,19 @@
"masterProfile": {
"count": 3,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"vmSize": "Standard_D4s_v3",
"OSDiskSizeGB": 200,
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"firstConsecutiveStaticIP": "10.239.255.239",
"vnetCidr": "10.239.0.0/16",
"availabilityZones": ["1", "2"]
"availabilityZones": ["1", "2"],
"ultraSSDEnabled": true
},
"agentPoolProfiles": [
{
"name": "poollinux",
"count": 1,
"vmSize": "Standard_D2_v3",
"vmSize": "Standard_D2s_v3",
"OSDiskSizeGB": 200,
"storageProfile": "ManagedDisks",
"diskSizesGB": [
Expand All @@ -71,7 +73,9 @@
"availabilityProfile": "VirtualMachineScaleSets",
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"osDiskCachingType": "ReadOnly",
"dataDiskCachingType": "ReadWrite"
"dataDiskCachingType": "ReadWrite",
"osDiskType": "StandardSSD",
"dataDiskType": "PremiumSSD"
},
{
"name": "poolwin",
Expand All @@ -85,10 +89,13 @@
{
"name": "pool1604vhd",
"count": 1,
"vmSize": "Standard_D2_v3",
"vmSize": "Standard_D4s_v3",
"distro": "aks-ubuntu-16.04",
"availabilityProfile": "VirtualMachineScaleSets",
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME"
"vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME",
"availabilityZones": ["1", "2"],
"osDiskType": "StandardSSD",
"ultraSSDEnabled": true
Copy link
Contributor

@CecileRobertMichon CecileRobertMichon Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should consider deprecating this altogether since it's redundant information and simply assume that if dataDisk type is ultraSSD it should be enabled on the VM. Is there a use case for enabling it but not using it for any disks?

Copy link
Member Author

@jackfrancis jackfrancis Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, it's complicated, as it is in fact a discrete configuration for the VM (arguably, unfortunately). This configuration tells Azure to "get me a VM that is capable of attaching to an Ultra SSD disk resource". There's a different configuration to actually connect an Ultra SSD disk.

Is there a use-case for getting a ultra ssd-enabled VM, but not actually using ultra ssd for the k8s IaaS bootstrapped by AKS Engine? I don't know... I think not knowing definitively that that will never make sense combined with the fact that that config property already exists... so yeah, just working with it, so to speak.

},
{
"name": "pool1604",
Expand Down
2 changes: 1 addition & 1 deletion examples/kubernetes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3"
"vmSize": "Standard_D2s_v3"
},
"agentPoolProfiles": [
{
Expand Down
8 changes: 8 additions & 0 deletions pkg/api/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,3 +518,11 @@ const TLSStrongCipherSuitesAPIServer = "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS

// TLSStrongCipherSuitesKubelet is a kube-bench-recommended allowed cipher suites for kubelet
const TLSStrongCipherSuitesKubelet = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"

// SSD Types
Copy link
Contributor

@CecileRobertMichon CecileRobertMichon Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same comment here, this could be StorageAccountTypes and we should consider using the same syntax as Azure (Standard_LRS, StandardSSD_LRS, Premium_LRS etc)

Copy link
Member Author

@jackfrancis jackfrancis Dec 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Except that the standard Azure names are so human-offending... How does capz expose this menu?

const (
UltraSSD string = "UltraSSD"
PremiumSSD string = "PremiumSSD"
StandardSSD string = "StandardSSD"
StandardHDD string = "StandardHDD"
)
4 changes: 4 additions & 0 deletions pkg/api/converterfromapi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,8 @@ func convertMasterProfileToVLabs(api *MasterProfile, vlabsProfile *vlabs.MasterP
vlabsProfile.SinglePlacementGroup = api.SinglePlacementGroup
vlabsProfile.CosmosEtcd = api.CosmosEtcd
vlabsProfile.AuditDEnabled = api.AuditDEnabled
vlabsProfile.OSDiskType = api.OSDiskType
vlabsProfile.DataDiskType = api.DataDiskType
vlabsProfile.UltraSSDEnabled = api.UltraSSDEnabled
vlabsProfile.EncryptionAtHost = api.EncryptionAtHost
vlabsProfile.ProximityPlacementGroupID = api.ProximityPlacementGroupID
Expand Down Expand Up @@ -610,6 +612,8 @@ func convertAgentPoolProfileToVLabs(api *AgentPoolProfile, p *vlabs.AgentPoolPro
p.OSDiskCachingType = api.OSDiskCachingType
p.DataDiskCachingType = api.DataDiskCachingType
p.VMSSName = api.VMSSName
p.OSDiskType = api.OSDiskType
p.DataDiskType = api.DataDiskType
}

func convertServicePrincipalProfileToVLabs(api *ServicePrincipalProfile, v *vlabs.ServicePrincipalProfile) {
Expand Down
4 changes: 4 additions & 0 deletions pkg/api/convertertoapi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,8 @@ func convertVLabsMasterProfile(vlabs *vlabs.MasterProfile, api *MasterProfile) {
api.PlatformUpdateDomainCount = vlabs.PlatformUpdateDomainCount
api.SinglePlacementGroup = vlabs.SinglePlacementGroup
api.CosmosEtcd = vlabs.CosmosEtcd
api.OSDiskType = vlabs.OSDiskType
api.DataDiskType = vlabs.DataDiskType
api.UltraSSDEnabled = vlabs.UltraSSDEnabled
api.EncryptionAtHost = vlabs.EncryptionAtHost
api.AuditDEnabled = vlabs.AuditDEnabled
Expand Down Expand Up @@ -650,6 +652,8 @@ func convertVLabsAgentPoolProfile(vlabs *vlabs.AgentPoolProfile, api *AgentPoolP
api.OSDiskCachingType = vlabs.OSDiskCachingType
api.DataDiskCachingType = vlabs.DataDiskCachingType
api.VMSSName = vlabs.VMSSName
api.OSDiskType = vlabs.OSDiskType
api.DataDiskType = vlabs.DataDiskType
}

func convertVLabsKeyVaultSecrets(vlabs *vlabs.KeyVaultSecrets, api *KeyVaultSecrets) {
Expand Down
111 changes: 105 additions & 6 deletions pkg/api/defaults.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (cs *ContainerService) SetPropertiesDefaults(params PropertiesDefaultsParam

// Set master profile defaults if this cluster configuration includes master node(s)
if cs.Properties.MasterProfile != nil {
properties.setMasterProfileDefaults()
properties.setMasterProfileDefaults(params.IsUpgrade)
}

properties.setAgentProfileDefaults(params.IsUpgrade, params.IsScale)
Expand Down Expand Up @@ -389,6 +389,23 @@ func (cs *ContainerService) setOrchestratorDefaults(isUpgrade, isScale bool) {
}
}

if !isUpgrade && !isScale {
if cs.Properties.MasterProfile != nil {
if cs.Properties.MasterProfile.DataDiskType == UltraSSD {
if o.KubernetesConfig.EtcdDiskIOPS == 0 || o.KubernetesConfig.EtcdDiskMBPS == 0 {
etcdSizeGB, _ := strconv.Atoi(o.KubernetesConfig.EtcdDiskSizeGB)
ultraSSDConfig := getDefaultUltraSSDConfig(etcdSizeGB)
if o.KubernetesConfig.EtcdDiskIOPS == 0 {
o.KubernetesConfig.EtcdDiskIOPS = ultraSSDConfig.iops
}
if o.KubernetesConfig.EtcdDiskMBPS == 0 {
o.KubernetesConfig.EtcdDiskMBPS = ultraSSDConfig.mbps
}
}
}
}
}

if a.OrchestratorProfile.KubernetesConfig.EtcdStorageLimitGB == 0 {
a.OrchestratorProfile.KubernetesConfig.EtcdStorageLimitGB = DefaultEtcdStorageLimitGB
}
Expand Down Expand Up @@ -657,7 +674,7 @@ func (p *Properties) setExtensionDefaults() {
}
}

func (p *Properties) setMasterProfileDefaults() {
func (p *Properties) setMasterProfileDefaults(isUpgrade bool) {
// set default to VMAS for now
if p.MasterProfile.AvailabilityProfile == "" {
p.MasterProfile.AvailabilityProfile = AvailabilitySet
Expand Down Expand Up @@ -695,6 +712,20 @@ func (p *Properties) setMasterProfileDefaults() {
if p.MasterProfile.OSDiskCachingType == "" {
p.MasterProfile.OSDiskCachingType = string(compute.CachingTypesReadWrite)
}

if !isUpgrade {
if p.MasterProfile.DataDiskType == "" {
p.MasterProfile.DataDiskType = PremiumSSD
}
if p.MasterProfile.DataDiskType == UltraSSD {
if p.MasterProfile.UltraSSDEnabled == nil {
Copy link
Contributor

@CecileRobertMichon CecileRobertMichon Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens if UltraSSDEnabled is false and DataDiskSSDType is Ultra?

Copy link
Member Author

@jackfrancis jackfrancis Sep 30, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That will be disallowed in validation (// TODO)

p.MasterProfile.UltraSSDEnabled = to.BoolPtr(true)
}
}
if p.MasterProfile.OSDiskType == "" {
p.MasterProfile.OSDiskType = PremiumSSD
}
}
}

func (p *Properties) setAgentProfileDefaults(isUpgrade, isScale bool) {
Expand Down Expand Up @@ -764,13 +795,19 @@ func (p *Properties) setAgentProfileDefaults(isUpgrade, isScale bool) {
profile.EnableVMSSNodePublicIP = to.BoolPtr(DefaultEnableVMSSNodePublicIP)
}

if profile.OSDiskCachingType == "" {
if profile.IsEphemeral() {
if profile.IsEphemeral() {
if profile.OSDiskCachingType == "" {
profile.OSDiskCachingType = string(compute.CachingTypesReadOnly)
} else {
profile.OSDiskCachingType = string(compute.CachingTypesReadWrite)
}
if profile.OSDiskType == "" {
profile.OSDiskType = StandardHDD
}
}

if profile.OSDiskCachingType == "" {
profile.OSDiskCachingType = string(compute.CachingTypesReadWrite)
}

if profile.DataDiskCachingType == "" {
profile.DataDiskCachingType = string(compute.CachingTypesReadOnly)
}
Expand Down Expand Up @@ -1167,3 +1204,65 @@ func (cs *ContainerService) setCSIProxyDefaults() {
}
}
}

type ssdConfig struct {
iops int
mbps int
}

// getDefaultUltraSSDConfig returns a known-working IOPS + MBPS config based on the size of the etcd disk
// See https://docs.microsoft.com/en-us/azure/virtual-machines/disks-types#ultra-disk
func getDefaultUltraSSDConfig(etcdDiskSizeGB int) ssdConfig {
if etcdDiskSizeGB >= 1024 {
return ssdConfig{
iops: 160000,
mbps: 2000,
}
}
if etcdDiskSizeGB >= 512 {
return ssdConfig{
iops: 80000,
mbps: 2000,
}
}
if etcdDiskSizeGB >= 256 {
return ssdConfig{
iops: 76800,
mbps: 2000,
}
}
if etcdDiskSizeGB >= 128 {
return ssdConfig{
iops: 38400,
mbps: 2000,
}
}
if etcdDiskSizeGB >= 64 {
return ssdConfig{
iops: 19200,
mbps: 2000,
}
}
if etcdDiskSizeGB >= 32 {
return ssdConfig{
iops: 9600,
mbps: 2000,
}
}
if etcdDiskSizeGB >= 16 {
return ssdConfig{
iops: 4800,
mbps: 1200,
}
}
if etcdDiskSizeGB >= 8 {
return ssdConfig{
iops: 2400,
mbps: 600,
}
}
return ssdConfig{
iops: 1200,
mbps: 300,
}
}
Loading