diff --git a/Release.toml b/Release.toml index d965f51cdf7..ffbc6cf7609 100644 --- a/Release.toml +++ b/Release.toml @@ -319,4 +319,5 @@ version = "1.21.0" "migrate_v1.21.0_pod-infra-container-image-remove-settings-generator.lz4", "migrate_v1.21.0_pod-infra-container-image-affected-services.lz4", "migrate_v1.21.0_pod-infra-container-image-services.lz4", + "migrate_v1.21.0_k8s-reserved-cpus-v0-1-0.lz4", ] diff --git a/packages/kubernetes-1.23/kubelet-config b/packages/kubernetes-1.23/kubelet-config index 1d6fdaa126d..1594713a182 100644 --- a/packages/kubernetes-1.23/kubelet-config +++ b/packages/kubernetes-1.23/kubelet-config @@ -87,13 +87,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -182,3 +186,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.24/kubelet-config b/packages/kubernetes-1.24/kubelet-config index ecfaf92b456..14a7abac6bd 100644 --- a/packages/kubernetes-1.24/kubelet-config +++ b/packages/kubernetes-1.24/kubelet-config @@ -87,13 +87,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -181,3 +185,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.25/kubelet-config b/packages/kubernetes-1.25/kubelet-config index 1c0e94ced4b..8fa3567e1ad 100644 --- a/packages/kubernetes-1.25/kubelet-config +++ b/packages/kubernetes-1.25/kubelet-config @@ -87,13 +87,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -182,3 +186,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.26/kubelet-config b/packages/kubernetes-1.26/kubelet-config index 1c0e94ced4b..8fa3567e1ad 100644 --- a/packages/kubernetes-1.26/kubelet-config +++ b/packages/kubernetes-1.26/kubelet-config @@ -87,13 +87,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -182,3 +186,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.27/kubelet-config b/packages/kubernetes-1.27/kubelet-config index 2efd361818a..593a5660cd8 100644 --- a/packages/kubernetes-1.27/kubelet-config +++ b/packages/kubernetes-1.27/kubelet-config @@ -91,13 +91,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -183,3 +187,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.28/kubelet-config b/packages/kubernetes-1.28/kubelet-config index b196ac1d0b1..84eaae341ae 100644 --- a/packages/kubernetes-1.28/kubelet-config +++ b/packages/kubernetes-1.28/kubelet-config @@ -91,13 +91,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -181,3 +185,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.29/kubelet-config b/packages/kubernetes-1.29/kubelet-config index b196ac1d0b1..84eaae341ae 100644 --- a/packages/kubernetes-1.29/kubelet-config +++ b/packages/kubernetes-1.29/kubelet-config @@ -91,13 +91,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -181,3 +185,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/packages/kubernetes-1.30/kubelet-config b/packages/kubernetes-1.30/kubelet-config index b196ac1d0b1..84eaae341ae 100644 --- a/packages/kubernetes-1.30/kubelet-config +++ b/packages/kubernetes-1.30/kubelet-config @@ -91,13 +91,17 @@ kubeReserved: {{/if}} {{/if}} ephemeral-storage: "{{default "1Gi" settings.kubernetes.kube-reserved.ephemeral-storage}}" +{{#unless settings.kubernetes.reserved-cpus}} kubeReservedCgroup: "/runtime" +{{/unless}} {{#if settings.kubernetes.system-reserved}} systemReserved: {{#each settings.kubernetes.system-reserved}} {{@key}}: "{{this}}" {{/each}} +{{#unless settings.kubernetes.reserved-cpus}} systemReservedCgroup: "/system" +{{/unless}} {{/if}} cpuCFSQuota: {{default true settings.kubernetes.cpu-cfs-quota-enforced}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} @@ -181,3 +185,6 @@ reservedMemory: {{/each}} {{/if}} {{/if}} +{{#if settings.kubernetes.reserved-cpus}} +reservedSystemCPUs: {{settings.kubernetes.reserved-cpus}} +{{/if}} diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 0840683f25e..0a43370f27b 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -2391,6 +2391,13 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "k8s-reserved-cpus-v0-1-0" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "language-tags" version = "0.3.2" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 43575f3fb72..0ca12eed0a9 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -28,6 +28,7 @@ members = [ "api/migration/migrations/v1.21.0/pod-infra-container-image-affected-services", "api/migration/migrations/v1.21.0/pod-infra-container-image-remove-settings-generator", "api/migration/migrations/v1.21.0/pod-infra-container-image-services", + "api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0", "bloodhound", diff --git a/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/Cargo.toml b/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/Cargo.toml new file mode 100644 index 00000000000..4ec663aaa3e --- /dev/null +++ b/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "k8s-reserved-cpus-v0-1-0" +version = "0.1.0" +authors = ["James Masson "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false + +[dependencies] +migration-helpers = { path = "../../../migration-helpers", version = "0.1.0" } diff --git a/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/src/main.rs b/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/src/main.rs new file mode 100644 index 00000000000..54cba0056d5 --- /dev/null +++ b/sources/api/migration/migrations/v1.21.0/k8s-reserved-cpus-v0-1-0/src/main.rs @@ -0,0 +1,18 @@ +use migration_helpers::common_migrations::AddSettingsMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// Add the option to set Kubernetes reserved-cpus +fn run() -> Result<()> { + migrate(AddSettingsMigration(&["settings.kubernetes.reserved-cpus"])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} diff --git a/sources/models/modeled-types/src/lib.rs b/sources/models/modeled-types/src/lib.rs index 659788dd8d3..9ee46c04715 100644 --- a/sources/models/modeled-types/src/lib.rs +++ b/sources/models/modeled-types/src/lib.rs @@ -150,6 +150,9 @@ pub mod error { input: String, source: std::num::ParseIntError, }, + + #[snafu(display("Invalid Kernel CpuSet value '{}'", input))] + InvalidKernelCpuSetValue { input: String }, } /// Creates a `ValidationError` with a consistent message for strings with regex validations diff --git a/sources/models/modeled-types/src/shared.rs b/sources/models/modeled-types/src/shared.rs index 5efb119da85..829c382c22f 100644 --- a/sources/models/modeled-types/src/shared.rs +++ b/sources/models/modeled-types/src/shared.rs @@ -1178,3 +1178,78 @@ mod test_positive_integer { assert!(NonNegativeInteger::try_from(-1).is_err()); } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KernelCpuSetValue represents a string that contains a valid Kernel CpuSet Value from +/// here: https://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS. This matches the +/// logic from https://github.com/kubernetes/utils/blob/d93618cff8a22d3aea7bf78d9d528fd859720c2d/cpuset/cpuset.go#L203 +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KernelCpuSetValue { + inner: String, +} + +lazy_static! { + pub(crate) static ref KERNAL_CPU_SET_VALUE: Regex = + Regex::new(r"^([0-9]+(-[0-9]+)?,?)*([0-9]+(-[0-9]+)?)+$").unwrap(); +} + +impl TryFrom<&str> for KernelCpuSetValue { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + !input.is_empty(), + error::InvalidKernelCpuSetValueSnafu { input } + ); + ensure!( + KERNAL_CPU_SET_VALUE.is_match(input), + error::InvalidKernelCpuSetValueSnafu { input } + ); + Ok(KernelCpuSetValue { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KernelCpuSetValue, "KernelCpuSetValue"); + +#[cfg(test)] +mod test_kernel_cpu_set_value { + use super::KernelCpuSetValue; + use std::convert::TryFrom; + + #[test] + fn good_tokens() { + for ok in &[ + "1", + "0,1,2,3", + "1-4", + "1,6-9", + "1-4,6", + "1-3,6-9,10-15", + "100,101", + "1,2,3,4,5,6,7,8,9,10", + ] { + KernelCpuSetValue::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_names() { + for err in &[ + "", + "100.0", + "0...4", + "1..5", + "ten", + "1-", + "-3", + "9,-10", + "1,2,3,4,5,6,7,8,9,10,", + &"a".repeat(23), + ] { + KernelCpuSetValue::try_from(*err).unwrap_err(); + } + } +} diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index ef4e7ce1fa2..a827636539b 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -48,13 +48,13 @@ use std::net::IpAddr; use crate::de::{deserialize_limit, deserialize_mirrors, deserialize_node_taints}; use modeled_types::{ BootConfigKey, BootConfigValue, BootstrapContainerMode, CpuManagerPolicy, CredentialProvider, - DNSDomain, EtcHostsEntries, Identifier, IntegerPercent, KubernetesAuthenticationMode, - KubernetesBootstrapToken, KubernetesCloudProvider, KubernetesClusterDnsIp, - KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey, KubernetesLabelValue, - KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue, - KubernetesThresholdValue, OciDefaultsCapability, OciDefaultsResourceLimitType, - PemCertificateString, SingleLineString, TopologyManagerPolicy, TopologyManagerScope, Url, - ValidBase64, ValidLinuxHostname, + DNSDomain, EtcHostsEntries, Identifier, IntegerPercent, KernelCpuSetValue, + KubernetesAuthenticationMode, KubernetesBootstrapToken, KubernetesCloudProvider, + KubernetesClusterDnsIp, KubernetesClusterName, KubernetesDurationValue, KubernetesLabelKey, + KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey, + KubernetesTaintValue, KubernetesThresholdValue, OciDefaultsCapability, + OciDefaultsResourceLimitType, PemCertificateString, SingleLineString, TopologyManagerPolicy, + TopologyManagerScope, Url, ValidBase64, ValidLinuxHostname, }; #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] @@ -137,6 +137,7 @@ struct KubernetesSettings { shutdown_grace_period_for_critical_pods: KubernetesDurationValue, memory_manager_reserved_memory: HashMap, memory_manager_policy: KubernetesMemoryManagerPolicy, + reserved_cpus: KernelCpuSetValue, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not.