From d5a6667ba539387ea41e70764789ae7652fd5e9e Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Tue, 29 Jun 2021 19:17:36 +0000 Subject: [PATCH 1/4] kubelet: add setting for configuring cpuManagerPolicy pass cpu-manager-policy argument to kubelet --- README.md | 1 + packages/kubernetes-1.16/kubelet-config | 1 + packages/kubernetes-1.17/kubelet-config | 1 + packages/kubernetes-1.18/kubelet-config | 1 + packages/kubernetes-1.19/kubelet-config | 1 + packages/kubernetes-1.20/kubelet-config | 2 +- packages/kubernetes-1.21/kubelet-config | 2 +- sources/models/src/lib.rs | 3 +- .../models/src/modeled_types/kubernetes.rs | 49 +++++++++++++++++++ sources/models/src/modeled_types/mod.rs | 6 +++ 10 files changed, 64 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e47139ccb88..752fa700f97 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,7 @@ The following settings are optional and allow you to further configure your clus * `settings.kubernetes.kube-api-burst`: The burst to allow while talking with kubernetes. * `settings.kubernetes.container-log-max-size`: The maximum size of container log file before it is rotated. * `settings.kubernetes.container-log-max-files`: The maximum number of container log files that can be present for a container. +* `settings.kubernetes.cpu-manager-policy`: Specifies the CPU manager policy. Possible values are `static` and `none`. Defaults to `none`. If you want to allow pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node, you can set this setting to `static`. You should reboot if you change this setting after startup - try `apiclient reboot`. You can also optionally specify static pods for your node with the following settings. Static pods can be particularly useful when running in standalone mode. diff --git a/packages/kubernetes-1.16/kubelet-config b/packages/kubernetes-1.16/kubelet-config index 131a54a02ac..d934102e5db 100644 --- a/packages/kubernetes-1.16/kubelet-config +++ b/packages/kubernetes-1.16/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{@key}}: "{{this}}" {{~/each}} {{~/if}} +cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index ea8020eba43..5fbd58e4b0e 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{@key}}: "{{this}}" {{~/each}} {{~/if}} +cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index ea8020eba43..5fbd58e4b0e 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{@key}}: "{{this}}" {{~/each}} {{~/if}} +cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index 9e91f7c430b..3fe27e1adc4 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -74,6 +74,7 @@ systemReserved: {{@key}}: "{{this}}" {{~/each}} {{~/if}} +cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config index da033f02d31..3fe27e1adc4 100644 --- a/packages/kubernetes-1.20/kubelet-config +++ b/packages/kubernetes-1.20/kubelet-config @@ -74,7 +74,7 @@ systemReserved: {{@key}}: "{{this}}" {{~/each}} {{~/if}} -cpuManagerPolicy: "static" +cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.21/kubelet-config b/packages/kubernetes-1.21/kubelet-config index da033f02d31..3fe27e1adc4 100644 --- a/packages/kubernetes-1.21/kubelet-config +++ b/packages/kubernetes-1.21/kubelet-config @@ -74,7 +74,7 @@ systemReserved: {{@key}}: "{{this}}" {{~/each}} {{~/if}} -cpuManagerPolicy: "static" +cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index c4c89b4a09f..bbed738319f 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -113,7 +113,7 @@ use std::collections::HashMap; use std::net::Ipv4Addr; use crate::modeled_types::{ - BootstrapContainerMode, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, + BootstrapContainerMode, CpuManagerPolicy, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, FriendlyVersion, Identifier, KubernetesAuthenticationMode, KubernetesBootstrapToken, KubernetesCloudProvider, KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey, KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey, @@ -158,6 +158,7 @@ struct KubernetesSettings { kube_api_burst: i32, container_log_max_size: KubernetesQuantityValue, container_log_max_files: i32, + cpu_manager_policy: CpuManagerPolicy, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not. diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs index e1fd1b781c5..186fa0bc453 100644 --- a/sources/models/src/modeled_types/kubernetes.rs +++ b/sources/models/src/modeled_types/kubernetes.rs @@ -768,3 +768,52 @@ mod test_kubernetes_cloud_provider { } } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// CpuManagerPolicy represents a string that contains a valid cpu management policy. Default: none +/// https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/ + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct CpuManagerPolicy { + inner: String, +} +#[derive(Debug, Clone, Eq, PartialEq, Hash, Deserialize)] +#[serde(rename_all = "lowercase")] +enum ValidCpuManagerPolicy { + Static, + None, +} + +impl TryFrom<&str> for CpuManagerPolicy { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + serde_plain::from_str::(&input).context( + error::InvalidCpuManagerPolicy { input })?; + Ok(CpuManagerPolicy { + inner: input.to_string(), + }) + } +} +string_impls_for!(CpuManagerPolicy, "CpuManagerPolicy"); + +#[cfg(test)] +mod test_cpu_manager_policy { + use super::CpuManagerPolicy; + use std::convert::TryFrom; + + #[test] + fn good_cpu_manager_policy() { + for ok in &["static", "none"] { + CpuManagerPolicy::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_cpu_manager_policy() { + for err in &["", "bad", "100", &"a".repeat(64)] { + CpuManagerPolicy::try_from(*err).unwrap_err(); + } + } +} diff --git a/sources/models/src/modeled_types/mod.rs b/sources/models/src/modeled_types/mod.rs index 14df969cedc..8df67b809e7 100644 --- a/sources/models/src/modeled_types/mod.rs +++ b/sources/models/src/modeled_types/mod.rs @@ -74,6 +74,12 @@ pub mod error { input: String, source: std::num::ParseFloatError, }, + + #[snafu(display("Invalid Cpu Manager policy '{}'", input))] + InvalidCpuManagerPolicy { + input: String, + source: serde_plain::Error, + }, } } From c003262b8018ab6ec1ff2ed1a06061a4a578edee Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Tue, 29 Jun 2021 21:21:33 +0000 Subject: [PATCH 2/4] fix cpu-manager-policy unusual checkpoint upgrade: remove the cpu manager policy state file on boot downgrade: add a migration `kubelet-cpu-manager-state` to remove the cpu manager policy state file --- Release.toml | 3 ++ .../kubernetes-1.16/kubernetes-tmpfiles.conf | 1 + .../kubernetes-1.17/kubernetes-tmpfiles.conf | 1 + .../kubernetes-1.18/kubernetes-tmpfiles.conf | 1 + .../kubernetes-1.19/kubernetes-tmpfiles.conf | 1 + .../kubernetes-1.20/kubernetes-tmpfiles.conf | 1 + .../kubernetes-1.21/kubernetes-tmpfiles.conf | 1 + sources/Cargo.lock | 8 +++ sources/Cargo.toml | 1 + .../migration/migration-helpers/src/error.rs | 7 +++ .../kubelet-cpu-manager-state/Cargo.toml | 13 +++++ .../kubelet-cpu-manager-state/src/main.rs | 54 +++++++++++++++++++ 12 files changed, 92 insertions(+) create mode 100644 sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/Cargo.toml create mode 100644 sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/src/main.rs diff --git a/Release.toml b/Release.toml index 446a0f311f8..676f0c4c9a4 100644 --- a/Release.toml +++ b/Release.toml @@ -54,3 +54,6 @@ version = "1.1.2" "migrate_v1.1.2_admin-container-v0-7-1.lz4", "migrate_v1.1.2_control-container-v0-5-1.lz4", ] +"(1.1.2, 1.1.3)" = [ + "migrate_v1.1.3_kubelet-cpu-manager-state.lz4", +] diff --git a/packages/kubernetes-1.16/kubernetes-tmpfiles.conf b/packages/kubernetes-1.16/kubernetes-tmpfiles.conf index 8b18963ae64..63f34ee0780 100644 --- a/packages/kubernetes-1.16/kubernetes-tmpfiles.conf +++ b/packages/kubernetes-1.16/kubernetes-tmpfiles.conf @@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - - L /etc/kubernetes/manifests - - - - static-pods d /var/lib/kubelet/providers/secrets-store - - - - L /etc/kubernetes/secrets-store-csi-providers - - - - /var/lib/kubelet/providers/secrets-store +r! /var/lib/kubelet/cpu_manager_state diff --git a/packages/kubernetes-1.17/kubernetes-tmpfiles.conf b/packages/kubernetes-1.17/kubernetes-tmpfiles.conf index 8b18963ae64..63f34ee0780 100644 --- a/packages/kubernetes-1.17/kubernetes-tmpfiles.conf +++ b/packages/kubernetes-1.17/kubernetes-tmpfiles.conf @@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - - L /etc/kubernetes/manifests - - - - static-pods d /var/lib/kubelet/providers/secrets-store - - - - L /etc/kubernetes/secrets-store-csi-providers - - - - /var/lib/kubelet/providers/secrets-store +r! /var/lib/kubelet/cpu_manager_state diff --git a/packages/kubernetes-1.18/kubernetes-tmpfiles.conf b/packages/kubernetes-1.18/kubernetes-tmpfiles.conf index 8b18963ae64..63f34ee0780 100644 --- a/packages/kubernetes-1.18/kubernetes-tmpfiles.conf +++ b/packages/kubernetes-1.18/kubernetes-tmpfiles.conf @@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - - L /etc/kubernetes/manifests - - - - static-pods d /var/lib/kubelet/providers/secrets-store - - - - L /etc/kubernetes/secrets-store-csi-providers - - - - /var/lib/kubelet/providers/secrets-store +r! /var/lib/kubelet/cpu_manager_state diff --git a/packages/kubernetes-1.19/kubernetes-tmpfiles.conf b/packages/kubernetes-1.19/kubernetes-tmpfiles.conf index 8b18963ae64..63f34ee0780 100644 --- a/packages/kubernetes-1.19/kubernetes-tmpfiles.conf +++ b/packages/kubernetes-1.19/kubernetes-tmpfiles.conf @@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - - L /etc/kubernetes/manifests - - - - static-pods d /var/lib/kubelet/providers/secrets-store - - - - L /etc/kubernetes/secrets-store-csi-providers - - - - /var/lib/kubelet/providers/secrets-store +r! /var/lib/kubelet/cpu_manager_state diff --git a/packages/kubernetes-1.20/kubernetes-tmpfiles.conf b/packages/kubernetes-1.20/kubernetes-tmpfiles.conf index 8b18963ae64..63f34ee0780 100644 --- a/packages/kubernetes-1.20/kubernetes-tmpfiles.conf +++ b/packages/kubernetes-1.20/kubernetes-tmpfiles.conf @@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - - L /etc/kubernetes/manifests - - - - static-pods d /var/lib/kubelet/providers/secrets-store - - - - L /etc/kubernetes/secrets-store-csi-providers - - - - /var/lib/kubelet/providers/secrets-store +r! /var/lib/kubelet/cpu_manager_state diff --git a/packages/kubernetes-1.21/kubernetes-tmpfiles.conf b/packages/kubernetes-1.21/kubernetes-tmpfiles.conf index 8b18963ae64..63f34ee0780 100644 --- a/packages/kubernetes-1.21/kubernetes-tmpfiles.conf +++ b/packages/kubernetes-1.21/kubernetes-tmpfiles.conf @@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - - L /etc/kubernetes/manifests - - - - static-pods d /var/lib/kubelet/providers/secrets-store - - - - L /etc/kubernetes/secrets-store-csi-providers - - - - /var/lib/kubelet/providers/secrets-store +r! /var/lib/kubelet/cpu_manager_state diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 0937c40a289..a2f03d5a6ae 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1543,6 +1543,14 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "kubelet-cpu-manager-state" +version = "0.1.0" +dependencies = [ + "migration-helpers", + "snafu", +] + [[package]] name = "kubelet-event-qps-event-burst" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 2d20d725511..38a3be672fe 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -35,6 +35,7 @@ members = [ "api/migration/migrations/v1.1.2/kubelet-system-reserved", "api/migration/migrations/v1.1.2/admin-container-v0-7-1", "api/migration/migrations/v1.1.2/control-container-v0-5-1", + "api/migration/migrations/v1.1.3/kubelet-cpu-manager-state", "bottlerocket-release", diff --git a/sources/api/migration/migration-helpers/src/error.rs b/sources/api/migration/migration-helpers/src/error.rs index ed401815f81..d27adaaa6a2 100644 --- a/sources/api/migration/migration-helpers/src/error.rs +++ b/sources/api/migration/migration-helpers/src/error.rs @@ -1,6 +1,7 @@ //! Contains the Error and Result types used by the migration helper functions and migrations. use snafu::Snafu; +use std::path::PathBuf; /// Error contains the errors that can happen in the migration helper functions and in migrations. #[derive(Debug, Snafu)] @@ -99,6 +100,12 @@ pub enum Error { metadata: String, data: Vec, }, + + #[snafu(display("Failed to delete file '{}': '{}'", path.display(), source))] + RemoveFile { + path: PathBuf, + source: std::io::Error, + }, } /// Result alias containing our Error type. diff --git a/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/Cargo.toml b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/Cargo.toml new file mode 100644 index 00000000000..eeb8e4dcb08 --- /dev/null +++ b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "kubelet-cpu-manager-state" +version = "0.1.0" +authors = ["Tianhao Geng "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } +snafu = "0.6" diff --git a/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/src/main.rs b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/src/main.rs new file mode 100644 index 00000000000..5fecf90c559 --- /dev/null +++ b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/src/main.rs @@ -0,0 +1,54 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::{error, migrate, Migration, MigrationData, Result}; +use snafu::ResultExt; +use std::fs; +use std::io; +use std::process; + +const CPU_MANAGER_POLICY_CHECKPOINT: &str = "/var/lib/kubelet/cpu_manager_state"; + +/// forward - We always remove the state file on boot, therefore we don't need to explicitly +/// remove the file during forward migration. +/// backward - We remove cpu manager policy checkpoint value on downgrade, since older versions did not +/// clean up this state file on boot. +pub struct CpuManagerPolicyCleaner; + +impl Migration for CpuManagerPolicyCleaner { + fn forward(&mut self, input: MigrationData) -> Result { + println!("CpuManagerPolicyCleaner has no work to do on upgrade."); + Ok(input) + } + + fn backward(&mut self, input: MigrationData) -> Result { + // removing existing cpu_manager_policy_state file + println!( + "Deleting existing cpu manager policy checkpoint: '{}'", + CPU_MANAGER_POLICY_CHECKPOINT + ); + if let Err(e) = fs::remove_file(CPU_MANAGER_POLICY_CHECKPOINT) { + if e.kind() != io::ErrorKind::NotFound { + return Err(e).context(error::RemoveFile { + path: CPU_MANAGER_POLICY_CHECKPOINT, + }); + } else { + println!("NotFound: '{}'", CPU_MANAGER_POLICY_CHECKPOINT) + } + } + Ok(input) + } +} +/// We changed the default for CPU manager policy and need to handle kubelet's state file. +fn run() -> Result<()> { + migrate(CpuManagerPolicyCleaner) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} From db6c8d482486e080690f163bf2a66593f78aa3a7 Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Thu, 1 Jul 2021 21:07:37 +0000 Subject: [PATCH 3/4] kubelet: add setting for configuring cpuManagerReconcilePeriod pass cpu-manager-reconcile-period argument to kubelet --- README.md | 1 + packages/kubernetes-1.16/kubelet-config | 3 + packages/kubernetes-1.17/kubelet-config | 3 + packages/kubernetes-1.18/kubelet-config | 3 + packages/kubernetes-1.19/kubelet-config | 3 + packages/kubernetes-1.20/kubelet-config | 3 + packages/kubernetes-1.21/kubelet-config | 3 + sources/models/src/lib.rs | 3 +- .../models/src/modeled_types/kubernetes.rs | 73 +++++++++++++++++++ sources/models/src/modeled_types/mod.rs | 3 + 10 files changed, 97 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 752fa700f97..62b93684ba2 100644 --- a/README.md +++ b/README.md @@ -359,6 +359,7 @@ The following settings are optional and allow you to further configure your clus * `settings.kubernetes.container-log-max-size`: The maximum size of container log file before it is rotated. * `settings.kubernetes.container-log-max-files`: The maximum number of container log files that can be present for a container. * `settings.kubernetes.cpu-manager-policy`: Specifies the CPU manager policy. Possible values are `static` and `none`. Defaults to `none`. If you want to allow pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node, you can set this setting to `static`. You should reboot if you change this setting after startup - try `apiclient reboot`. +* `settings.kubernetes.cpu-manager-reconcile-period`: Specifies the CPU manager reconcile period, which controls how often updated CPU assignments are written to cgroupfs. The value is a duration like `30s` for 30 seconds or `1h5m` for 1 hour and 5 minutes. You can also optionally specify static pods for your node with the following settings. Static pods can be particularly useful when running in standalone mode. diff --git a/packages/kubernetes-1.16/kubelet-config b/packages/kubernetes-1.16/kubelet-config index d934102e5db..de71a2ad73d 100644 --- a/packages/kubernetes-1.16/kubelet-config +++ b/packages/kubernetes-1.16/kubelet-config @@ -75,6 +75,9 @@ systemReserved: {{~/each}} {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} +{{~#if settings.kubernetes.cpu-manager-reconcile-period}} +cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} +{{~/if}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.17/kubelet-config b/packages/kubernetes-1.17/kubelet-config index 5fbd58e4b0e..f72becb2acf 100644 --- a/packages/kubernetes-1.17/kubelet-config +++ b/packages/kubernetes-1.17/kubelet-config @@ -75,6 +75,9 @@ systemReserved: {{~/each}} {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} +{{~#if settings.kubernetes.cpu-manager-reconcile-period}} +cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} +{{~/if}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.18/kubelet-config b/packages/kubernetes-1.18/kubelet-config index 5fbd58e4b0e..f72becb2acf 100644 --- a/packages/kubernetes-1.18/kubelet-config +++ b/packages/kubernetes-1.18/kubelet-config @@ -75,6 +75,9 @@ systemReserved: {{~/each}} {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} +{{~#if settings.kubernetes.cpu-manager-reconcile-period}} +cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} +{{~/if}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.19/kubelet-config b/packages/kubernetes-1.19/kubelet-config index 3fe27e1adc4..bd81a83c6e9 100644 --- a/packages/kubernetes-1.19/kubelet-config +++ b/packages/kubernetes-1.19/kubelet-config @@ -75,6 +75,9 @@ systemReserved: {{~/each}} {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} +{{~#if settings.kubernetes.cpu-manager-reconcile-period}} +cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} +{{~/if}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.20/kubelet-config b/packages/kubernetes-1.20/kubelet-config index 3fe27e1adc4..bd81a83c6e9 100644 --- a/packages/kubernetes-1.20/kubelet-config +++ b/packages/kubernetes-1.20/kubelet-config @@ -75,6 +75,9 @@ systemReserved: {{~/each}} {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} +{{~#if settings.kubernetes.cpu-manager-reconcile-period}} +cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} +{{~/if}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/packages/kubernetes-1.21/kubelet-config b/packages/kubernetes-1.21/kubelet-config index 3fe27e1adc4..bd81a83c6e9 100644 --- a/packages/kubernetes-1.21/kubelet-config +++ b/packages/kubernetes-1.21/kubelet-config @@ -75,6 +75,9 @@ systemReserved: {{~/each}} {{~/if}} cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} +{{~#if settings.kubernetes.cpu-manager-reconcile-period}} +cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} +{{~/if}} resolvConf: "/etc/resolv.conf" hairpinMode: hairpin-veth readOnlyPort: 0 diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index bbed738319f..fe26586163e 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -115,7 +115,7 @@ use std::net::Ipv4Addr; use crate::modeled_types::{ BootstrapContainerMode, CpuManagerPolicy, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, FriendlyVersion, Identifier, KubernetesAuthenticationMode, KubernetesBootstrapToken, - KubernetesCloudProvider, KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey, + KubernetesCloudProvider, KubernetesClusterName, KubernetesDurationValue, KubernetesEvictionHardKey, KubernetesLabelKey, KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue, KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url, ValidBase64, @@ -159,6 +159,7 @@ struct KubernetesSettings { container_log_max_size: KubernetesQuantityValue, container_log_max_files: i32, cpu_manager_policy: CpuManagerPolicy, + cpu_manager_reconcile_period: KubernetesDurationValue, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not. diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs index 186fa0bc453..8cabe3811c9 100644 --- a/sources/models/src/modeled_types/kubernetes.rs +++ b/sources/models/src/modeled_types/kubernetes.rs @@ -817,3 +817,76 @@ mod test_cpu_manager_policy { } } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesDurationValue represents a string that contains a valid Kubernetes duration value. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesDurationValue { + inner: String, +} + +lazy_static! { + pub(crate) static ref KUBERNETES_DURATION_VALUE: Regex = Regex::new( + r"^(([0-9]+\.)?[0-9]+h)?(([0-9]+\.)?[0-9]+m)?(([0-9]+\.)?[0-9]+s)?(([0-9]+\.)?[0-9]+ms)?$" + ) + .unwrap(); +} + +impl TryFrom<&str> for KubernetesDurationValue { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + !input.is_empty(), + error::InvalidKubernetesDurationValue { input } + ); + ensure!( + KUBERNETES_DURATION_VALUE.is_match(input), + error::InvalidKubernetesDurationValue { input } + ); + Ok(KubernetesDurationValue { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesDurationValue, "KubernetesDurationValue"); + +#[cfg(test)] +mod test_kubernetes_duration_value { + use super::KubernetesDurationValue; + use std::convert::TryFrom; + + #[test] + fn good_tokens() { + for ok in &[ + "9ms", + "99s", + "20m", + "1h", + "1h2m3s10ms", + "4m5s10ms", + "2h3s10ms", + "1.5h3.5m", + ] { + KubernetesDurationValue::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_names() { + for err in &[ + "", + "100", + "...3ms", + "1..5s", + "ten second", + "1m2h", + "9ns", + &"a".repeat(23), + ] { + KubernetesDurationValue::try_from(*err).unwrap_err(); + } + } +} diff --git a/sources/models/src/modeled_types/mod.rs b/sources/models/src/modeled_types/mod.rs index 8df67b809e7..25344122df2 100644 --- a/sources/models/src/modeled_types/mod.rs +++ b/sources/models/src/modeled_types/mod.rs @@ -80,6 +80,9 @@ pub mod error { input: String, source: serde_plain::Error, }, + + #[snafu(display("Invalid Kubernetes duration value '{}'", input))] + InvalidKubernetesDurationValue { input: String }, } } From 48638ed18a9901ff6296bbb8ab2c5d849de7df94 Mon Sep 17 00:00:00 2001 From: Tianhao Geng Date: Fri, 2 Jul 2021 19:16:06 +0000 Subject: [PATCH 4/4] migrations: add migrations for cpu-manager-policy and cpu-manager-reconcile-period adds migration for two new settings `settings.kubernetes.cpu-manager-policy` and `settings.kubernetes.cpu-manager-reconcile-period` --- Release.toml | 1 + sources/Cargo.lock | 7 ++++++ sources/Cargo.toml | 1 + .../v1.1.3/kubelet-cpu-manager/Cargo.toml | 12 ++++++++++ .../v1.1.3/kubelet-cpu-manager/src/main.rs | 24 +++++++++++++++++++ 5 files changed, 45 insertions(+) create mode 100644 sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/Cargo.toml create mode 100644 sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/src/main.rs diff --git a/Release.toml b/Release.toml index 676f0c4c9a4..65185f57fe3 100644 --- a/Release.toml +++ b/Release.toml @@ -56,4 +56,5 @@ version = "1.1.2" ] "(1.1.2, 1.1.3)" = [ "migrate_v1.1.3_kubelet-cpu-manager-state.lz4", + "migrate_v1.1.3_kubelet-cpu-manager.lz4", ] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index a2f03d5a6ae..aff41b4405b 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1543,6 +1543,13 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "kubelet-cpu-manager" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "kubelet-cpu-manager-state" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 38a3be672fe..0d21508636c 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -36,6 +36,7 @@ members = [ "api/migration/migrations/v1.1.2/admin-container-v0-7-1", "api/migration/migrations/v1.1.2/control-container-v0-5-1", "api/migration/migrations/v1.1.3/kubelet-cpu-manager-state", + "api/migration/migrations/v1.1.3/kubelet-cpu-manager", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/Cargo.toml b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/Cargo.toml new file mode 100644 index 00000000000..46e91cfd799 --- /dev/null +++ b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kubelet-cpu-manager" +version = "0.1.0" +authors = ["Tianhao Geng "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } diff --git a/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/src/main.rs b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/src/main.rs new file mode 100644 index 00000000000..eda792dabfe --- /dev/null +++ b/sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/src/main.rs @@ -0,0 +1,24 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::common_migrations::AddSettingsMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We added two new settings for configuring kubelet, `settings.kubernetes.cpu-manager-reconcile-period` +/// and `settings.kubernetes.cpu-manager-policy` +fn run() -> Result<()> { + migrate(AddSettingsMigration(&[ + "settings.kubernetes.cpu-manager-policy", + "settings.kubernetes.cpu-manager-reconcile-period", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +}