Merge pull request #1638 from gthao313/kubelet-settings

kubelet: add setting for configuring cpuManagerPolicy and cpuManagerReconcilePeriod

README.md

@@ -356,6 +356,8 @@ The following settings are optional and allow you to further configure your clus
 * `settings.kubernetes.kube-api-burst`: The burst to allow while talking with kubernetes.
 * `settings.kubernetes.container-log-max-size`: The maximum size of container log file before it is rotated.
 * `settings.kubernetes.container-log-max-files`: The maximum number of container log files that can be present for a container.
+* `settings.kubernetes.cpu-manager-policy`: Specifies the CPU manager policy. Possible values are `static` and `none`. Defaults to `none`. If you want to allow pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node, you can set this setting to `static`. You should reboot if you change this setting after startup - try `apiclient reboot`.
+* `settings.kubernetes.cpu-manager-reconcile-period`: Specifies the CPU manager reconcile period, which controls how often updated CPU assignments are written to cgroupfs. The value is a duration like `30s` for 30 seconds or `1h5m` for 1 hour and 5 minutes.
 
 You can also optionally specify static pods for your node with the following settings.
 Static pods can be particularly useful when running in standalone mode.

Release.toml

@@ -54,3 +54,7 @@ version = "1.1.2"
     "migrate_v1.1.2_admin-container-v0-7-1.lz4",
     "migrate_v1.1.2_control-container-v0-5-1.lz4",
 ]
+"(1.1.2, 1.1.3)" = [
+    "migrate_v1.1.3_kubelet-cpu-manager-state.lz4",
+    "migrate_v1.1.3_kubelet-cpu-manager.lz4",
+]

packages/kubernetes-1.16/kubelet-config

@@ -74,6 +74,10 @@ systemReserved:
   {{@key}}: "{{this}}"
   {{~/each}}
 {{~/if}}
+cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
+{{~#if settings.kubernetes.cpu-manager-reconcile-period}}
+cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

packages/kubernetes-1.16/kubernetes-tmpfiles.conf

@@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - -
 L /etc/kubernetes/manifests - - - - static-pods
 d /var/lib/kubelet/providers/secrets-store - - - -
 L /etc/kubernetes/secrets-store-csi-providers  - - - - /var/lib/kubelet/providers/secrets-store
+r! /var/lib/kubelet/cpu_manager_state

packages/kubernetes-1.17/kubelet-config

@@ -74,6 +74,10 @@ systemReserved:
   {{@key}}: "{{this}}"
   {{~/each}}
 {{~/if}}
+cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
+{{~#if settings.kubernetes.cpu-manager-reconcile-period}}
+cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

packages/kubernetes-1.17/kubernetes-tmpfiles.conf

@@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - -
 L /etc/kubernetes/manifests - - - - static-pods
 d /var/lib/kubelet/providers/secrets-store - - - -
 L /etc/kubernetes/secrets-store-csi-providers  - - - - /var/lib/kubelet/providers/secrets-store
+r! /var/lib/kubelet/cpu_manager_state

packages/kubernetes-1.18/kubelet-config

@@ -74,6 +74,10 @@ systemReserved:
   {{@key}}: "{{this}}"
   {{~/each}}
 {{~/if}}
+cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
+{{~#if settings.kubernetes.cpu-manager-reconcile-period}}
+cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

packages/kubernetes-1.18/kubernetes-tmpfiles.conf

@@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - -
 L /etc/kubernetes/manifests - - - - static-pods
 d /var/lib/kubelet/providers/secrets-store - - - -
 L /etc/kubernetes/secrets-store-csi-providers  - - - - /var/lib/kubelet/providers/secrets-store
+r! /var/lib/kubelet/cpu_manager_state

packages/kubernetes-1.19/kubelet-config

@@ -74,6 +74,10 @@ systemReserved:
   {{@key}}: "{{this}}"
   {{~/each}}
 {{~/if}}
+cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
+{{~#if settings.kubernetes.cpu-manager-reconcile-period}}
+cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

packages/kubernetes-1.19/kubernetes-tmpfiles.conf

@@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - -
 L /etc/kubernetes/manifests - - - - static-pods
 d /var/lib/kubelet/providers/secrets-store - - - -
 L /etc/kubernetes/secrets-store-csi-providers  - - - - /var/lib/kubelet/providers/secrets-store
+r! /var/lib/kubelet/cpu_manager_state

packages/kubernetes-1.20/kubelet-config

@@ -74,7 +74,10 @@ systemReserved:
   {{@key}}: "{{this}}"
   {{~/each}}
 {{~/if}}
-cpuManagerPolicy: "static"
+cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
+{{~#if settings.kubernetes.cpu-manager-reconcile-period}}
+cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

packages/kubernetes-1.20/kubernetes-tmpfiles.conf

@@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - -
 L /etc/kubernetes/manifests - - - - static-pods
 d /var/lib/kubelet/providers/secrets-store - - - -
 L /etc/kubernetes/secrets-store-csi-providers  - - - - /var/lib/kubelet/providers/secrets-store
+r! /var/lib/kubelet/cpu_manager_state

packages/kubernetes-1.21/kubelet-config

@@ -74,7 +74,10 @@ systemReserved:
   {{@key}}: "{{this}}"
   {{~/each}}
 {{~/if}}
-cpuManagerPolicy: "static"
+cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
+{{~#if settings.kubernetes.cpu-manager-reconcile-period}}
+cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
+{{~/if}}
 resolvConf: "/etc/resolv.conf"
 hairpinMode: hairpin-veth
 readOnlyPort: 0

packages/kubernetes-1.21/kubernetes-tmpfiles.conf

@@ -2,3 +2,4 @@ d /etc/kubernetes/static-pods - - - -
 L /etc/kubernetes/manifests - - - - static-pods
 d /var/lib/kubelet/providers/secrets-store - - - -
 L /etc/kubernetes/secrets-store-csi-providers  - - - - /var/lib/kubelet/providers/secrets-store
+r! /var/lib/kubelet/cpu_manager_state

sources/Cargo.toml

@@ -35,6 +35,8 @@ members = [
     "api/migration/migrations/v1.1.2/kubelet-system-reserved",
     "api/migration/migrations/v1.1.2/admin-container-v0-7-1",
     "api/migration/migrations/v1.1.2/control-container-v0-5-1",
+    "api/migration/migrations/v1.1.3/kubelet-cpu-manager-state",
+    "api/migration/migrations/v1.1.3/kubelet-cpu-manager",
 
     "bottlerocket-release",
 

sources/api/migration/migration-helpers/src/error.rs

@@ -1,6 +1,7 @@
 //! Contains the Error and Result types used by the migration helper functions and migrations.
 
 use snafu::Snafu;
+use std::path::PathBuf;
 
 /// Error contains the errors that can happen in the migration helper functions and in migrations.
 #[derive(Debug, Snafu)]
@@ -99,6 +100,12 @@ pub enum Error {
         metadata: String,
         data: Vec<serde_json::Value>,
     },
+
+    #[snafu(display("Failed to delete file '{}': '{}'", path.display(), source))]
+    RemoveFile {
+        path: PathBuf,
+        source: std::io::Error,
+    },
 }
 
 /// Result alias containing our Error type.

sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "kubelet-cpu-manager-state"
+version = "0.1.0"
+authors = ["Tianhao Geng <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers" }
+snafu = "0.6"

sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager-state/src/main.rs

@@ -0,0 +1,54 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::{error, migrate, Migration, MigrationData, Result};
+use snafu::ResultExt;
+use std::fs;
+use std::io;
+use std::process;
+
+const CPU_MANAGER_POLICY_CHECKPOINT: &str = "/var/lib/kubelet/cpu_manager_state";
+
+/// forward - We always remove the state file on boot, therefore we don't need to explicitly
+/// remove the file during forward migration.
+/// backward - We remove cpu manager policy checkpoint value on downgrade, since older versions did not
+/// clean up this state file on boot.
+pub struct CpuManagerPolicyCleaner;
+
+impl Migration for CpuManagerPolicyCleaner {
+    fn forward(&mut self, input: MigrationData) -> Result<MigrationData> {
+        println!("CpuManagerPolicyCleaner has no work to do on upgrade.");
+        Ok(input)
+    }
+
+    fn backward(&mut self, input: MigrationData) -> Result<MigrationData> {
+        // removing existing cpu_manager_policy_state file
+        println!(
+            "Deleting existing cpu manager policy checkpoint: '{}'",
+            CPU_MANAGER_POLICY_CHECKPOINT
+        );
+        if let Err(e) = fs::remove_file(CPU_MANAGER_POLICY_CHECKPOINT) {
+            if e.kind() != io::ErrorKind::NotFound {
+                return Err(e).context(error::RemoveFile {
+                    path: CPU_MANAGER_POLICY_CHECKPOINT,
+                });
+            } else {
+                println!("NotFound: '{}'", CPU_MANAGER_POLICY_CHECKPOINT)
+            }
+        }
+        Ok(input)
+    }
+}
+/// We changed the default for CPU manager policy and need to handle kubelet's state file.
+fn run() -> Result<()> {
+    migrate(CpuManagerPolicyCleaner)
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}

sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "kubelet-cpu-manager"
+version = "0.1.0"
+authors = ["Tianhao Geng <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers" }

sources/api/migration/migrations/v1.1.3/kubelet-cpu-manager/src/main.rs

@@ -0,0 +1,24 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::common_migrations::AddSettingsMigration;
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// We added two new settings for configuring kubelet, `settings.kubernetes.cpu-manager-reconcile-period`
+/// and `settings.kubernetes.cpu-manager-policy`
+fn run() -> Result<()> {
+    migrate(AddSettingsMigration(&[
+        "settings.kubernetes.cpu-manager-policy",
+        "settings.kubernetes.cpu-manager-reconcile-period",
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}

sources/models/src/lib.rs

@@ -113,9 +113,9 @@ use std::collections::HashMap;
 use std::net::Ipv4Addr;
 
 use crate::modeled_types::{
-    BootstrapContainerMode, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue,
+    BootstrapContainerMode, CpuManagerPolicy, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue,
     FriendlyVersion, Identifier, KubernetesAuthenticationMode, KubernetesBootstrapToken,
-    KubernetesCloudProvider, KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey,
+    KubernetesCloudProvider, KubernetesClusterName, KubernetesDurationValue, KubernetesEvictionHardKey, KubernetesLabelKey,
     KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey,
     KubernetesTaintValue, KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url,
     ValidBase64,
@@ -158,6 +158,8 @@ struct KubernetesSettings {
     kube_api_burst: i32,
     container_log_max_size: KubernetesQuantityValue,
     container_log_max_files: i32,
+    cpu_manager_policy: CpuManagerPolicy,
+    cpu_manager_reconcile_period: KubernetesDurationValue,
 
     // Settings where we generate a value based on the runtime environment.  The user can specify a
     // value to override the generated one, but typically would not.