Add a vmware-k8s-1.20 variant

.github/workflows/build.yml

@@ -20,6 +20,9 @@ jobs:
           - variant: vmware-dev
             arch: x86_64
             supported: false
+          - variant: vmware-k8s-1.20
+            arch: x86_64
+            supported: true
       fail-fast: false
     steps:
       - uses: actions/checkout@v2

README.md

@@ -60,6 +60,10 @@ We also have a variant designed to work with ECS, currently in preview:
 
 - `aws-ecs-1`
 
+Another variant we have in preview is designed to be a Kubernetes worker node in VMware:
+
+- `vmware-k8s-1.20`
+
 The `aws-k8s-1.15` variant is deprecated and will no longer be supported in Bottlerocket releases.
 We recommend users replace `aws-k8s-1.15` nodes with the [latest variant compatible with their cluster](variants/).
 

packages/os/os.spec

@@ -1,5 +1,6 @@
 %global _cross_first_party 1
-%global _is_k8s_variant %(if echo %{_cross_variant} | grep -q "k8s"; then echo 1; else echo 0; fi)
+%global _is_k8s_variant %(if echo %{_cross_variant} | grep -Fqw "k8s"; then echo 1; else echo 0; fi)
+%global _is_aws_variant %(if echo %{_cross_variant} | grep -Fqw "aws"; then echo 1; else echo 0; fi)
 %undefine _debugsource_packages
 
 Name: %{_cross_os}os
@@ -72,7 +73,9 @@ Requires: %{_cross_os}thar-be-updates
 Requires: %{_cross_os}updog
 
 %if %{_is_k8s_variant}
+%if %{_is_aws_variant}
 Requires: %{_cross_os}pluto
+%endif
 Requires: %{_cross_os}static-pods
 %endif
 
@@ -206,10 +209,12 @@ Summary: Settings generator for ECS
 %endif
 
 %if %{_is_k8s_variant}
+%if %{_is_aws_variant}
 %package -n %{_cross_os}pluto
 Summary: Dynamic setting generator for kubernetes
 %description -n %{_cross_os}pluto
 %{summary}.
+%endif
 
 %package -n %{_cross_os}static-pods
 Summary: Manages user-defined K8S static pods
@@ -284,7 +289,9 @@ echo "** Output from non-static builds:"
     -p ecs-settings-applier \
 %endif
 %if %{_is_k8s_variant}
+%if %{_is_aws_variant}
     -p pluto \
+%endif
     -p static-pods \
 %endif
     %{nil}
@@ -311,7 +318,10 @@ for p in \
   ecs-settings-applier \
 %endif
 %if %{_is_k8s_variant}
-  pluto static-pods \
+%if %{_is_aws_variant}
+  pluto \
+%endif
+  static-pods \
 %endif
 ; do
   install -p -m 0755 ${HOME}/.cache/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir}
@@ -348,9 +358,11 @@ install -d %{buildroot}%{_cross_sysusersdir}
 install -p -m 0644 %{S:2} %{buildroot}%{_cross_sysusersdir}/api.conf
 
 %if %{_is_k8s_variant}
+%if %{_is_aws_variant}
 install -d %{buildroot}%{_cross_datadir}/eks
 install -p -m 0644 %{S:3} %{buildroot}%{_cross_datadir}/eks
 %endif
+%endif
 
 install -d %{buildroot}%{_cross_datadir}/updog
 install -p -m 0644 %{_cross_repo_root_json} %{buildroot}%{_cross_datadir}/updog
@@ -479,10 +491,12 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor
 %endif
 
 %if %{_is_k8s_variant}
+%if %{_is_aws_variant}
 %files -n %{_cross_os}pluto
 %{_cross_bindir}/pluto
 %dir %{_cross_datadir}/eks
 %{_cross_datadir}/eks/eni-max-pods
+%endif
 
 %files -n %{_cross_os}static-pods
 %{_cross_bindir}/static-pods

sources/logdog/conf/logdog.vmware-k8s-1.20.conf

@@ -0,0 +1 @@
+vmware-k8s.conf

sources/logdog/conf/vmware-k8s.conf

@@ -0,0 +1 @@
+exec kube-status systemctl status kube* -l --no-pager

sources/models/README.md

@@ -62,6 +62,11 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and
 * [Model](src/vmware-dev/mod.rs)
 * [Default settings](src/vmware-dev/defaults.d/)
 
+### vmware-k8s-1.20: VMware Kubernetes 1.20
+
+* [Model](src/vmware-k8s-1.20/mod.rs)
+* [Default settings](src/vmware-k8s-1.20/defaults.d/)
+
 ## This directory
 
 We use `build.rs` to symlink the proper API model source code for Cargo to build.

sources/models/shared-defaults/kubernetes-vmware.toml

@@ -0,0 +1,19 @@
+[settings.kubernetes]
+cluster-domain = "cluster.local"
+standalone-mode = false
+authentication-mode = "tls"
+pod-infra-container-image = "k8s.gcr.io/pause:3.2"
+server-tls-bootstrap = false
+cloud-provider = "external"
+
+[metadata.settings.kubernetes]
+node-ip.setting-generator = "netdog node-ip"
+affected-services = ["kubernetes"]
+
+# Metrics
+[settings.metrics]
+service-checks = ["apiserver", "chronyd", "containerd", "host-containerd", "kubelet", "vmtoolsd"]
+
+# Network
+[metadata.settings.network]
+affected-services = ["containerd", "kubernetes", "host-containerd", "host-containers"]

sources/models/shared-defaults/vmware-host-containers.toml

@@ -0,0 +1,14 @@
+# Both containers are disabled by default in VMware because the user must
+# supply user data in order to use the containers.  The admin container isn't
+# useful without SSH keys/CA certs, and the control container can only be used
+# with hybrid SSM off of AWS.  VMware users might not want to use either of
+# those options.
+[settings.host-containers.admin]
+enabled = false
+superpowered = true
+source = "public.ecr.aws/bottlerocket/bottlerocket-admin:v0.7.0"
+
+[settings.host-containers.control]
+enabled = false
+superpowered = false
+source = "public.ecr.aws/bottlerocket/bottlerocket-control:v0.5.0"

sources/models/src/lib.rs

@@ -59,6 +59,11 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and
 * [Model](src/vmware-dev/mod.rs)
 * [Default settings](src/vmware-dev/defaults.d/)
 
+## vmware-k8s-1.20: VMware Kubernetes 1.20
+
+* [Model](src/vmware-k8s-1.20/mod.rs)
+* [Default settings](src/vmware-k8s-1.20/defaults.d/)
+
 # This directory
 
 We use `build.rs` to symlink the proper API model source code for Cargo to build.

sources/models/src/vmware-k8s-1.20/defaults.d/10-defaults.toml

@@ -0,0 +1 @@
+../../../shared-defaults/defaults.toml

sources/models/src/vmware-k8s-1.20/defaults.d/20-vmware-host-containers.toml

@@ -0,0 +1 @@
+../../../shared-defaults/vmware-host-containers.toml

sources/models/src/vmware-k8s-1.20/defaults.d/30-metrics.toml

@@ -0,0 +1 @@
+../../../shared-defaults/metrics.toml

sources/models/src/vmware-k8s-1.20/defaults.d/50-kubernetes-vmware.toml

@@ -0,0 +1 @@
+../../../shared-defaults/kubernetes-vmware.toml

sources/models/src/vmware-k8s-1.20/defaults.d/51-kubernetes-containerd.toml

@@ -0,0 +1 @@
+../../../shared-defaults/kubernetes-containerd.toml

sources/models/src/vmware-k8s-1.20/defaults.d/52-kubernetes-services.toml

@@ -0,0 +1 @@
+../../../shared-defaults/kubernetes-services.toml

sources/models/src/vmware-k8s-1.20/defaults.d/60-lockdown-integrity.toml

@@ -0,0 +1 @@
+../../../shared-defaults/lockdown-integrity.toml

sources/models/src/vmware-k8s-1.20/defaults.d/70-public-ntp.toml

@@ -0,0 +1 @@
+../../../shared-defaults/public-ntp.toml

sources/models/src/vmware-k8s-1.20/mod.rs

@@ -0,0 +1,24 @@
+use model_derive::model;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+use crate::modeled_types::Identifier;
+use crate::{
+    BootstrapContainer, HostContainer, KernelSettings, KubernetesSettings, MetricsSettings,
+    NetworkSettings, NtpSettings, UpdatesSettings,
+};
+
+// Note: we have to use 'rename' here because the top-level Settings structure is the only one
+// that uses its name in serialization; internal structures use the field name that points to it
+#[model(rename = "settings", impl_default = true)]
+struct Settings {
+    motd: String,
+    kubernetes: KubernetesSettings,
+    updates: UpdatesSettings,
+    host_containers: HashMap<Identifier, HostContainer>,
+    bootstrap_containers: HashMap<Identifier, BootstrapContainer>,
+    ntp: NtpSettings,
+    network: NetworkSettings,
+    kernel: KernelSettings,
+    metrics: MetricsSettings,
+}

variants/Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "aws-k8s-1.19",
     "aws-k8s-1.20",
     "vmware-dev",
+    "vmware-k8s-1.20",
 ]
 
 [profile.dev]

variants/README.md

@@ -77,6 +77,15 @@ It includes tools for troubleshooting as well as Docker for running containers.
 User data will be read from a mounted CD-ROM (from a file named "user-data" or from an OVF file), and from VMware's guestinfo interface.
 If user data exists at both places, settings read from guestinfo will override identical settings from CD-ROM.
 
+### vmware-k8s-1.20: VMware Kubernetes 1.20 node
+
+The [vmware-k8s-1.20](vmware-k8s-1.20/Cargo.toml) variant includes the packages needed to run a Kubernetes worker node as a VMware guest.
+It supports self-hosted clusters.
+User data will be read from a mounted CD-ROM (from a file named "user-data" or from an OVF file), and from VMware's guestinfo interface.
+If user data exists at both places, settings read from guestinfo will override identical settings from CD-ROM.
+
+This variant is compatible with Kubernetes 1.20, 1.21, and 1.22 clusters.
+
 ### Deprecated variants
 
 #### aws-k8s-1.15: Kubernetes 1.15 node

variants/vmware-k8s-1.20/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+# This is the vmware-k8s-1.20 variant.  "." is not allowed in crate names, but
+# we don't use this crate name anywhere.
+name = "vmware-k8s-1_20"
+version = "0.1.0"
+edition = "2018"
+publish = false
+build = "build.rs"
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[package.metadata.build-variant]
+image-format = "vmdk"
+supported-arches = ["x86_64"]
+kernel-parameters = [
+    "console=ttyS0",
+    "console=tty1",
+]
+included-packages = [
+    "cni",
+    "cni-plugins",
+    "kernel-5.10",
+    "kubelet-1.20",
+    "open-vm-tools",
+    "release",
+]
+
+[lib]
+path = "lib.rs"
+
+[build-dependencies]
+cni = { path = "../../packages/cni" }
+cni-plugins = { path = "../../packages/cni-plugins" }
+kernel-5_10 = { path = "../../packages/kernel-5.10" }
+kubernetes-1_20 = { path = "../../packages/kubernetes-1.20" }
+open-vm-tools = { path = "../../packages/open-vm-tools" }
+release = { path = "../../packages/release" }

variants/vmware-k8s-1.20/build.rs

@@ -0,0 +1,9 @@
+use std::process::{exit, Command};
+
+fn main() -> Result<(), std::io::Error> {
+    let ret = Command::new("buildsys").arg("build-variant").status()?;
+    if !ret.success() {
+        exit(1);
+    }
+    Ok(())
+}

variants/vmware-k8s-1.20/lib.rs

@@ -0,0 +1 @@
+// not used

variants/vmware-k8s-1.20/template.ovf

@@ -0,0 +1 @@
+../vmware-dev/template.ovf