From 4af05d1740e2c1dcd639021eb3515b13b15ef3cf Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Mon, 19 Apr 2021 18:42:15 +0000 Subject: [PATCH] Add a vmware-k8s-1.20 variant This change adds the necessary files for a VMware Kubernetes 1.20 variant. --- .github/workflows/build.yml | 3 ++ README.md | 4 ++ packages/os/os.spec | 18 ++++++++- .../logdog/conf/logdog.vmware-k8s-1.20.conf | 1 + sources/logdog/conf/vmware-k8s.conf | 1 + sources/models/README.md | 5 +++ .../shared-defaults/kubernetes-vmware.toml | 19 ++++++++++ .../vmware-host-containers.toml | 14 +++++++ sources/models/src/lib.rs | 5 +++ .../defaults.d/10-defaults.toml | 1 + .../defaults.d/20-vmware-host-containers.toml | 1 + .../defaults.d/30-metrics.toml | 1 + .../defaults.d/50-kubernetes-vmware.toml | 1 + .../defaults.d/51-kubernetes-containerd.toml | 1 + .../defaults.d/52-kubernetes-services.toml | 1 + .../defaults.d/60-lockdown-integrity.toml | 1 + .../defaults.d/70-public-ntp.toml | 1 + sources/models/src/vmware-k8s-1.20/mod.rs | 24 ++++++++++++ variants/Cargo.lock | 12 ++++++ variants/Cargo.toml | 1 + variants/README.md | 9 +++++ variants/vmware-k8s-1.20/Cargo.toml | 37 +++++++++++++++++++ variants/vmware-k8s-1.20/build.rs | 9 +++++ variants/vmware-k8s-1.20/lib.rs | 1 + variants/vmware-k8s-1.20/template.ovf | 1 + 25 files changed, 170 insertions(+), 2 deletions(-) create mode 120000 sources/logdog/conf/logdog.vmware-k8s-1.20.conf create mode 100644 sources/logdog/conf/vmware-k8s.conf create mode 100644 sources/models/shared-defaults/kubernetes-vmware.toml create mode 100644 sources/models/shared-defaults/vmware-host-containers.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/10-defaults.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/20-vmware-host-containers.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/30-metrics.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/50-kubernetes-vmware.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/51-kubernetes-containerd.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/52-kubernetes-services.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/60-lockdown-integrity.toml create mode 120000 sources/models/src/vmware-k8s-1.20/defaults.d/70-public-ntp.toml create mode 100644 sources/models/src/vmware-k8s-1.20/mod.rs create mode 100644 variants/vmware-k8s-1.20/Cargo.toml create mode 100644 variants/vmware-k8s-1.20/build.rs create mode 100644 variants/vmware-k8s-1.20/lib.rs create mode 120000 variants/vmware-k8s-1.20/template.ovf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e177f8461c1..6cc38417dac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,9 @@ jobs: - variant: vmware-dev arch: x86_64 supported: false + - variant: vmware-k8s-1.20 + arch: x86_64 + supported: true fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index e90904b56f3..ca6929f0bca 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,10 @@ We also have a variant designed to work with ECS, currently in preview: - `aws-ecs-1` +Another variant we have in preview is designed to be a Kubernetes worker node in VMware: + +- `vmware-k8s-1.20` + The `aws-k8s-1.15` variant is deprecated and will no longer be supported in Bottlerocket releases. We recommend users replace `aws-k8s-1.15` nodes with the [latest variant compatible with their cluster](variants/). diff --git a/packages/os/os.spec b/packages/os/os.spec index 4317bb85cbe..8aee63d80d6 100644 --- a/packages/os/os.spec +++ b/packages/os/os.spec @@ -1,5 +1,6 @@ %global _cross_first_party 1 -%global _is_k8s_variant %(if echo %{_cross_variant} | grep -q "k8s"; then echo 1; else echo 0; fi) +%global _is_k8s_variant %(if echo %{_cross_variant} | grep -Fqw "k8s"; then echo 1; else echo 0; fi) +%global _is_aws_variant %(if echo %{_cross_variant} | grep -Fqw "aws"; then echo 1; else echo 0; fi) %undefine _debugsource_packages Name: %{_cross_os}os @@ -72,7 +73,9 @@ Requires: %{_cross_os}thar-be-updates Requires: %{_cross_os}updog %if %{_is_k8s_variant} +%if %{_is_aws_variant} Requires: %{_cross_os}pluto +%endif Requires: %{_cross_os}static-pods %endif @@ -206,10 +209,12 @@ Summary: Settings generator for ECS %endif %if %{_is_k8s_variant} +%if %{_is_aws_variant} %package -n %{_cross_os}pluto Summary: Dynamic setting generator for kubernetes %description -n %{_cross_os}pluto %{summary}. +%endif %package -n %{_cross_os}static-pods Summary: Manages user-defined K8S static pods @@ -284,7 +289,9 @@ echo "** Output from non-static builds:" -p ecs-settings-applier \ %endif %if %{_is_k8s_variant} +%if %{_is_aws_variant} -p pluto \ +%endif -p static-pods \ %endif %{nil} @@ -311,7 +318,10 @@ for p in \ ecs-settings-applier \ %endif %if %{_is_k8s_variant} - pluto static-pods \ +%if %{_is_aws_variant} + pluto \ +%endif + static-pods \ %endif ; do install -p -m 0755 ${HOME}/.cache/%{__cargo_target}/release/${p} %{buildroot}%{_cross_bindir} @@ -348,9 +358,11 @@ install -d %{buildroot}%{_cross_sysusersdir} install -p -m 0644 %{S:2} %{buildroot}%{_cross_sysusersdir}/api.conf %if %{_is_k8s_variant} +%if %{_is_aws_variant} install -d %{buildroot}%{_cross_datadir}/eks install -p -m 0644 %{S:3} %{buildroot}%{_cross_datadir}/eks %endif +%endif install -d %{buildroot}%{_cross_datadir}/updog install -p -m 0644 %{_cross_repo_root_json} %{buildroot}%{_cross_datadir}/updog @@ -479,10 +491,12 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor %endif %if %{_is_k8s_variant} +%if %{_is_aws_variant} %files -n %{_cross_os}pluto %{_cross_bindir}/pluto %dir %{_cross_datadir}/eks %{_cross_datadir}/eks/eni-max-pods +%endif %files -n %{_cross_os}static-pods %{_cross_bindir}/static-pods diff --git a/sources/logdog/conf/logdog.vmware-k8s-1.20.conf b/sources/logdog/conf/logdog.vmware-k8s-1.20.conf new file mode 120000 index 00000000000..348d909532e --- /dev/null +++ b/sources/logdog/conf/logdog.vmware-k8s-1.20.conf @@ -0,0 +1 @@ +vmware-k8s.conf \ No newline at end of file diff --git a/sources/logdog/conf/vmware-k8s.conf b/sources/logdog/conf/vmware-k8s.conf new file mode 100644 index 00000000000..83a05b3a4f0 --- /dev/null +++ b/sources/logdog/conf/vmware-k8s.conf @@ -0,0 +1 @@ +exec kube-status systemctl status kube* -l --no-pager diff --git a/sources/models/README.md b/sources/models/README.md index 726ecce5b77..c90ca5bc3e4 100644 --- a/sources/models/README.md +++ b/sources/models/README.md @@ -62,6 +62,11 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and * [Model](src/vmware-dev/mod.rs) * [Default settings](src/vmware-dev/defaults.d/) +### vmware-k8s-1.20: VMware Kubernetes 1.20 + +* [Model](src/vmware-k8s-1.20/mod.rs) +* [Default settings](src/vmware-k8s-1.20/defaults.d/) + ## This directory We use `build.rs` to symlink the proper API model source code for Cargo to build. diff --git a/sources/models/shared-defaults/kubernetes-vmware.toml b/sources/models/shared-defaults/kubernetes-vmware.toml new file mode 100644 index 00000000000..8854ed5a722 --- /dev/null +++ b/sources/models/shared-defaults/kubernetes-vmware.toml @@ -0,0 +1,19 @@ +[settings.kubernetes] +cluster-domain = "cluster.local" +standalone-mode = false +authentication-mode = "tls" +pod-infra-container-image = "k8s.gcr.io/pause:3.2" +server-tls-bootstrap = false +cloud-provider = "external" + +[metadata.settings.kubernetes] +node-ip.setting-generator = "netdog node-ip" +affected-services = ["kubernetes"] + +# Metrics +[settings.metrics] +service-checks = ["apiserver", "chronyd", "containerd", "host-containerd", "kubelet", "vmtoolsd"] + +# Network +[metadata.settings.network] +affected-services = ["containerd", "kubernetes", "host-containerd", "host-containers"] diff --git a/sources/models/shared-defaults/vmware-host-containers.toml b/sources/models/shared-defaults/vmware-host-containers.toml new file mode 100644 index 00000000000..278cbcf2565 --- /dev/null +++ b/sources/models/shared-defaults/vmware-host-containers.toml @@ -0,0 +1,14 @@ +# Both containers are disabled by default in VMware because the user must +# supply user data in order to use the containers. The admin container isn't +# useful without SSH keys/CA certs, and the control container can only be used +# with hybrid SSM off of AWS. VMware users might not want to use either of +# those options. +[settings.host-containers.admin] +enabled = false +superpowered = true +source = "public.ecr.aws/bottlerocket/bottlerocket-admin:v0.7.0" + +[settings.host-containers.control] +enabled = false +superpowered = false +source = "public.ecr.aws/bottlerocket/bottlerocket-control:v0.5.0" diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 556a39fc9a7..0e71d903ee9 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -59,6 +59,11 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and * [Model](src/vmware-dev/mod.rs) * [Default settings](src/vmware-dev/defaults.d/) +## vmware-k8s-1.20: VMware Kubernetes 1.20 + +* [Model](src/vmware-k8s-1.20/mod.rs) +* [Default settings](src/vmware-k8s-1.20/defaults.d/) + # This directory We use `build.rs` to symlink the proper API model source code for Cargo to build. diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/10-defaults.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/10-defaults.toml new file mode 120000 index 00000000000..a202ba61a4c --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/10-defaults.toml @@ -0,0 +1 @@ +../../../shared-defaults/defaults.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/20-vmware-host-containers.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/20-vmware-host-containers.toml new file mode 120000 index 00000000000..9d8a60c521c --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/20-vmware-host-containers.toml @@ -0,0 +1 @@ +../../../shared-defaults/vmware-host-containers.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/30-metrics.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/30-metrics.toml new file mode 120000 index 00000000000..99f0b2b6980 --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/30-metrics.toml @@ -0,0 +1 @@ +../../../shared-defaults/metrics.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/50-kubernetes-vmware.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/50-kubernetes-vmware.toml new file mode 120000 index 00000000000..f67755f3b0a --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/50-kubernetes-vmware.toml @@ -0,0 +1 @@ +../../../shared-defaults/kubernetes-vmware.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/51-kubernetes-containerd.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/51-kubernetes-containerd.toml new file mode 120000 index 00000000000..08196a5edef --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/51-kubernetes-containerd.toml @@ -0,0 +1 @@ +../../../shared-defaults/kubernetes-containerd.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/52-kubernetes-services.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/52-kubernetes-services.toml new file mode 120000 index 00000000000..afab5a7f15a --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/52-kubernetes-services.toml @@ -0,0 +1 @@ +../../../shared-defaults/kubernetes-services.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/60-lockdown-integrity.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/60-lockdown-integrity.toml new file mode 120000 index 00000000000..8b4de873a39 --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/60-lockdown-integrity.toml @@ -0,0 +1 @@ +../../../shared-defaults/lockdown-integrity.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/defaults.d/70-public-ntp.toml b/sources/models/src/vmware-k8s-1.20/defaults.d/70-public-ntp.toml new file mode 120000 index 00000000000..eb5f4bcd0a7 --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/defaults.d/70-public-ntp.toml @@ -0,0 +1 @@ +../../../shared-defaults/public-ntp.toml \ No newline at end of file diff --git a/sources/models/src/vmware-k8s-1.20/mod.rs b/sources/models/src/vmware-k8s-1.20/mod.rs new file mode 100644 index 00000000000..ea06aa7118e --- /dev/null +++ b/sources/models/src/vmware-k8s-1.20/mod.rs @@ -0,0 +1,24 @@ +use model_derive::model; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use crate::modeled_types::Identifier; +use crate::{ + BootstrapContainer, HostContainer, KernelSettings, KubernetesSettings, MetricsSettings, + NetworkSettings, NtpSettings, UpdatesSettings, +}; + +// Note: we have to use 'rename' here because the top-level Settings structure is the only one +// that uses its name in serialization; internal structures use the field name that points to it +#[model(rename = "settings", impl_default = true)] +struct Settings { + motd: String, + kubernetes: KubernetesSettings, + updates: UpdatesSettings, + host_containers: HashMap, + bootstrap_containers: HashMap, + ntp: NtpSettings, + network: NetworkSettings, + kernel: KernelSettings, + metrics: MetricsSettings, +} diff --git a/variants/Cargo.lock b/variants/Cargo.lock index cae424d0de4..61d30ffcb31 100644 --- a/variants/Cargo.lock +++ b/variants/Cargo.lock @@ -745,6 +745,18 @@ dependencies = [ "tcpdump", ] +[[package]] +name = "vmware-k8s-1_20" +version = "0.1.0" +dependencies = [ + "cni", + "cni-plugins", + "kernel-5_10", + "kubernetes-1_20", + "open-vm-tools", + "release", +] + [[package]] name = "wicked" version = "0.1.0" diff --git a/variants/Cargo.toml b/variants/Cargo.toml index ff42807acc1..4314f15c2c2 100644 --- a/variants/Cargo.toml +++ b/variants/Cargo.toml @@ -8,6 +8,7 @@ members = [ "aws-k8s-1.19", "aws-k8s-1.20", "vmware-dev", + "vmware-k8s-1.20", ] [profile.dev] diff --git a/variants/README.md b/variants/README.md index a86882d018c..ce84aa60361 100644 --- a/variants/README.md +++ b/variants/README.md @@ -77,6 +77,15 @@ It includes tools for troubleshooting as well as Docker for running containers. User data will be read from a mounted CD-ROM (from a file named "user-data" or from an OVF file), and from VMware's guestinfo interface. If user data exists at both places, settings read from guestinfo will override identical settings from CD-ROM. +### vmware-k8s-1.20: VMware Kubernetes 1.20 node + +The [vmware-k8s-1.20](vmware-k8s-1.20/Cargo.toml) variant includes the packages needed to run a Kubernetes worker node as a VMware guest. +It supports self-hosted clusters. +User data will be read from a mounted CD-ROM (from a file named "user-data" or from an OVF file), and from VMware's guestinfo interface. +If user data exists at both places, settings read from guestinfo will override identical settings from CD-ROM. + +This variant is compatible with Kubernetes 1.20, 1.21, and 1.22 clusters. + ### Deprecated variants #### aws-k8s-1.15: Kubernetes 1.15 node diff --git a/variants/vmware-k8s-1.20/Cargo.toml b/variants/vmware-k8s-1.20/Cargo.toml new file mode 100644 index 00000000000..1d8663722ef --- /dev/null +++ b/variants/vmware-k8s-1.20/Cargo.toml @@ -0,0 +1,37 @@ +[package] +# This is the vmware-k8s-1.20 variant. "." is not allowed in crate names, but +# we don't use this crate name anywhere. +name = "vmware-k8s-1_20" +version = "0.1.0" +edition = "2018" +publish = false +build = "build.rs" +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[package.metadata.build-variant] +image-format = "vmdk" +supported-arches = ["x86_64"] +kernel-parameters = [ + "console=ttyS0", + "console=tty1", +] +included-packages = [ + "cni", + "cni-plugins", + "kernel-5.10", + "kubelet-1.20", + "open-vm-tools", + "release", +] + +[lib] +path = "lib.rs" + +[build-dependencies] +cni = { path = "../../packages/cni" } +cni-plugins = { path = "../../packages/cni-plugins" } +kernel-5_10 = { path = "../../packages/kernel-5.10" } +kubernetes-1_20 = { path = "../../packages/kubernetes-1.20" } +open-vm-tools = { path = "../../packages/open-vm-tools" } +release = { path = "../../packages/release" } diff --git a/variants/vmware-k8s-1.20/build.rs b/variants/vmware-k8s-1.20/build.rs new file mode 100644 index 00000000000..d6a90e4df44 --- /dev/null +++ b/variants/vmware-k8s-1.20/build.rs @@ -0,0 +1,9 @@ +use std::process::{exit, Command}; + +fn main() -> Result<(), std::io::Error> { + let ret = Command::new("buildsys").arg("build-variant").status()?; + if !ret.success() { + exit(1); + } + Ok(()) +} diff --git a/variants/vmware-k8s-1.20/lib.rs b/variants/vmware-k8s-1.20/lib.rs new file mode 100644 index 00000000000..d799fb2d44c --- /dev/null +++ b/variants/vmware-k8s-1.20/lib.rs @@ -0,0 +1 @@ +// not used diff --git a/variants/vmware-k8s-1.20/template.ovf b/variants/vmware-k8s-1.20/template.ovf new file mode 120000 index 00000000000..00688303038 --- /dev/null +++ b/variants/vmware-k8s-1.20/template.ovf @@ -0,0 +1 @@ +../vmware-dev/template.ovf \ No newline at end of file