From 78e9a71e2173fe8b03d211200e82ef654e6db6a9 Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Mon, 23 Aug 2021 21:43:12 +0000 Subject: [PATCH 1/3] schnauzer: Add a new template helper: `add_unresolvable_hostname` This change adds a new template helper `add_unresolvable_hostname` that will attempt to resolve the hostname given to it in the template (from `settings.network.hostname`). If the hostname is unresolvable or resolves to localhost (127.0.0.1 or ::1), the helper adds an alias to the hostname for both the IPv4 and IPv6 addresses. --- sources/Cargo.lock | 1 + sources/api/schnauzer/Cargo.toml | 1 + sources/api/schnauzer/src/helpers.rs | 134 ++++++++++++++++++++++++++- sources/api/schnauzer/src/lib.rs | 5 + 4 files changed, 138 insertions(+), 3 deletions(-) diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 73d32523baf..7796ab3a82d 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -2689,6 +2689,7 @@ dependencies = [ "bottlerocket-release", "cargo-readme", "constants", + "dns-lookup", "handlebars", "http", "lazy_static", diff --git a/sources/api/schnauzer/Cargo.toml b/sources/api/schnauzer/Cargo.toml index 760450497a2..fc17604b305 100644 --- a/sources/api/schnauzer/Cargo.toml +++ b/sources/api/schnauzer/Cargo.toml @@ -14,6 +14,7 @@ apiclient = { path = "../apiclient" } base64 = "0.13" constants = { path = "../../constants" } bottlerocket-release = { path = "../../bottlerocket-release" } +dns-lookup = "1.0" handlebars = "4.1" http = "0.2" lazy_static = "1.4" diff --git a/sources/api/schnauzer/src/helpers.rs b/sources/api/schnauzer/src/helpers.rs index cc633bab634..0b40f08afe7 100644 --- a/sources/api/schnauzer/src/helpers.rs +++ b/sources/api/schnauzer/src/helpers.rs @@ -2,6 +2,7 @@ // be registered with the Handlebars library to assist in manipulating // text at render time. +use dns_lookup::lookup_host; use handlebars::{Context, Handlebars, Helper, Output, RenderContext, RenderError}; use lazy_static::lazy_static; use num_cpus; @@ -10,6 +11,7 @@ use snafu::{OptionExt, ResultExt}; use std::borrow::Borrow; use std::collections::HashMap; use std::convert::TryFrom; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use url::Url; lazy_static! { @@ -98,6 +100,9 @@ const KUBE_RESERVE_3_CORES: f32 = KUBE_RESERVE_2_CORES + 5.0; const KUBE_RESERVE_4_CORES: f32 = KUBE_RESERVE_3_CORES + 5.0; const KUBE_RESERVE_ADDITIONAL: f32 = 2.5; +const IPV4_LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); +const IPV6_LOCALHOST: IpAddr = IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); + /// Potential errors during helper execution mod error { use handlebars::RenderError; @@ -869,6 +874,77 @@ pub fn kube_reserve_cpu( Ok(()) } +/// Attempts to resolve the current hostname in DNS. If unsuccessful, returns an entry formatted +/// for `/etc/hosts` that aliases the hostname to both the IPV4/6 localhost addresses. +pub fn add_unresolvable_hostname( + helper: &Helper<'_, '_>, + _: &Handlebars, + _: &Context, + renderctx: &mut RenderContext<'_, '_>, + out: &mut dyn Output, +) -> Result<(), RenderError> { + // To give context to our errors, get the template name, if available. + trace!("Starting base64_decode helper"); + let template_name = template_name(renderctx); + trace!("Template name: {}", &template_name); + + // Check number of parameters, must be exactly one + trace!("Number of params: {}", helper.params().len()); + check_param_count(helper, template_name, 1)?; + + // Get the resolved key out of the template (param(0)). value() returns + // a serde_json::Value + let hostname_value = helper + .param(0) + .map(|v| v.value()) + .context(error::Internal { + msg: "Found no params after confirming there is one param", + })?; + trace!("Hostname value from template: {}", hostname_value); + + // Create an &str from the serde_json::Value + let hostname_str = hostname_value + .as_str() + .context(error::InvalidTemplateValue { + expected: "string", + value: hostname_value.to_owned(), + template: template_name.to_owned(), + })?; + trace!("Hostname string from template: {}", hostname_str); + + // Attempt to resolve the hostname + let hostname_resolveable = match lookup_host(hostname_str) { + Ok(ip_list) => { + // If the list of IPs is empty or resolves to localhost, consider the hostname + // unresolvable + let resolves_to_localhost = ip_list + .iter() + .any(|ip| ip == &IPV4_LOCALHOST || ip == &IPV6_LOCALHOST); + if ip_list.is_empty() || resolves_to_localhost { + false + } else { + true + } + } + Err(e) => { + trace!("DNS hostname lookup failed: {},", e); + false + } + }; + + // Only write an entry to the template if the hostname is unresolvable + if !hostname_resolveable { + let ipv4_entry = format!("{} {}", IPV4_LOCALHOST, hostname_str); + let ipv6_entry = format!("{} {}", IPV6_LOCALHOST, hostname_str); + let entries = format!("{}\n{}", ipv4_entry, ipv6_entry); + + out.write(&entries).context(error::TemplateWrite { + template: template_name.to_owned(), + })?; + } + Ok(()) +} + // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= // helpers to the helpers @@ -935,8 +1011,9 @@ fn pause_registry>(region: S) -> String { } /// Calculates and returns the amount of CPU to reserve -fn kube_cpu_helper(num_cores: usize) -> Result{ - let num_cores = u16::try_from(num_cores).context(error::ConvertUsizeToU16 { number: num_cores })?; +fn kube_cpu_helper(num_cores: usize) -> Result { + let num_cores = + u16::try_from(num_cores).context(error::ConvertUsizeToU16 { number: num_cores })?; let millicores_unit = "m"; let cpu_to_reserve = match num_cores { 0 => 0.0, @@ -949,7 +1026,11 @@ fn kube_cpu_helper(num_cores: usize) -> Result{ KUBE_RESERVE_4_CORES + ((num_cores - 4.0) * KUBE_RESERVE_ADDITIONAL) } }; - Ok(format!("{}{}", cpu_to_reserve.floor().to_string(), millicores_unit)) + Ok(format!( + "{}{}", + cpu_to_reserve.floor().to_string(), + millicores_unit + )) } // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= @@ -1607,3 +1688,50 @@ mod test_kube_cpu_helper { } } } + +#[cfg(test)] +mod test_add_unresolvable_hostname { + use super::*; + use handlebars::RenderError; + use serde::Serialize; + use serde_json::json; + + // A thin wrapper around the handlebars render_template method that includes + // setup and registration of helpers + fn setup_and_render_template(tmpl: &str, data: &T) -> Result + where + T: Serialize, + { + let mut registry = Handlebars::new(); + registry.register_helper( + "add_unresolvable_hostname", + Box::new(add_unresolvable_hostname), + ); + + registry.render_template(tmpl, data) + } + + #[test] + fn resolves_to_localhost_renders_entries() { + let result = setup_and_render_template( + "{{add_unresolvable_hostname name}}", + &json!({"name": "localhost"}), + ) + .unwrap(); + assert_eq!( + result, + "127.0.0.1 localhost +::1 localhost" + ) + } + + #[test] + fn resolvable_hostname_renders_nothing() { + let result = setup_and_render_template( + "{{add_unresolvable_hostname name}}", + &json!({"name": "amazon.com"}), + ) + .unwrap(); + assert_eq!(result, "") + } +} diff --git a/sources/api/schnauzer/src/lib.rs b/sources/api/schnauzer/src/lib.rs index 1a67be49692..f386927ac58 100644 --- a/sources/api/schnauzer/src/lib.rs +++ b/sources/api/schnauzer/src/lib.rs @@ -118,6 +118,7 @@ pub fn build_template_registry() -> Result> { // but isn't provided in the data given to the renderer template_registry.set_strict_mode(true); + // Prefer snake case for helper names (we accidentally created a few with kabob case) template_registry.register_helper("base64_decode", Box::new(helpers::base64_decode)); template_registry.register_helper("join_map", Box::new(helpers::join_map)); template_registry.register_helper("default", Box::new(helpers::default)); @@ -131,6 +132,10 @@ pub fn build_template_registry() -> Result> { "kube_reserve_memory", Box::new(helpers::kube_reserve_memory), ); + template_registry.register_helper( + "add_unresolvable_hostname", + Box::new(helpers::add_unresolvable_hostname), + ); Ok(template_registry) } From a7532a79796b06e33522e740ad03325c66837a88 Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Mon, 23 Aug 2021 21:46:51 +0000 Subject: [PATCH 2/3] Add a hostname entry to `/etc/hosts` if hostname is unresolvable Previously, `/etc/hosts` was a static file (in the `release` package) that we copied into the image when building Bottlerocket. In order to have an entry in the file corresponding to hostname, a new "hosts" service was added to the affected services for `settings.network.hostname`. This new "hosts" service has a templated configuration file that uses `settings.network.hostname` to populate the entry in `/etc/hosts` if the current hostname is unresolvable in DNS. The templated configuration file uses the previously added `add_unresolvable_hostname` template renderer helper. --- packages/release/{hosts => hosts.template} | 1 + packages/release/release.spec | 7 ++++--- sources/models/shared-defaults/defaults.toml | 10 +++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) rename packages/release/{hosts => hosts.template} (72%) diff --git a/packages/release/hosts b/packages/release/hosts.template similarity index 72% rename from packages/release/hosts rename to packages/release/hosts.template index 4abeeef8c91..1980e24faae 100644 --- a/packages/release/hosts +++ b/packages/release/hosts.template @@ -1,2 +1,3 @@ 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +{{add_unresolvable_hostname settings.network.hostname}} diff --git a/packages/release/release.spec b/packages/release/release.spec index e752faa01d4..eb6e289c70e 100644 --- a/packages/release/release.spec +++ b/packages/release/release.spec @@ -6,7 +6,6 @@ Release: 0%{?dist} Summary: Bottlerocket release License: Apache-2.0 OR MIT -Source10: hosts Source11: nsswitch.conf Source97: release-sysctl.conf Source98: release-systemd-system.conf @@ -15,6 +14,7 @@ Source99: release-tmpfiles.conf Source200: motd.template Source201: proxy-env Source202: hostname-env +Source203: hosts.template Source1000: eth0.xml Source1001: multi-user.target @@ -91,7 +91,7 @@ Requires: %{_cross_os}wicked %install install -d %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir} -install -p -m 0644 %{S:10} %{S:11} %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir} +install -p -m 0644 %{S:11} %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir} install -d %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir}/wicked/ifconfig install -p -m 0644 %{S:1000} %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir}/wicked/ifconfig @@ -135,6 +135,7 @@ install -d %{buildroot}%{_cross_templatedir} install -p -m 0644 %{S:200} %{buildroot}%{_cross_templatedir}/motd install -p -m 0644 %{S:201} %{buildroot}%{_cross_templatedir}/proxy-env install -p -m 0644 %{S:202} %{buildroot}%{_cross_templatedir}/hostname-env +install -p -m 0644 %{S:203} %{buildroot}%{_cross_templatedir}/hosts install -d %{buildroot}%{_cross_udevrulesdir} install -p -m 0644 %{S:1016} %{buildroot}%{_cross_udevrulesdir}/61-mount-cdrom.rules @@ -142,7 +143,6 @@ install -p -m 0644 %{S:1016} %{buildroot}%{_cross_udevrulesdir}/61-mount-cdrom.r ln -s %{_cross_unitdir}/preconfigured.target %{buildroot}%{_cross_unitdir}/default.target %files -%{_cross_factorydir}%{_cross_sysconfdir}/hosts %{_cross_factorydir}%{_cross_sysconfdir}/nsswitch.conf %{_cross_factorydir}%{_cross_sysconfdir}/wicked/ifconfig/eth0.xml %{_cross_sysctldir}/80-release.conf @@ -175,6 +175,7 @@ ln -s %{_cross_unitdir}/preconfigured.target %{buildroot}%{_cross_unitdir}/defau %{_cross_templatedir}/motd %{_cross_templatedir}/proxy-env %{_cross_templatedir}/hostname-env +%{_cross_templatedir}/hosts %{_cross_udevrulesdir}/61-mount-cdrom.rules %changelog diff --git a/sources/models/shared-defaults/defaults.toml b/sources/models/shared-defaults/defaults.toml index d2f2eccc33e..757c4b0b05a 100644 --- a/sources/models/shared-defaults/defaults.toml +++ b/sources/models/shared-defaults/defaults.toml @@ -81,7 +81,7 @@ template-path = "/usr/share/templates/proxy-env" affected-services = ["containerd", "host-containerd", "host-containers"] [metadata.settings.network.hostname] -affected-services = ["hostname"] +affected-services = ["hostname", "hosts"] setting-generator = "netdog generate-hostname" [services.hostname] @@ -92,6 +92,14 @@ restart-commands = ["/bin/systemctl try-restart set-hostname.service"] path = "/etc/network/hostname.env" template-path = "/usr/share/templates/hostname-env" +[services.hosts] +configuration-files = ["hosts"] +restart-commands = [] + +[configuration-files.hosts] +path = "/etc/hosts" +template-path = "/usr/share/templates/hosts" + # NTP [settings.ntp] From 1c25ade1ba046233320acb47ad12d755e500445b Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Wed, 18 Aug 2021 15:51:54 +0000 Subject: [PATCH 3/3] migrations: Add migrations for `/etc/hosts` "service" This change includes the 2 migrations necessary to make `/etc/hosts` an "affected service" of `settings.network.hostname`: one to add the service and configuration file for the hosts "service", and another to add the hosts service to `settings.network.hostname`. --- Release.toml | 4 +++ sources/Cargo.lock | 14 +++++++++ sources/Cargo.toml | 5 ++-- .../v1.2.1/etc-hosts-service/Cargo.toml | 12 ++++++++ .../v1.2.1/etc-hosts-service/src/main.rs | 23 ++++++++++++++ .../hostname-affects-etc-hosts/Cargo.toml | 12 ++++++++ .../hostname-affects-etc-hosts/src/main.rs | 30 +++++++++++++++++++ 7 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 sources/api/migration/migrations/v1.2.1/etc-hosts-service/Cargo.toml create mode 100644 sources/api/migration/migrations/v1.2.1/etc-hosts-service/src/main.rs create mode 100644 sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/Cargo.toml create mode 100644 sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/src/main.rs diff --git a/Release.toml b/Release.toml index 7a2686791b1..0134f443b61 100644 --- a/Release.toml +++ b/Release.toml @@ -68,3 +68,7 @@ version = "1.2.0" "migrate_v1.2.0_container-registry-config-restarts.lz4", "migrate_v1.2.0_admin-container-v0-7-2.lz4", ] +"(1.2.0, 1.2.1)" = [ + "migrate_v1.2.1_etc-hosts-service.lz4", + "migrate_v1.2.1_hostname-affects-etc-hosts.lz4", +] diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 7796ab3a82d..5079ce3a365 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1037,6 +1037,13 @@ dependencies = [ "libc", ] +[[package]] +name = "etc-hosts-service" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -1390,6 +1397,13 @@ dependencies = [ "tokio", ] +[[package]] +name = "hostname-affects-etc-hosts" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "http" version = "0.2.4" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index cb32159abc6..2b1ebdba65c 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -24,8 +24,9 @@ members = [ "api/migration/migration-helpers", "api/shibaken", - # "api/migration/migrations/vX.Y.Z/... - # (all migrations currently archived; replace this line with new ones) + # "api/migration/migrations/vX.Y.Z/..." + "api/migration/migrations/v1.2.1/etc-hosts-service", + "api/migration/migrations/v1.2.1/hostname-affects-etc-hosts", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.2.1/etc-hosts-service/Cargo.toml b/sources/api/migration/migrations/v1.2.1/etc-hosts-service/Cargo.toml new file mode 100644 index 00000000000..27648d18190 --- /dev/null +++ b/sources/api/migration/migrations/v1.2.1/etc-hosts-service/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "etc-hosts-service" +version = "0.1.0" +authors = ["Zac Mrowicki "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } diff --git a/sources/api/migration/migrations/v1.2.1/etc-hosts-service/src/main.rs b/sources/api/migration/migrations/v1.2.1/etc-hosts-service/src/main.rs new file mode 100644 index 00000000000..5363d9f74e2 --- /dev/null +++ b/sources/api/migration/migrations/v1.2.1/etc-hosts-service/src/main.rs @@ -0,0 +1,23 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::common_migrations::AddPrefixesMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We added a new setting and generator for configuring hostname +fn run() -> Result<()> { + migrate(AddPrefixesMigration(vec![ + "services.hosts", + "configuration-files.hosts", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} diff --git a/sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/Cargo.toml b/sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/Cargo.toml new file mode 100644 index 00000000000..3d26b8b1c4e --- /dev/null +++ b/sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "hostname-affects-etc-hosts" +version = "0.1.0" +authors = ["Zac Mrowicki "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } diff --git a/sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/src/main.rs b/sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/src/main.rs new file mode 100644 index 00000000000..e2e5dee0579 --- /dev/null +++ b/sources/api/migration/migrations/v1.2.1/hostname-affects-etc-hosts/src/main.rs @@ -0,0 +1,30 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::common_migrations::{ + MetadataListReplacement, ReplaceMetadataListsMigration, +}; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We updated the 'affected-services' list metadata for 'settings.network.hostname' to include the +/// hosts "service" on upgrade, and to remove it on downgrade. +fn run() -> Result<()> { + migrate(ReplaceMetadataListsMigration(vec![ + MetadataListReplacement { + setting: "settings.network.hostname", + metadata: "affected-services", + old_vals: &["hostname"], + new_vals: &["hostname", "hosts"], + }, + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +}