model: container-registry.mirrors different settings representation

README.md

@@ -411,13 +411,18 @@ These settings can be changed at any time.
 #### Container image registry settings
 
 The following setting is optional and allows you to configure image registry mirrors and pull-through caches for your containers.
-* `settings.container-registry.mirrors`: A mapping of container image registry to a list of image registry URL endpoints.  When pulling an image from a registry, the container runtime will try the endpoints one by one and use the first working one.
+* `settings.container-registry.mirrors`: An array of container image registry mirror settings.  Each element specifies the registry and the endpoints for said registry.
+When pulling an image from a registry, the container runtime will try the endpoints one by one and use the first working one.
   (Docker and containerd will still try the default registry URL if the mirrors fail.)
   * Example user data for setting up image registry mirrors:
   ```
-  [settings.container-registry.mirrors]
-  "docker.io" = ["https://<my-docker-hub-mirror-host>"]
-  "gcr.io" = ["https://<my-gcr-mirror-host>","http://<my-gcr-mirror-host-2>"]
+  [[settings.container-registry.mirrors]]
+  registry = "*"
+  endpoint = ["https://<example-mirror>","https://<example-mirror-2>"]
+
+  [[settings.container-registry.mirrors]]
+  registry = "docker.io"
+  endpoint = [ "https://<my-docker-hub-mirror-host>", "https://<my-docker-hub-mirror-host-2>"]
   ```
   If you use a Bottlerocket variant that uses Docker as the container runtime, like `aws-ecs-1`, you should be aware that Docker only supports pull-through caches for images from Docker Hub (docker.io).  Mirrors for other registries are ignored in this case.
 

Release.toml

@@ -74,3 +74,6 @@ version = "1.3.0"
     "migrate_v1.3.0_hostname-affects-etc-hosts.lz4",
     "migrate_v1.3.0_control-container-v0-5-2.lz4",
 ]
+"(1.3.0, 1.4.0)" = [
+    "migrate_v1.4.0_registry-mirror-representation.lz4",
+]

packages/containerd/containerd-config-toml_k8s

@@ -32,7 +32,7 @@ conf_dir = "/etc/cni/net.d"
 
 {{#if settings.container-registry.mirrors}}
 {{#each settings.container-registry.mirrors}}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{@key}}"]
-endpoint = [{{join_array ", " this }}]
+[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{registry}}"]
+endpoint = [{{join_array ", " endpoint }}]
 {{/each}}
 {{/if}}

packages/docker-engine/daemon-json

@@ -7,7 +7,9 @@
   "data-root": "/var/lib/docker",
   "selinux-enabled": true,
   "default-ulimits": { "nofile": { "Name": "nofile", "Soft": 1024, "Hard": 4096 } }
-  {{#if settings.container-registry.mirrors.[docker.io]}},
-  "registry-mirrors": [{{join_array ", " settings.container-registry.mirrors.[docker.io]}}]
+  {{#each settings.container-registry.mirrors}}
+  {{#if (eq registry "docker.io" )}},
+  "registry-mirrors": [{{join_array ", " endpoint}}]
   {{/if}}
+  {{/each}}
 }

packages/os/host-ctr-toml

@@ -1,6 +1,6 @@
 {{#if settings.container-registry.mirrors}}
 {{#each settings.container-registry.mirrors}}
-[mirrors."{{@key}}"]
-endpoints = [{{join_array ", " this }}]
+[mirrors."{{registry}}"]
+endpoints = [{{join_array ", " endpoint }}]
 {{/each}}
 {{/if}}

sources/Cargo.toml

@@ -28,6 +28,7 @@ members = [
     "api/migration/migrations/v1.3.0/etc-hosts-service",
     "api/migration/migrations/v1.3.0/hostname-affects-etc-hosts",
     "api/migration/migrations/v1.3.0/control-container-v0-5-2",
+    "api/migration/migrations/v1.4.0/registry-mirror-representation",
 
     "bottlerocket-release",
 

sources/api/migration/migrations/v1.4.0/registry-mirror-representation/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "registry-mirror-representation"
+version = "0.1.0"
+authors = ["Erikson Tung <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers", version = "0.1.0"}
+serde_json = "1.0"

sources/api/migration/migrations/v1.4.0/registry-mirror-representation/src/main.rs

@@ -0,0 +1,133 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::{migrate, Migration, MigrationData, Result};
+use serde_json::{Map, Value};
+use std::collections::HashMap;
+use std::process;
+
+const MIRRORS_SETTING_NAME: &'static str = "settings.container-registry.mirrors";
+const DATASTORE_KEY_SEPARATOR: char = '.';
+
+/// This migration changes the model type of `settings.container-registry.mirrors` from `HashMap<SingleLineString, Vec<Url>>`
+/// to `Vec<RegistryMirrors>` on upgrade and vice-versa on downgrades.
+pub struct ChangeRegistryMirrorsType;
+
+// Snapshot of the `datastore::Key::valid_character` method in Bottlerocket version 1.3.0
+//
+// Determines whether a character is acceptable within a segment of a key name.  This is
+// separate from quoting; if a character isn't valid, it isn't valid quoted, either.
+fn valid_character(c: char) -> bool {
+    match c {
+        'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' | '/' => true,
+        _ => false,
+    }
+}
+
+impl Migration for ChangeRegistryMirrorsType {
+    /// Newer versions store `settings.container-registry.mirrors` as `Vec<RegistryMirrors>`.
+    /// Need to convert from `HashMap<SingleLineString, Vec<Url>>`.
+    fn forward(&mut self, mut input: MigrationData) -> Result<MigrationData> {
+        let mirrors: HashMap<_, _> = input
+            .data
+            .iter()
+            .filter(|&(k, _)| k.starts_with(format!("{}.", MIRRORS_SETTING_NAME).as_str()))
+            .map(|(k, v)| (k.to_owned(), v.to_owned()))
+            .collect();
+        let mut new_mirrors = Vec::new();
+        for (setting, endpoint) in mirrors {
+            // Get the registry name from the settings name. Trim any quotes the settings name might have.
+            let registry = setting
+                .strip_prefix(&format!("{}.", MIRRORS_SETTING_NAME))
+                .unwrap_or_default()
+                .trim_matches('"');
+            let mut registry_mirrors = Map::new();
+            registry_mirrors.insert("registry".to_string(), Value::String(registry.to_string()));
+            registry_mirrors.insert("endpoint".to_string(), endpoint.to_owned());
+            new_mirrors.push(Value::Object(registry_mirrors));
+            if let Some(data) = input.data.remove(&setting) {
+                println!("Removed setting '{}', which was set to '{}'", setting, data);
+            }
+        }
+        let data = Value::Array(new_mirrors);
+        println!(
+            "Creating new setting '{}', which is set to '{}'",
+            MIRRORS_SETTING_NAME, &data
+        );
+        input.data.insert(MIRRORS_SETTING_NAME.to_string(), data);
+        Ok(input)
+    }
+
+    /// Older versions store `settings.container-registry.mirrors` as `HashMap<SingleLineString, Vec<Url>>`.
+    /// Need to convert from `Vec<RegistryMirrors>`.
+    fn backward(&mut self, mut input: MigrationData) -> Result<MigrationData> {
+        if let Some(data) = input.data.get_mut(MIRRORS_SETTING_NAME).cloned() {
+            match data {
+                Value::Array(arr) => {
+                    if let Some(data) = input.data.remove(MIRRORS_SETTING_NAME) {
+                        println!(
+                            "Removed setting '{}', which was set to '{}'",
+                            MIRRORS_SETTING_NAME, data
+                        );
+                    }
+                    for obj in arr {
+                        if let Some(obj) = obj.as_object() {
+                            if let (Some(registry), Some(endpoint)) = (
+                                obj.get("registry").and_then(|s| s.as_str()),
+                                obj.get("endpoint"),
+                            ) {
+                                // Ensure the registry contains valid datastore key characters.
+                                // If we encounter any invalid key characters, we skip writing out
+                                // the setting key to prevent breakage of the datastore.
+                                if registry
+                                    .chars()
+                                    .all(|c| valid_character(c) || c == DATASTORE_KEY_SEPARATOR)
+                                {
+                                    let setting_name =
+                                        format!(r#"{}."{}""#, MIRRORS_SETTING_NAME, registry);
+                                    println!(
+                                        "Creating new setting '{}', which is set to '{}'",
+                                        setting_name, &endpoint
+                                    );
+                                    input.data.insert(setting_name, endpoint.to_owned());
+                                } else {
+                                    eprintln!(
+                                        "Container registry '{}' contains invalid datastore key character(s). Skipping to prevent datastore breakage...",
+                                        registry
+                                    );
+                                }
+                            }
+                        } else {
+                            println!(
+                                "'{}' contains non-JSON Object value: '{}'.",
+                                MIRRORS_SETTING_NAME, obj
+                            );
+                        }
+                    }
+                }
+                _ => {
+                    println!(
+                        "'{}' is not a JSON Array value: '{}'.",
+                        MIRRORS_SETTING_NAME, data
+                    );
+                }
+            }
+        } else {
+            println!("Didn't find setting '{}'", MIRRORS_SETTING_NAME);
+        }
+        Ok(input)
+    }
+}
+
+fn run() -> Result<()> {
+    migrate(ChangeRegistryMirrorsType)
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}

sources/models/src/de.rs

@@ -0,0 +1,72 @@
+use crate::RegistryMirror;
+use serde::de::value::SeqAccessDeserializer;
+use serde::de::{MapAccess, SeqAccess, Visitor};
+use serde::{Deserialize, Deserializer};
+use std::fmt::Formatter;
+
+// Our standard representation of registry mirrors is a `Vec` of `RegistryMirror`; for backward compatibility, we also allow a `HashMap` of registry to endpoints.
+pub(crate) fn deserialize_mirrors<'de, D>(
+    deserializer: D,
+) -> Result<Option<Vec<RegistryMirror>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    struct TableOrArray;
+
+    impl<'de> Visitor<'de> for TableOrArray {
+        type Value = Option<Vec<RegistryMirror>>;
+
+        fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
+            formatter.write_str("TOML array or TOML table")
+        }
+
+        fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+        where
+            A: SeqAccess<'de>,
+        {
+            Ok(Some(Deserialize::deserialize(SeqAccessDeserializer::new(
+                seq,
+            ))?))
+        }
+
+        fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>
+        where
+            M: MapAccess<'de>,
+        {
+            let mut vec = Vec::new();
+            while let Some((k, v)) = map.next_entry()? {
+                vec.push(RegistryMirror {
+                    registry: Some(k),
+                    endpoint: Some(v),
+                });
+            }
+            Ok(Some(vec))
+        }
+    }
+    deserializer.deserialize_any(TableOrArray)
+}
+
+#[cfg(test)]
+mod mirrors_tests {
+    use crate::RegistrySettings;
+    static TEST_MIRRORS_ARRAY: &str = include_str!("../tests/data/mirrors-array");
+    static TEST_MIRRORS_TABLE: &str = include_str!("../tests/data/mirrors-table");
+
+    #[test]
+    fn registry_mirrors_array_representation() {
+        assert!(toml::from_str::<RegistrySettings>(TEST_MIRRORS_ARRAY).is_ok());
+    }
+
+    #[test]
+    fn registry_mirrors_table_representation() {
+        assert!(toml::from_str::<RegistrySettings>(TEST_MIRRORS_TABLE).is_ok());
+    }
+
+    #[test]
+    fn representation_equal() {
+        assert_eq!(
+            toml::from_str::<RegistrySettings>(TEST_MIRRORS_TABLE).unwrap(),
+            toml::from_str::<RegistrySettings>(TEST_MIRRORS_ARRAY).unwrap()
+        );
+    }
+}

sources/models/src/lib.rs

@@ -96,6 +96,9 @@ pub mod modeled_types;
 // The "variant" module is just a directory where we symlink in the user's requested build
 // variant; each variant defines a top-level Settings structure and we re-export the current one.
 mod variant;
+// The "de" module contains custom deserialization trait implementation for models.
+mod de;
+
 pub use variant::*;
 
 // Below, we define common structures used in the API surface; specific variants build a Settings
@@ -107,6 +110,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::net::IpAddr;
 
+use crate::de::deserialize_mirrors;
 use crate::modeled_types::{
     BootstrapContainerMode, CpuManagerPolicy, DNSDomain, ECSAgentLogLevel, ECSAttributeKey,
     ECSAttributeValue, FriendlyVersion, Identifier, KubernetesAuthenticationMode,
@@ -179,10 +183,17 @@ struct ECSSettings {
     enable_spot_instance_draining: bool,
 }
 
+#[model]
+struct RegistryMirror {
+    registry: SingleLineString,
+    endpoint: Vec<Url>,
+}
+
 // Image registry settings for the container runtimes.
 #[model]
 struct RegistrySettings {
-    mirrors: HashMap<SingleLineString, Vec<Url>>,
+    #[serde(deserialize_with = "deserialize_mirrors")]
+    mirrors: Vec<RegistryMirror>,
 }
 
 // Update settings. Taken from userdata. The 'seed' setting is generated

sources/models/tests/data/mirrors-array

@@ -0,0 +1,7 @@
+[[mirrors]]
+registry = "*"
+endpoint = ["hello"]
+
+[[mirrors]]
+registry = "example"
+endpoint = [ "hello", "hellohello"]

sources/models/tests/data/mirrors-table

@@ -0,0 +1,3 @@
+[mirrors]
+"*" = ["hello"]
+"example" = ["hello", "hellohello"]