model: change container-registry.mirrors type

Changes the model for `container-registry.mirrors` from a `HashMap` to a vector of
structs.
Adds a custom deserializer to deserialize from either a TOML table or a
TOML sequence.

README.md

@@ -411,13 +411,18 @@ These settings can be changed at any time.
 #### Container image registry settings
 
 The following setting is optional and allows you to configure image registry mirrors and pull-through caches for your containers.
-* `settings.container-registry.mirrors`: A mapping of container image registry to a list of image registry URL endpoints.  When pulling an image from a registry, the container runtime will try the endpoints one by one and use the first working one.
+* `settings.container-registry.mirrors`: An array of container image registry mirror settings.  Each element specifies the registry and the endpoints for said registry.
+When pulling an image from a registry, the container runtime will try the endpoints one by one and use the first working one.
   (Docker and containerd will still try the default registry URL if the mirrors fail.)
   * Example user data for setting up image registry mirrors:
   ```
-  [settings.container-registry.mirrors]
-  "docker.io" = ["https://<my-docker-hub-mirror-host>"]
-  "gcr.io" = ["https://<my-gcr-mirror-host>","http://<my-gcr-mirror-host-2>"]
+  [[settings.container-registry.mirrors]]
+  registry = "*"
+  endpoint = ["https://<example-mirror>","https://<example-mirror-2>"]
+
+  [[settings.container-registry.mirrors]]
+  registry = "docker.io"
+  endpoint = [ "https://<my-docker-hub-mirror-host>", "https://<my-docker-hub-mirror-host-2>"]
   ```
   If you use a Bottlerocket variant that uses Docker as the container runtime, like `aws-ecs-1`, you should be aware that Docker only supports pull-through caches for images from Docker Hub (docker.io).  Mirrors for other registries are ignored in this case.
 

packages/containerd/containerd-config-toml_k8s

@@ -32,7 +32,7 @@ conf_dir = "/etc/cni/net.d"
 
 {{#if settings.container-registry.mirrors}}
 {{#each settings.container-registry.mirrors}}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{@key}}"]
-endpoint = [{{join_array ", " this }}]
+[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{registry}}"]
+endpoint = [{{join_array ", " endpoint }}]
 {{/each}}
 {{/if}}

packages/docker-engine/daemon-json

@@ -7,7 +7,9 @@
   "data-root": "/var/lib/docker",
   "selinux-enabled": true,
   "default-ulimits": { "nofile": { "Name": "nofile", "Soft": 1024, "Hard": 4096 } }
-  {{#if settings.container-registry.mirrors.[docker.io]}},
-  "registry-mirrors": [{{join_array ", " settings.container-registry.mirrors.[docker.io]}}]
+  {{#each settings.container-registry.mirrors}}
+  {{#if (eq registry "docker.io" )}},
+  "registry-mirrors": [{{join_array ", " endpoint}}]
   {{/if}}
+  {{/each}}
 }

packages/os/host-ctr-toml

@@ -1,6 +1,6 @@
 {{#if settings.container-registry.mirrors}}
 {{#each settings.container-registry.mirrors}}
-[mirrors."{{@key}}"]
-endpoints = [{{join_array ", " this }}]
+[mirrors."{{registry}}"]
+endpoints = [{{join_array ", " endpoint }}]
 {{/each}}
 {{/if}}

sources/models/src/de.rs

@@ -0,0 +1,72 @@
+use crate::RegistryMirror;
+use serde::de::value::SeqAccessDeserializer;
+use serde::de::{MapAccess, SeqAccess, Visitor};
+use serde::{Deserialize, Deserializer};
+use std::fmt::Formatter;
+
+// Our standard representation of registry mirrors is a `Vec` of `RegistryMirror`; for backward compatibility, we also allow a `HashMap` of registry to endpoints.
+pub(crate) fn deserialize_mirrors<'de, D>(
+    deserializer: D,
+) -> Result<Option<Vec<RegistryMirror>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    struct TableOrArray;
+
+    impl<'de> Visitor<'de> for TableOrArray {
+        type Value = Option<Vec<RegistryMirror>>;
+
+        fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
+            formatter.write_str("TOML array or TOML table")
+        }
+
+        fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
+        where
+            A: SeqAccess<'de>,
+        {
+            Ok(Some(Deserialize::deserialize(SeqAccessDeserializer::new(
+                seq,
+            ))?))
+        }
+
+        fn visit_map<M>(self, mut map: M) -> Result<Self::Value, M::Error>
+        where
+            M: MapAccess<'de>,
+        {
+            let mut vec = Vec::new();
+            while let Some((k, v)) = map.next_entry()? {
+                vec.push(RegistryMirror {
+                    registry: Some(k),
+                    endpoint: Some(v),
+                });
+            }
+            Ok(Some(vec))
+        }
+    }
+    deserializer.deserialize_any(TableOrArray)
+}
+
+#[cfg(test)]
+mod mirrors_tests {
+    use crate::RegistrySettings;
+    static TEST_MIRRORS_ARRAY: &str = include_str!("../tests/data/mirrors-array");
+    static TEST_MIRRORS_TABLE: &str = include_str!("../tests/data/mirrors-table");
+
+    #[test]
+    fn registry_mirrors_array_representation() {
+        assert!(toml::from_str::<RegistrySettings>(TEST_MIRRORS_ARRAY).is_ok());
+    }
+
+    #[test]
+    fn registry_mirrors_table_representation() {
+        assert!(toml::from_str::<RegistrySettings>(TEST_MIRRORS_TABLE).is_ok());
+    }
+
+    #[test]
+    fn representation_equal() {
+        assert_eq!(
+            toml::from_str::<RegistrySettings>(TEST_MIRRORS_TABLE).unwrap(),
+            toml::from_str::<RegistrySettings>(TEST_MIRRORS_ARRAY).unwrap()
+        );
+    }
+}

sources/models/src/lib.rs

@@ -96,6 +96,9 @@ pub mod modeled_types;
 // The "variant" module is just a directory where we symlink in the user's requested build
 // variant; each variant defines a top-level Settings structure and we re-export the current one.
 mod variant;
+// The "de" module contains custom deserialization trait implementation for models.
+mod de;
+
 pub use variant::*;
 
 // Below, we define common structures used in the API surface; specific variants build a Settings
@@ -107,6 +110,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::net::IpAddr;
 
+use crate::de::deserialize_mirrors;
 use crate::modeled_types::{
     BootstrapContainerMode, CpuManagerPolicy, DNSDomain, ECSAgentLogLevel, ECSAttributeKey,
     ECSAttributeValue, FriendlyVersion, Identifier, KubernetesAuthenticationMode,
@@ -179,10 +183,17 @@ struct ECSSettings {
     enable_spot_instance_draining: bool,
 }
 
+#[model]
+struct RegistryMirror {
+    registry: SingleLineString,
+    endpoint: Vec<Url>,
+}
+
 // Image registry settings for the container runtimes.
 #[model]
 struct RegistrySettings {
-    mirrors: HashMap<SingleLineString, Vec<Url>>,
+    #[serde(deserialize_with = "deserialize_mirrors")]
+    mirrors: Vec<RegistryMirror>,
 }
 
 // Update settings. Taken from userdata. The 'seed' setting is generated

sources/models/tests/data/mirrors-array

@@ -0,0 +1,7 @@
+[[mirrors]]
+registry = "*"
+endpoint = ["hello"]
+
+[[mirrors]]
+registry = "example"
+endpoint = [ "hello", "hellohello"]

sources/models/tests/data/mirrors-table

@@ -0,0 +1,3 @@
+[mirrors]
+"*" = ["hello"]
+"example" = ["hello", "hellohello"]