Merge pull request #1629 from etungsten/image-registry-cache

docker, containerd: image registry mirrors

README.md

@@ -408,6 +408,21 @@ These settings can be changed at any time.
   Supported values are `debug`, `info`, `warn`, `error`, and `crit`, and the default is `info`.
 * `settings.ecs.enable-spot-instance-draining`: If the instance receives a spot termination notice, the agent will set the instance's state to `DRAINING`, so the workload can be moved gracefully before the instance is removed. Defaults to `false`.
 
+#### Container image registry settings
+
+The following setting is optional and allows you to configure image registry mirrors and pull-through caches for your containers.
+* `settings.container-registry.mirrors`: A mapping of container image registry to a list of image registry URL endpoints.  When pulling an image from a registry, the container runtime will try the endpoints one by one and use the first working one.
+  (Docker and containerd will still try the default registry URL if the mirrors fail.)
+  * Example user data for setting up image registry mirrors:
+  ```
+  [settings.container-registry.mirrors]
+  "docker.io" = ["https://<my-docker-hub-mirror-host>"]
+  "gcr.io" = ["https://<my-gcr-mirror-host>","http://<my-gcr-mirror-host-2>"]
+  ```
+  If you use a Bottlerocket variant that uses Docker as the container runtime, like `aws-ecs-1`, you should be aware that Docker only supports pull-through caches for images from Docker Hub (docker.io).  Mirrors for other registries are ignored in this case.
+
+For [host-container](#host-containers-settings) and [bootstrap-container](#bootstrap-containers-settings) images from Amazon ECR private repositories, registry mirrors are currently unsupported.
+
 #### Updates settings
 
 * `settings.updates.metadata-base-url`: The common portion of all URIs used to download update metadata.

Release.toml

@@ -64,4 +64,6 @@ version = "1.1.4"
     "migrate_v1.2.0_hostname-setting-metadata.lz4",
     "migrate_v1.2.0_add-custom-certificates.lz4",
     "migrate_v1.2.0_kubelet-topology-manager.lz4",
+    "migrate_v1.2.0_container-registry-mirrors.lz4",
+    "migrate_v1.2.0_container-registry-config-restarts.lz4",
 ]

packages/containerd/containerd-config-toml_k8s

@@ -29,3 +29,10 @@ SystemdCgroup = true
 [plugins."io.containerd.grpc.v1.cri".cni]
 bin_dir = "/opt/cni/bin"
 conf_dir = "/etc/cni/net.d"
+
+{{~#if settings.container-registry.mirrors}}
+{{~#each settings.container-registry.mirrors}}
+[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{@key}}"]
+endpoint = [{{join_array ", " this }}]
+{{~/each}}
+{{~/if}}

packages/docker-engine/daemon.json → packages/docker-engine/daemon-json

@@ -7,4 +7,7 @@
   "data-root": "/var/lib/docker",
   "selinux-enabled": true,
   "default-ulimits": { "nofile": { "Name": "nofile", "Soft": 1024, "Hard": 4096 } }
+  {{~#if settings.container-registry.mirrors.[docker.io]}},
+  "registry-mirrors": [{{join_array ", " settings.container-registry.mirrors.[docker.io]}}]
+  {{~/if}}
 }

packages/docker-engine/docker-engine.spec

@@ -21,8 +21,7 @@ Source0: https://%{repo}/archive/v%{gover}/%{project}-%{gover}.tar.gz
 Source1: docker.service
 Source2: docker.socket
 Source3: docker-sysusers.conf
-Source4: daemon.json
-Source5: docker-tmpfiles.conf
+Source4: daemon-json
 Source1000: clarify.toml
 
 BuildRequires: git
@@ -68,11 +67,8 @@ install -p -m 0644 %{S:2} %{buildroot}%{_cross_unitdir}/docker.socket
 install -d %{buildroot}%{_cross_sysusersdir}
 install -p -m 0644 %{S:3} %{buildroot}%{_cross_sysusersdir}/docker.conf
 
-install -d %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir}/docker
-install -p -m 0644 %{S:4} %{buildroot}%{_cross_factorydir}%{_cross_sysconfdir}/docker/daemon.json
-
-install -d %{buildroot}%{_cross_tmpfilesdir}
-install -p -m 0644 %{S:5} %{buildroot}%{_cross_tmpfilesdir}/docker.conf
+install -d %{buildroot}%{_cross_templatedir}
+install -p -m 0644 %{S:4} %{buildroot}%{_cross_templatedir}/docker-daemon-json
 
 %cross_scan_attribution --clarify %{S:1000} go-vendor vendor
 
@@ -83,7 +79,6 @@ install -p -m 0644 %{S:5} %{buildroot}%{_cross_tmpfilesdir}/docker.conf
 %{_cross_unitdir}/docker.service
 %{_cross_unitdir}/docker.socket
 %{_cross_sysusersdir}/docker.conf
-%{_cross_factorydir}%{_cross_sysconfdir}/docker
-%{_cross_tmpfilesdir}/docker.conf
+%{_cross_templatedir}/docker-daemon-json
 
 %changelog

packages/kubernetes-1.17/kubelet.service

@@ -15,7 +15,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --containerd-socket=/run/dockershim.sock \
     --namespace=k8s.io \
     pull-image \
-    --source=${POD_INFRA_CONTAINER_IMAGE}
+    --source=${POD_INFRA_CONTAINER_IMAGE} \
+    --registry-config=/etc/host-containers/host-ctr.toml
 # Must be overridden by a drop-in file or `kubelet` won't start
 ExecStart=/usr/bin/false
 

packages/kubernetes-1.18/kubelet.service

@@ -15,7 +15,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --containerd-socket=/run/dockershim.sock \
     --namespace=k8s.io \
     pull-image \
-    --source=${POD_INFRA_CONTAINER_IMAGE}
+    --source=${POD_INFRA_CONTAINER_IMAGE} \
+    --registry-config=/etc/host-containers/host-ctr.toml
 # Must be overridden by a drop-in file or `kubelet` won't start
 ExecStart=/usr/bin/false
 

packages/kubernetes-1.19/kubelet.service

@@ -15,7 +15,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --containerd-socket=/run/dockershim.sock \
     --namespace=k8s.io \
     pull-image \
-    --source=${POD_INFRA_CONTAINER_IMAGE}
+    --source=${POD_INFRA_CONTAINER_IMAGE} \
+    --registry-config=/etc/host-containers/host-ctr.toml
 # Must be overridden by a drop-in file or `kubelet` won't start
 ExecStart=/usr/bin/false
 

packages/kubernetes-1.20/kubelet.service

@@ -15,7 +15,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --containerd-socket=/run/dockershim.sock \
     --namespace=k8s.io \
     pull-image \
-    --source=${POD_INFRA_CONTAINER_IMAGE}
+    --source=${POD_INFRA_CONTAINER_IMAGE} \
+    --registry-config=/etc/host-containers/host-ctr.toml
 # Must be overridden by a drop-in file or `kubelet` won't start
 ExecStart=/usr/bin/false
 

packages/kubernetes-1.21/kubelet.service

@@ -15,7 +15,8 @@ ExecStartPre=/usr/bin/host-ctr \
     --containerd-socket=/run/dockershim.sock \
     --namespace=k8s.io \
     pull-image \
-    --source=${POD_INFRA_CONTAINER_IMAGE}
+    --source=${POD_INFRA_CONTAINER_IMAGE} \
+    --registry-config=/etc/host-containers/host-ctr.toml
 # Must be overridden by a drop-in file or `kubelet` won't start
 ExecStart=/usr/bin/false
 

packages/os/bootstrap-containers@.service

@@ -20,7 +20,8 @@ ExecStart=/usr/bin/touch /run/bootstrap-containers/%i.ran
 ExecStart=/usr/bin/host-ctr run \
     --container-id='%i' \
     --source='${CTR_SOURCE}' \
-    --container-type='bootstrap'
+    --container-type='bootstrap' \
+    --registry-config=/etc/host-containers/host-ctr.toml
 ExecStartPost=/usr/bin/bootstrap-containers mark-bootstrap \
     --container-id '%i' \
     --mode '${CTR_MODE}'

packages/os/host-containers@.service

@@ -11,7 +11,8 @@ Environment=LOCAL_DIR=/local
 ExecStart=/usr/bin/host-ctr run \
     --container-id='%i' \
     --source='${CTR_SOURCE}' \
-    --superpowered='${CTR_SUPERPOWERED}'
+    --superpowered='${CTR_SUPERPOWERED}' \
+    --registry-config=/etc/host-containers/host-ctr.toml
 Restart=always
 RestartSec=45
 TimeoutStopSec=60

packages/os/host-ctr-toml

@@ -0,0 +1,6 @@
+{{~#if settings.container-registry.mirrors}}
+{{~#each settings.container-registry.mirrors}}
+[mirrors."{{@key}}"]
+endpoints = [{{join_array ", " this }}]
+{{~/each}}
+{{~/if}}

packages/os/os.spec

@@ -23,6 +23,7 @@ Source3: eni-max-pods
 
 Source5: updog-toml
 Source6: metricdog-toml
+Source7: host-ctr-toml
 
 # 1xx sources: systemd units
 Source100: apiserver.service
@@ -394,7 +395,7 @@ install -d %{buildroot}%{_cross_datadir}/updog
 install -p -m 0644 %{_cross_repo_root_json} %{buildroot}%{_cross_datadir}/updog
 
 install -d %{buildroot}%{_cross_templatedir}
-install -p -m 0644 %{S:5} %{S:6} %{buildroot}%{_cross_templatedir}
+install -p -m 0644 %{S:5} %{S:6} %{S:7} %{buildroot}%{_cross_templatedir}
 
 install -d %{buildroot}%{_cross_unitdir}
 install -p -m 0644 \
@@ -464,6 +465,8 @@ install -p -m 0644 %{S:300} %{buildroot}%{_cross_udevrulesdir}/80-ephemeral-stor
 %{_cross_bindir}/host-containers
 %{_cross_unitdir}/[email protected]
 %{_cross_tmpfilesdir}/host-containers.conf
+%dir %{_cross_templatedir}
+%{_cross_templatedir}/host-ctr-toml
 
 %files -n %{_cross_os}storewolf
 %{_cross_bindir}/storewolf

sources/Cargo.toml

@@ -42,6 +42,8 @@ members = [
     "api/migration/migrations/v1.2.0/hostname-setting-metadata",
     "api/migration/migrations/v1.2.0/add-custom-certificates",
     "api/migration/migrations/v1.2.0/kubelet-topology-manager",
+    "api/migration/migrations/v1.2.0/container-registry-mirrors",
+    "api/migration/migrations/v1.2.0/container-registry-config-restarts",
 
     "bottlerocket-release",
 

sources/api/migration/migrations/v1.2.0/container-registry-config-restarts/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "container-registry-config-restarts"
+version = "0.1.0"
+authors = ["Erikson Tung <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers" }

sources/api/migration/migrations/v1.2.0/container-registry-config-restarts/src/main.rs

@@ -0,0 +1,37 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::common_migrations::{ListReplacement, ReplaceListsMigration};
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// We templatized the configuration file for the Docker daemon.
+/// We also added a new configuration file for host-containers and bootstrap-containers
+fn run() -> Result<()> {
+    migrate(ReplaceListsMigration(vec![
+        ListReplacement {
+            setting: "services.docker.configuration-files",
+            old_vals: &["proxy-env"],
+            new_vals: &["docker-daemon-config", "proxy-env"],
+        },
+        ListReplacement {
+            setting: "services.bootstrap-containers.configuration-files",
+            old_vals: &[],
+            new_vals: &["host-ctr-toml"],
+        },
+        ListReplacement {
+            setting: "services.host-containers.configuration-files",
+            old_vals: &[],
+            new_vals: &["host-ctr-toml"],
+        },
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}

sources/api/migration/migrations/v1.2.0/container-registry-mirrors/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "container-registry-mirrors"
+version = "0.1.0"
+authors = ["Erikson Tung <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers" }

sources/api/migration/migrations/v1.2.0/container-registry-mirrors/src/main.rs

@@ -0,0 +1,25 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::common_migrations::AddPrefixesMigration;
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// We added a new setting for configuring image registries, `settings.container-registry`
+/// We also added a new configuration template file for the Docker daemon
+fn run() -> Result<()> {
+    migrate(AddPrefixesMigration(vec![
+        "settings.container-registry",
+        "configuration-files.docker-daemon-config",
+        "configuration-files.host-ctr-toml",
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}