pre-fetching sandbox image during bootstrap in containerd runtime (#730)

Co-authored-by: Sinha <[email protected]>
awslabs · Aug 10, 2021 · 5d109a1 · 5d109a1
1 parent 73693cd
commit 5d109a1
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 3 deletions.
diff --git a/files/bootstrap.sh b/files/bootstrap.sh
@@ -406,12 +406,18 @@ fi
 if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then
     sudo mkdir -p /etc/containerd
     sudo mkdir -p /etc/cni/net.d
+    sudo sed -i s,SANDBOX_IMAGE,$PAUSE_CONTAINER,g /etc/eks/containerd/containerd-config.toml  
     sudo mv /etc/eks/containerd/containerd-config.toml /etc/containerd/config.toml
+    sudo mv /etc/eks/containerd/sandbox-image.service /etc/systemd/system/sandbox-image.service
     sudo mv /etc/eks/containerd/kubelet-containerd.service /etc/systemd/system/kubelet.service
     sudo chown root:root /etc/systemd/system/kubelet.service
+    sudo chown root:root /etc/systemd/system/sandbox-image.service
     systemctl daemon-reload
     systemctl enable containerd
-    systemctl start containerd
+    systemctl restart containerd
+    systemctl enable sandbox-image
+    systemctl start sandbox-image
+
 elif [[ "$CONTAINER_RUNTIME" = "dockerd" ]]; then
     mkdir -p /etc/docker
     bash -c "/sbin/iptables-save > /etc/sysconfig/iptables"

diff --git a/files/containerd-config.toml b/files/containerd-config.toml
@@ -8,6 +8,9 @@ address = "/run/dockershim.sock"
 [plugins."io.containerd.grpc.v1.cri".containerd]
 default_runtime_name = "runc"
 
+[plugins."io.containerd.grpc.v1.cri"]
+sandbox_image = "SANDBOX_IMAGE"
+
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
 runtime_type = "io.containerd.runc.v2"
 

diff --git a/files/kubelet-containerd.service b/files/kubelet-containerd.service
@@ -1,8 +1,8 @@
 [Unit]
 Description=Kubernetes Kubelet
 Documentation=https://github.com/kubernetes/kubernetes
-After=containerd.service
-Requires=containerd.service
+After=containerd.service sandbox-image.service
+Requires=containerd.service sandbox-image.service
 
 [Service]
 ExecStartPre=/sbin/iptables -P FORWARD ACCEPT -w 5

diff --git a/files/pull-sandbox-image.sh b/files/pull-sandbox-image.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+### fetching sandbox image from /etc/containerd/config.toml
+sandbox_image=$(awk -F'[ ="]+' '$1 == "sandbox_image" { print $2 }' /etc/containerd/config.toml)
+region=$(echo "$sandbox_image" | cut -f4 -d ".")
+ecr_password=$(aws ecr get-login-password --region $region)
+API_RETRY_ATTEMPTS=5
+
+for attempt in `seq 0 $API_RETRY_ATTEMPTS`; do
+	rc=0
+    if [[ $attempt -gt 0 ]]; then
+        echo "Attempt $attempt of $API_RETRY_ATTEMPTS"
+    fi
+	### pull sandbox image from ecr
+	### username will always be constant i.e; AWS
+	sudo ctr --address=/run/dockershim.sock --namespace k8s.io image pull $sandbox_image --user AWS:$ecr_password
+	rc=$?;
+	if [[ $rc -eq 0 ]]; then
+		break
+	fi
+	if [[ $attempt -eq $API_RETRY_ATTEMPTS ]]; then
+        exit $rc
+    fi
+    jitter=$((1 + RANDOM % 10))
+    sleep_sec="$(( $(( 5 << $((1+$attempt)) )) + $jitter))"
+    sleep $sleep_sec
+done
diff --git a/files/sandbox-image.service b/files/sandbox-image.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=pull sandbox image defined in containerd config.toml
+# pulls sandbox image using ctr tool
+After=containerd.service
+Requires=containerd.service
+
+[Service]
+Type=oneshot
+ExecStart=/etc/eks/containerd/pull-sandbox-image.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/log-collector-script/linux/eks-log-collector.sh b/log-collector-script/linux/eks-log-collector.sh
@@ -58,6 +58,7 @@ COMMON_DIRECTORIES=(
   storage
   var_log
   networking
+  sandbox-image # eks
   ipamd # eks
   sysctls # eks
   kubelet # eks
@@ -258,6 +259,7 @@ collect() {
   get_networking_info
   get_cni_config
   get_docker_logs
+  get_sandboxImage_info
 }
 
 pack() {
@@ -551,6 +553,12 @@ get_containerd_info() {
    ok
 }
 
+get_sandboxImage_info() {
+    try "Collect sandbox-image daemon information"      
+      timeout 75 journalctl -u sandbox-image > "${COLLECT_DIR}"/sandbox-image/sandbox-image-log.txt 2>&1 || echo -e "\tTimed out, ignoring \"sandbox-image info output \" "
+   ok
+}
+
 get_docker_info() {
   try "collect Docker daemon information"
 

diff --git a/scripts/install-worker.sh b/scripts/install-worker.sh
@@ -150,6 +150,9 @@ else
 fi
 
 sudo mv $TEMPLATE_DIR/kubelet-containerd.service /etc/eks/containerd/kubelet-containerd.service
+sudo mv $TEMPLATE_DIR/sandbox-image.service /etc/eks/containerd/sandbox-image.service
+sudo mv $TEMPLATE_DIR/pull-sandbox-image.sh /etc/eks/containerd/pull-sandbox-image.sh
+sudo chmod 777 /etc/eks/containerd/pull-sandbox-image.sh
 
 cat <<EOF | sudo tee -a /etc/modules-load.d/containerd.conf
 overlay