From fd1e3da785765a478f35bd98f044c67ba6e70d5e Mon Sep 17 00:00:00 2001
From: ruquanzhao <ruquan.zhao@arm.com>
Date: Wed, 10 Nov 2021 03:37:22 +0000
Subject: [PATCH] Add arm64 support of kubekins-e2e and krte images.

Signed-off-by: Ruquan Zhao ruquan.zhao@arm.com
---
 images/krte/Dockerfile              |  8 +++--
 images/krte/cloudbuild.yaml         | 41 +++++++++++++----------
 images/kubekins-e2e/Dockerfile      | 30 +++++++++++++----
 images/kubekins-e2e/cloudbuild.yaml | 51 ++++++++++++++++-------------
 images/kubekins-e2e/variants.yaml   |  8 +++++
 5 files changed, 87 insertions(+), 51 deletions(-)

diff --git a/images/krte/Dockerfile b/images/krte/Dockerfile
index 79e34c958c4d..09416c0ba47f 100644
--- a/images/krte/Dockerfile
+++ b/images/krte/Dockerfile
@@ -49,6 +49,7 @@ COPY wrapper.sh /usr/local/bin/
 # We're already inside docker though so we can be sure these are already mounted.
 # Trying to remount these makes for a very noisy error block in the beginning of
 # the pod logs, so we just comment out the call to it... :shrug:
+ARG TARGETARCH
 RUN echo "Installing Packages ..." \
         && apt-get update \
         && apt-get install -y --no-install-recommends \
@@ -72,7 +73,7 @@ RUN echo "Installing Packages ..." \
             unzip \
         && rm -rf /var/lib/apt/lists/* \
     && echo "Installing Go ..." \
-        && export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz"\
+        && export GO_TARBALL="go${GO_VERSION}.linux-${TARGETARCH:-amd64}.tar.gz"\
         && curl -fsSL "https://storage.googleapis.com/golang/${GO_TARBALL}" --output "${GO_TARBALL}" \
         && tar xzf "${GO_TARBALL}" -C /usr/local \
         && rm "${GO_TARBALL}"\
@@ -86,11 +87,12 @@ RUN echo "Installing Packages ..." \
             --bash-completion=false \
             --path-update=false \
             --usage-reporting=false \
-        && gcloud components install kubectl \
+        && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH:-amd64}/kubectl" \
+        && chmod +x ./kubectl && mv ./kubectl /usr/local/bin/kubectl \
     && echo "Installing Docker ..." \
         && curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - \
         && add-apt-repository \
-            "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \
+            "deb [arch=${TARGETARCH:-amd64}] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \
             $(lsb_release -cs) stable" \
         && apt-get update \
         && apt-get install -y --no-install-recommends docker-ce \
diff --git a/images/krte/cloudbuild.yaml b/images/krte/cloudbuild.yaml
index 9617d8ca0e0b..54f0781bc516 100644
--- a/images/krte/cloudbuild.yaml
+++ b/images/krte/cloudbuild.yaml
@@ -1,23 +1,29 @@
 timeout: 1800s
 steps:
-  - name: gcr.io/cloud-builders/docker
+  - name: gcr.io/k8s-testimages/gcb-docker-gcloud
+    entrypoint: /buildx-entrypoint
     args:
-    - build
-    - --tag=gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG
-    - --build-arg=GO_VERSION=$_GO_VERSION
-    - --build-arg=K8S_RELEASE=$_K8S_RELEASE
-    - --build-arg=BAZEL_VERSION_ARG=$_BAZEL_VERSION
-    - --build-arg=OLD_BAZEL_VERSION=$_OLD_BAZEL_VERSION
-    - --build-arg=CFSSL_VERSION=$_CFSSL_VERSION
-    - --build-arg=UPGRADE_DOCKER_ARG=$_UPGRADE_DOCKER
-    - --build-arg=IMAGE_ARG=gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG
-    - .
+      - build
+      - --tag=gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG
+      - --platform=linux/$_PLATFORM
+      - --build-arg=GO_VERSION=$_GO_VERSION
+      - --build-arg=K8S_RELEASE=$_K8S_RELEASE
+      - --build-arg=BAZEL_VERSION_ARG=$_BAZEL_VERSION
+      - --build-arg=OLD_BAZEL_VERSION=$_OLD_BAZEL_VERSION
+      - --build-arg=CFSSL_VERSION=$_CFSSL_VERSION
+      - --build-arg=UPGRADE_DOCKER_ARG=$_UPGRADE_DOCKER
+      - --build-arg=IMAGE_ARG=gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG
+      - --push
+      - .
     dir: images/krte
-  - name: gcr.io/cloud-builders/docker
+  - name: gcr.io/k8s-testimages/gcb-docker-gcloud
+    entrypoint: gcloud
     args:
-    - tag
-    - gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG
-    - gcr.io/$PROJECT_ID/krte:latest-$_CONFIG
+      - container
+      - images
+      - add-tag
+      - gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG
+      - gcr.io/$PROJECT_ID/krte:latest-$_CONFIG
     dir: images/krte
 substitutions:
   _GIT_TAG: '12345'
@@ -28,8 +34,7 @@ substitutions:
   _OLD_BAZEL_VERSION: 2.2.0
   _UPGRADE_DOCKER: 'false'
   _CFSSL_VERSION: R1.2
+  _GO_ARCH: amd64
+  _PLATFORM: amd64
 options:
   substitution_option: ALLOW_LOOSE
-images:
-  - 'gcr.io/$PROJECT_ID/krte:$_GIT_TAG-$_CONFIG'
-  - 'gcr.io/$PROJECT_ID/krte:latest-$_CONFIG'
diff --git a/images/kubekins-e2e/Dockerfile b/images/kubekins-e2e/Dockerfile
index 38ac22a97516..074369c97617 100644
--- a/images/kubekins-e2e/Dockerfile
+++ b/images/kubekins-e2e/Dockerfile
@@ -50,22 +50,31 @@ RUN pip install --no-cache-dir awscli
 # The invocation at the end is to prevent download failures downloads as in the bug.
 # TODO(porridge): bump CFSSL_VERSION to one where cfssljson supports the -version flag and test it as well.
 ARG CFSSL_VERSION
-RUN wget -q -O cfssl "https://pkg.cfssl.org/${CFSSL_VERSION}/cfssl_linux-amd64" && \
+ARG TARGETARCH
+RUN if [ "${TARGETARCH}" = "amd64" ]; then \
+    wget -q -O cfssl "https://pkg.cfssl.org/${CFSSL_VERSION}/cfssl_linux-amd64" && \
     wget -q -O cfssljson "https://pkg.cfssl.org/${CFSSL_VERSION}/cfssljson_linux-amd64" && \
     chmod +x cfssl cfssljson && \
     mv cfssl cfssljson /usr/local/bin && \
-    cfssl version
+    cfssl version; \
+    fi
+RUN if [ "${TARGETARCH}" = "arm64" ]; then \
+    apt-get update && \
+    apt-get install -y golang-cfssl && \
+    rm -rf /var/lib/apt/lists/* && \
+    cfssl version; \
+    fi
 
 # replace kubectl with one from K8S_RELEASE
 ARG K8S_RELEASE=latest
 RUN rm -f $(which kubectl) && \
     export KUBECTL_VERSION=$(curl https://storage.googleapis.com/kubernetes-release/release/${K8S_RELEASE}.txt) && \
-    wget https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl && \
+    wget -q https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH:-amd64}/kubectl -O /usr/local/bin/kubectl && \
     chmod +x /usr/local/bin/kubectl
 
 # install go
 ARG GO_VERSION
-ENV GO_TARBALL "go${GO_VERSION}.linux-amd64.tar.gz"
+ENV GO_TARBALL "go${GO_VERSION}.linux-${TARGETARCH:-amd64}.tar.gz"
 RUN wget -q "https://storage.googleapis.com/golang/${GO_TARBALL}" && \
     tar xzf "${GO_TARBALL}" -C /usr/local && \
     rm "${GO_TARBALL}"
@@ -74,7 +83,8 @@ RUN wget -q "https://storage.googleapis.com/golang/${GO_TARBALL}" && \
 ARG BAZEL_VERSION_ARG
 ENV BAZEL_VERSION=${BAZEL_VERSION_ARG}
 COPY images/kubekins-e2e/install-bazel.sh /
-RUN bash /install-bazel.sh
+RUN if [ "${TARGETARCH}" = "amd64" ]; then bash /install-bazel.sh; fi
+RUN if [ "${TARGETARCH}" = "arm64" ]; then go install github.com/bazelbuild/bazelisk@latest; fi
 
 ARG OLD_BAZEL_VERSION
 COPY --from=old \
@@ -93,18 +103,24 @@ RUN [ "${UPGRADE_DOCKER_ARG}" = "true" ] && \
 # install kind if a version is provided
 ARG KIND_VERSION
 RUN if [ -n "${KIND_VERSION}" ]; then \
-    wget -q -O /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 && \
+    wget -q -O /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-${TARGETARCH:-amd64} && \
     chmod +x /usr/local/bin/kind; \
     fi
 
 # install kubetest2 binaries if a version is provided
 ARG KUBETEST2_VERSION
-RUN if [ -n "${KUBETEST2_VERSION}" ]; then \
+RUN if [ -n "${KUBETEST2_VERSION}" && "${TARGETARCH}" = "amd64" ]; then \
     wget -q -O /usr/local/bin/kubetest2.tgz https://storage.googleapis.com/k8s-staging-kubetest2/${KUBETEST2_VERSION}/linux-amd64.tgz && \
     tar -xzf /usr/local/bin/kubetest2.tgz -C /usr/local/bin && \
     rm -f /usr/local/bin/kubetest2.tgz && \
     chmod +x /usr/local/bin/kubetest2*; \
     fi
+RUN if [ -n "${KUBETEST2_VERSION}" && "${TARGETARCH}" = "arm64" ]; then \
+    wget -q -O kubetest2.zip https://github.com/kubernetes-sigs/kubetest2/archive/refs/heads/master.zip && \
+    unzip -q kubetest2.zip && \
+    cd kubetest2-master/ && make install-all && \
+    cd .. && rm -rf kubetest2-master && rm -f kubetest2.zip; \
+    fi
 
 # configure dockerd to use mirror.gcr.io
 # per instructions at https://cloud.google.com/container-registry/docs/pulling-cached-images
diff --git a/images/kubekins-e2e/cloudbuild.yaml b/images/kubekins-e2e/cloudbuild.yaml
index f995b094d7b4..60cf6aaa7454 100644
--- a/images/kubekins-e2e/cloudbuild.yaml
+++ b/images/kubekins-e2e/cloudbuild.yaml
@@ -9,32 +9,38 @@ steps:
     env:
     - CGO_ENABLED=0
     - GOOS=linux
-    - GOARCH=amd64
+    - GOARCH=$_GO_ARCH
     - GO111MODULE=on
     - GOPROXY=https://proxy.golang.org
     - GOSUMDB=sum.golang.org
-  - name: gcr.io/cloud-builders/docker
+  - name: gcr.io/k8s-testimages/gcb-docker-gcloud
+    entrypoint: /buildx-entrypoint
     args:
-    - build
-    - --tag=gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG
-    - --build-arg=BAZEL_VERSION_ARG=$_BAZEL_VERSION
-    - --build-arg=CFSSL_VERSION=$_CFSSL_VERSION
-    - --build-arg=GO_VERSION=$_GO_VERSION
-    - --build-arg=IMAGE_ARG=gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG
-    - --build-arg=K8S_RELEASE=$_K8S_RELEASE
-    - --build-arg=OLD_BAZEL_VERSION=$_OLD_BAZEL_VERSION
-    - --build-arg=UPGRADE_DOCKER_ARG=$_UPGRADE_DOCKER
-    - --build-arg=KIND_VERSION=$_KIND_VERSION
-    - --build-arg=KUBETEST2_VERSION=$_KUBETEST2_VERSION
-    - -f
-    - images/kubekins-e2e/Dockerfile
-    - .
+      - build
+      - --tag=gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG
+      - --platform=linux/$_PLATFORM
+      - --build-arg=BAZEL_VERSION_ARG=$_BAZEL_VERSION
+      - --build-arg=CFSSL_VERSION=$_CFSSL_VERSION
+      - --build-arg=GO_VERSION=$_GO_VERSION
+      - --build-arg=IMAGE_ARG=gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG
+      - --build-arg=K8S_RELEASE=$_K8S_RELEASE
+      - --build-arg=OLD_BAZEL_VERSION=$_OLD_BAZEL_VERSION
+      - --build-arg=UPGRADE_DOCKER_ARG=$_UPGRADE_DOCKER
+      - --build-arg=KIND_VERSION=$_KIND_VERSION
+      - --build-arg=KUBETEST2_VERSION=$_KUBETEST2_VERSION
+      - -f
+      - images/kubekins-e2e/Dockerfile
+      - --push
+      - .
     dir: .
-  - name: gcr.io/cloud-builders/docker
+  - name: gcr.io/k8s-testimages/gcb-docker-gcloud
+    entrypoint: gcloud
     args:
-    - tag
-    - gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG
-    - gcr.io/$PROJECT_ID/kubekins-e2e:latest-$_CONFIG
+      - container
+      - images
+      - add-tag
+      - gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG
+      - gcr.io/$PROJECT_ID/kubekins-e2e:latest-$_CONFIG
     dir: .
 substitutions:
   _BAZEL_VERSION: 3.4.1
@@ -47,8 +53,7 @@ substitutions:
   _UPGRADE_DOCKER: 'false'
   _KIND_VERSION: ''
   _KUBETEST2_VERSION: ''
+  _GO_ARCH: amd64
+  _PLATFORM: amd64
 options:
   substitution_option: ALLOW_LOOSE
-images:
-  - 'gcr.io/$PROJECT_ID/kubekins-e2e:$_GIT_TAG-$_CONFIG'
-  - 'gcr.io/$PROJECT_ID/kubekins-e2e:latest-$_CONFIG'
diff --git a/images/kubekins-e2e/variants.yaml b/images/kubekins-e2e/variants.yaml
index 93da9a5f1bbb..d46ed72d3bbf 100644
--- a/images/kubekins-e2e/variants.yaml
+++ b/images/kubekins-e2e/variants.yaml
@@ -56,3 +56,11 @@ variants:
     K8S_RELEASE: stable-1.18
     BAZEL_VERSION: 2.2.0
     OLD_BAZEL_VERSION: 0.23.2
+  arm64:
+    CONFIG: "arm64"
+    GO_VERSION: 1.17.3
+    K8S_RELEASE: stable
+    BAZEL_VERSION: 3.4.1
+    OLD_BAZEL_VERSION: 2.2.0
+    GO_ARCH: "arm64"
+    PLATFORM: "arm64/v8"