From 237f6c3fd9f96b4b68b4ad87ae9cb3749a2599f2 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Sat, 18 Apr 2020 22:46:18 -0500
Subject: [PATCH 01/21] cleaning root_test file

---
 cmd/minikube/cmd/root_test.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/root_test.go b/cmd/minikube/cmd/root_test.go
index b0c29ca4b14b..32df7d09d626 100644
--- a/cmd/minikube/cmd/root_test.go
+++ b/cmd/minikube/cmd/root_test.go
@@ -115,7 +115,12 @@ func hideEnv(t *testing.T) func(t *testing.T) {
 func TestPreRunDirectories(t *testing.T) {
 	// Make sure we create the required directories.
 	tempDir := tests.MakeTempDir()
-	defer os.RemoveAll(tempDir)
+	defer func() { //clean up tempdir
+		err := os.RemoveAll(tempDir)
+		if err != nil {
+			t.Errorf("failed to clean up temp folder  %q", tempDir)
+		}
+	}()
 
 	runCommand(RootCmd.PersistentPreRun)
 

From 7d3c68ca1247b52a60276065013243f6b8cc7d02 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Tue, 21 Apr 2020 23:55:57 -0500
Subject: [PATCH 02/21] add scripts

---
 hack/images/files/clean-install |  19 ++++
 hack/images/files/entrypoint    | 196 ++++++++++++++++++++++++++++++++
 hack/images/kicbase.Dockerfile  | 162 +++++++++++++++++++++++---
 3 files changed, 360 insertions(+), 17 deletions(-)
 create mode 100644 hack/images/files/clean-install
 create mode 100644 hack/images/files/entrypoint

diff --git a/hack/images/files/clean-install b/hack/images/files/clean-install
new file mode 100644
index 000000000000..dd769ff57dba
--- /dev/null
+++ b/hack/images/files/clean-install
@@ -0,0 +1,19 @@
+set -o errexit
+
+if [ $# = 0 ]; then
+  echo >&2 "No packages specified"
+  exit 1
+fi
+
+apt-get update
+apt-get install -y --no-install-recommends "$@"
+apt-get clean -y
+rm -rf \
+   /var/cache/debconf/* \
+   /var/lib/apt/lists/* \
+   /var/log/* \
+   /tmp/* \
+   /var/tmp/* \
+   /usr/share/doc/* \
+   /usr/share/man/* \
+   /usr/share/local/*
\ No newline at end of file
diff --git a/hack/images/files/entrypoint b/hack/images/files/entrypoint
new file mode 100644
index 000000000000..ebdc61546336
--- /dev/null
+++ b/hack/images/files/entrypoint
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+fix_mount() {
+  echo 'INFO: ensuring we can execute /bin/mount even with userns-remap'
+  # necessary only when userns-remap is enabled on the host, but harmless
+  # The binary /bin/mount should be owned by root and have the setuid bit
+  chown root:root /bin/mount
+  chmod -s /bin/mount
+
+  # This is a workaround to an AUFS bug that might cause `Text file
+  # busy` on `mount` command below. See more details in
+  # https://github.com/moby/moby/issues/9547
+  if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
+    echo 'INFO: detected aufs, calling sync'
+    sync
+  fi
+
+  echo 'INFO: remounting /sys read-only'
+  # systemd-in-a-container should have read only /sys
+  # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+  # however, we need other things from `docker run --privileged` ...
+  # and this flag also happens to make /sys rw, amongst other things
+  mount -o remount,ro /sys
+
+  echo 'INFO: making mounts shared'
+  # for mount propagation
+  mount --make-rshared /
+}
+
+fix_cgroup() {
+  echo 'INFO: fix cgroup mounts for all subsystems'
+  # For each cgroup subsystem, Docker does a bind mount from the current
+  # cgroup to the root of the cgroup subsystem. For instance:
+  #   /sys/fs/cgroup/memory/docker/<cid> -> /sys/fs/cgroup/memory
+  #
+  # This will confuse Kubelet and cadvisor and will dump the following error
+  # messages in kubelet log:
+  #   `summary_sys_containers.go:47] Failed to get system container stats for ".../kubelet.service"`
+  #
+  # This is because `/proc/<pid>/cgroup` is not affected by the bind mount.
+  # The following is a workaround to recreate the original cgroup
+  # environment by doing another bind mount for each subsystem.
+  local docker_cgroup_mounts
+  docker_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep docker || true)
+  if [[ -n "${docker_cgroup_mounts}" ]]; then
+    local docker_cgroup cgroup_subsystems subsystem
+    docker_cgroup=$(echo "${docker_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
+    cgroup_subsystems=$(echo "${docker_cgroup_mounts}" | cut -d' ' -f 5)
+    echo "${cgroup_subsystems}" |
+    while IFS= read -r subsystem; do
+      mkdir -p "${subsystem}${docker_cgroup}"
+      mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
+    done
+  fi
+}
+
+fix_machine_id() {
+  # Deletes the machine-id embedded in the node image and generates a new one.
+  # This is necessary because both kubelet and other components like weave net
+  # use machine-id internally to distinguish nodes.
+  echo 'INFO: clearing and regenerating /etc/machine-id'
+  rm -f /etc/machine-id
+  systemd-machine-id-setup
+}
+
+fix_product_name() {
+  # this is a small fix to hide the underlying hardware and fix issue #426
+  # https://github.com/kubernetes-sigs/kind/issues/426
+  if [[ -f /sys/class/dmi/id/product_name ]]; then
+    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"'
+    echo 'kind' > /kind/product_name
+    mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
+  fi
+}
+
+fix_product_uuid() {
+  # The system UUID is usually read from DMI via sysfs, the problem is that
+  # in the kind case this means that all (container) nodes share the same
+  # system/product uuid, as they share the same DMI.
+  # Note: The UUID is read from DMI, this tool is overwriting the sysfs files
+  # which should fix the attached issue, but this workaround does not address
+  # the issue if a tool is reading directly from DMI.
+  # https://github.com/kubernetes-sigs/kind/issues/1027
+  [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid
+  if [[ -f /sys/class/dmi/id/product_uuid ]]; then
+    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random'
+    mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid
+  fi
+  if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then
+    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well'
+    mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid
+  fi
+}
+
+fix_kmsg() {
+  # In environments where /dev/kmsg is not available, the kubelet (1.15+) won't
+  # start because it cannot open /dev/kmsg when starting the kmsgparser in the
+  # OOM parser.
+  # To support those environments, we link /dev/kmsg to /dev/console.
+  # https://github.com/kubernetes-sigs/kind/issues/662
+  if [[ ! -e /dev/kmsg ]]; then
+    if [[ -e /dev/console ]]; then
+      echo 'WARN: /dev/kmsg does not exist, symlinking /dev/console' >&2
+      ln -s /dev/console /dev/kmsg
+    else
+      echo 'WARN: /dev/kmsg does not exist, nor does /dev/console!' >&2
+    fi
+  fi
+}
+
+configure_proxy() {
+  # ensure all processes receive the proxy settings by default
+  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
+  mkdir -p /etc/systemd/system.conf.d/
+  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
+[Manager]
+DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
+EOF
+}
+
+select_iptables() {
+  # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper
+  local mode=nft
+  num_legacy_lines=$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l || true)
+  if [ "${num_legacy_lines}" -ge 10 ]; then
+    mode=legacy
+  else
+    num_nft_lines=$( (timeout 5 sh -c "iptables-nft-save; ip6tables-nft-save" || true) 2>/dev/null | grep '^-' | wc -l || true)
+    if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
+      mode=legacy
+    fi
+  fi
+
+  echo "INFO: setting iptables to detected mode: ${mode}"
+  update-alternatives --set iptables "/usr/sbin/iptables-${mode}" > /dev/null
+  update-alternatives --set ip6tables "/usr/sbin/ip6tables-${mode}" > /dev/null
+}
+
+enable_network_magic(){
+  # well-known docker embedded DNS is at 127.0.0.11:53
+  local docker_embedded_dns_ip='127.0.0.11'
+
+  # first we need to detect an IP to use for reaching the docker host
+  local docker_host_ip
+  docker_host_ip="$( (getent ahostsv4 'host.docker.internal' | head -n1 | cut -d' ' -f1) || true)"
+  if [[ -z "${docker_host_ip}" ]]; then
+    docker_host_ip=$(ip -4 route show default | cut -d' ' -f3)
+  fi
+
+  # patch docker's iptables rules to switch out the DNS IP
+  iptables-save \
+    | sed \
+      `# switch docker DNS DNAT rules to our chosen IP` \
+      -e "s/-d ${docker_embedded_dns_ip}/-d ${docker_host_ip}/g" \
+      `# we need to also apply these rules to non-local traffic (from pods)` \
+      -e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
+      `# switch docker DNS SNAT rules rules to our chosen IP` \
+      -e "s/--to-source :53/--to-source ${docker_host_ip}:53/g"\
+    | iptables-restore
+
+  # now we can ensure that DNS is configured to use our IP
+  cp /etc/resolv.conf /etc/resolv.conf.original
+  sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf
+}
+
+# run pre-init fixups
+fix_kmsg
+fix_mount
+fix_cgroup
+fix_machine_id
+fix_product_name
+fix_product_uuid
+configure_proxy
+select_iptables
+enable_network_magic
+
+# we want the command (expected to be systemd) to be PID1, so exec to it
+exec "$@"
\ No newline at end of file
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 97a29b73710b..9b06d026fe58 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -1,32 +1,160 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# kind node base image
+#
+# For systemd + docker configuration used below, see the following references:
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+
+# start from ubuntu 19.10, this image is reasonably small as a starting point
+# for a kubernetes node image, it doesn't contain much we don't need
+FROM ubuntu:19.10 as base
+
+# Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
+# The repository contains latest stable releases and nightlies built for multiple architectures
+ARG CONTAINERD_VERSION="v1.3.3-14-g449e9269"
+# Configure CNI binaries from upstream
+ARG CNI_VERSION="v0.8.5"
+# Configure crictl binary from upstream
+ARG CRICTL_VERSION="v1.17.0"
+
+# copy in static files (configs, scripts)
+COPY files/ /usr/local/bin/
+
+# Install dependencies, first from apt, then from release tarballs.
+# NOTE: we use one RUN to minimize layers.
+#
+# First we must ensure that our util scripts are executable.
+#
+# The base image already has: ssh, apt, snapd, but we need to install more packages.
+# Packages installed are broken down into (each on a line):
+# - packages needed to run services (systemd)
+# - packages needed for kubernetes components
+# - packages needed by the container runtime
+# - misc packages kind uses itself
+# After installing packages we cleanup by:
+# - removing unwanted systemd services
+# - disabling kmsg in journald (these log entries would be confusing)
+#
+# Then we install containerd from our nightly build infrastructure, as this
+# build for multiple architectures and allows us to upgrade to patched releases
+# more quickly.
+#
+# Next we download and extract crictl and CNI plugin binaries from upstream.
+#
+# Next we ensure the /etc/kubernetes/manifests directory exists. Normally
+# a kubeadm debain / rpm package would ensure that this exists but we install
+# freshly built binaries directly when we build the node image.
+#
+# Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m
+# This is plenty after we've done initial setup for a node, but before we are
+# likely to try to export logs etc.
+
+RUN echo "Ensuring scripts are executable ..." \
+    && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \
+ && echo "Installing Packages ..." \
+    && DEBIAN_FRONTEND=noninteractive clean-install \
+      systemd \
+      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod gnupg libglib2.0-0\
+      libseccomp2 \
+      bash ca-certificates curl rsync \
+    && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
+        curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
+        apt-key add - < Release.key && apt-get update && \
+        apt-get install -y --no-install-recommends cri-o-1.17 podman \
+    && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
+    && rm -f /lib/systemd/system/multi-user.target.wants/* \
+    && rm -f /etc/systemd/system/*.wants/* \
+    && rm -f /lib/systemd/system/local-fs.target.wants/* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
+    && rm -f /lib/systemd/system/basic.target.wants/* \
+    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
+    && ln -s "$(which systemd)" /sbin/init \
+ && echo "Installing containerd ..." \
+    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
+    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-1.3.3-61-g60bc1282" \
+    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-1.3.3-61-g60bc1282.linux-amd64.tar.gz" \
+    && tar -C /usr/local -xzvf /tmp/containerd.tgz \
+    && rm -rf /tmp/containerd.tgz \
+    && rm -f /usr/local/bin/containerd-stress /usr/local/bin/containerd-shim-runc-v1 \
+    && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \
+    && chmod 755 /usr/local/sbin/runc \
+    && containerd --version \
+    # && systemctl enable containerd \
+ && echo "Installing crictl ..." \
+    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
+ && echo "Installing CNI binaries ..." \
+    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
+    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
+    && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
+    && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
+    && mkdir -p /opt/cni/bin \
+    && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \
+    && rm -rf /tmp/cni.tgz \
+    && find /opt/cni/bin -type f -not \( \
+         -iname host-local \
+         -o -iname ptp \
+         -o -iname portmap \
+         -o -iname loopback \
+      \) \
+      -delete \
+ && echo "Ensuring /etc/kubernetes/manifests" \
+    && mkdir -p /etc/kubernetes/manifests \
+ && echo "Adjusting systemd-tmpfiles timer" \
+    && sed -i /usr/lib/systemd/system/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#'
+
+# tell systemd that it is in docker (it will check for the container env)
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+ENV container docker
+# systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it)
+# https://bugzilla.redhat.com/show_bug.cgi?id=1201657
+STOPSIGNAL SIGRTMIN+3
+# NOTE: this is *only* for documentation, the entrypoint is overridden later
+ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+
 ARG COMMIT_SHA
 # using base image created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/Dockerfile
 # which is an ubuntu 19.10 with an entry-point that helps running systemd
 # could be changed to any debian that can run systemd
-FROM kindest/base:v20200317-92225082 as base
+# FROM kindest/base:v20200317-92225082 as base
 USER root
+
+# --------
 # specify version of everything explicitly using 'apt-cache policy'
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    lz4=1.9.1-1 \
-    gnupg=2.2.12-1ubuntu3 \ 
-    sudo=1.8.27-1ubuntu4.1 \
-    docker.io=19.03.2-0ubuntu1 \
-    openssh-server=1:8.0p1-6build1 \
-    dnsutils=1:9.11.5.P4+dfsg-5.1ubuntu2.1 \
+    lz4 \
+    # gnupg=2.2.12-1ubuntu3 \
+    sudo \
+    docker.io \
+    openssh-server\
+    dnsutils \
     # libglib2.0-0 is required for conmon, which is required for podman
-    libglib2.0-0=2.62.1-1 \
+    libglib2.0-0 \
     && rm /etc/crictl.yaml
 # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
-RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \    
-    curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
-    apt-key add - < Release.key && apt-get update && \
-    apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
+# RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
+#     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
+#     apt-key add - < Release.key && apt-get update && \
+#     apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
 # install podman
-RUN apt-get install -y --no-install-recommends podman=1.8.2~144
+# RUN apt-get install -y --no-install-recommends podman=1.8.2~144
 # disable non-docker runtimes by default
-RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml
+RUN systemctl disable containerd && systemctl disable crio
 # enable docker which is default
 RUN systemctl enable docker
-# making SSH work for docker container 
+# making SSH work for docker container
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
 RUN mkdir /var/run/sshd
 RUN echo 'root:root' |chpasswd
@@ -34,7 +162,7 @@ RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_confi
 RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
 EXPOSE 22
 # create docker user for minikube ssh. to match VM using "docker" as username
-RUN adduser --ingroup docker --disabled-password --gecos '' docker 
+RUN adduser --ingroup docker --disabled-password --gecos '' docker
 RUN adduser docker sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER docker
@@ -53,4 +181,4 @@ RUN apt-get clean -y && rm -rf \
   /usr/share/doc/* \
   /usr/share/man/* \
   /usr/share/local/* \
-  RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
+RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file

From 47d66a14b88c8c50a765c8ffb27ccd9d58a3988f Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Tue, 21 Apr 2020 23:58:21 -0500
Subject: [PATCH 03/21] change kic-base-image image build

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6989593454c8..c0dfe62c0fc7 100755
--- a/Makefile
+++ b/Makefile
@@ -525,7 +525,7 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base .
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base ./hack/images/
 
 .PHONY: upload-preloaded-images-tar
 upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.

From ee6f2b1d79cda763d512a723704f69433f17981d Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Wed, 22 Apr 2020 00:22:09 -0500
Subject: [PATCH 04/21] add ubuntu pinned image to docker file

---
 hack/images/kicbase.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 9b06d026fe58..5186aff947be 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -19,7 +19,7 @@
 
 # start from ubuntu 19.10, this image is reasonably small as a starting point
 # for a kubernetes node image, it doesn't contain much we don't need
-FROM ubuntu:19.10 as base
+FROM ubuntu:focal-20200319 as base
 
 # Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
 # The repository contains latest stable releases and nightlies built for multiple architectures

From dde771d0060cc0324e1d717d0908b8691302a642 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Wed, 22 Apr 2020 20:28:10 -0500
Subject: [PATCH 05/21] remove some layers of dockerfile

---
 hack/images/kicbase.Dockerfile | 36 ++++++++++++----------------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 5186aff947be..621f176559b7 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -129,10 +129,7 @@ ARG COMMIT_SHA
 # which is an ubuntu 19.10 with an entry-point that helps running systemd
 # could be changed to any debian that can run systemd
 # FROM kindest/base:v20200317-92225082 as base
-USER root
 
-# --------
-# specify version of everything explicitly using 'apt-cache policy'
 RUN apt-get update && apt-get install -y --no-install-recommends \
     lz4 \
     # gnupg=2.2.12-1ubuntu3 \
@@ -141,30 +138,23 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     openssh-server\
     dnsutils \
     # libglib2.0-0 is required for conmon, which is required for podman
-    libglib2.0-0 \
-    && rm /etc/crictl.yaml
-# install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
-# RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
-#     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
-#     apt-key add - < Release.key && apt-get update && \
-#     apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
-# install podman
-# RUN apt-get install -y --no-install-recommends podman=1.8.2~144
-# disable non-docker runtimes by default
-RUN systemctl disable containerd && systemctl disable crio
-# enable docker which is default
-RUN systemctl enable docker
+    libglib2.0-0
+
 # making SSH work for docker container
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
-RUN mkdir /var/run/sshd
-RUN echo 'root:root' |chpasswd
-RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
-RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+RUN systemctl disable containerd && systemctl disable crio \
+ && systemctl enable docker \
+ && mkdir /var/run/sshd \
+ && echo 'root:root' |chpasswd \
+ && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
+ && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config \
+ && adduser --ingroup docker --disabled-password --gecos '' docker \
+ && adduser docker sudo \
+ && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
 EXPOSE 22
 # create docker user for minikube ssh. to match VM using "docker" as username
-RUN adduser --ingroup docker --disabled-password --gecos '' docker
-RUN adduser docker sudo
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
 USER docker
 RUN mkdir /home/docker/.ssh
 USER root

From 425478c1af164b2eb2c8b2a602bf914a5048dce1 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Wed, 22 Apr 2020 22:33:29 -0500
Subject: [PATCH 06/21] add comments to Dockerfile

---
 hack/images/kicbase.Dockerfile | 68 +++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 621f176559b7..3710d51b8d0f 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -17,7 +17,7 @@
 # For systemd + docker configuration used below, see the following references:
 # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 
-# start from ubuntu 19.10, this image is reasonably small as a starting point
+# start from ubuntu 20.04, this image is reasonably small as a starting point
 # for a kubernetes node image, it doesn't contain much we don't need
 FROM ubuntu:focal-20200319 as base
 
@@ -47,6 +47,9 @@ COPY files/ /usr/local/bin/
 # - removing unwanted systemd services
 # - disabling kmsg in journald (these log entries would be confusing)
 #
+# Then we install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
+# along with podman
+#
 # Then we install containerd from our nightly build infrastructure, as this
 # build for multiple architectures and allows us to upgrade to patched releases
 # more quickly.
@@ -92,7 +95,6 @@ RUN echo "Ensuring scripts are executable ..." \
     && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \
     && chmod 755 /usr/local/sbin/runc \
     && containerd --version \
-    # && systemctl enable containerd \
  && echo "Installing crictl ..." \
     && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
  && echo "Installing CNI binaries ..." \
@@ -125,14 +127,10 @@ STOPSIGNAL SIGRTMIN+3
 ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 
 ARG COMMIT_SHA
-# using base image created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/Dockerfile
-# which is an ubuntu 19.10 with an entry-point that helps running systemd
-# could be changed to any debian that can run systemd
-# FROM kindest/base:v20200317-92225082 as base
 
+# specify version of everything explicitly using 'apt-cache policy'
 RUN apt-get update && apt-get install -y --no-install-recommends \
     lz4 \
-    # gnupg=2.2.12-1ubuntu3 \
     sudo \
     docker.io \
     openssh-server\
@@ -140,35 +138,37 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     # libglib2.0-0 is required for conmon, which is required for podman
     libglib2.0-0
 
-# making SSH work for docker container
+# In this step we First disable non-docker runtimes by default
+# then enable docker which is default
+# next making SSH work for docker container
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
-RUN systemctl disable containerd && systemctl disable crio \
- && systemctl enable docker \
- && mkdir /var/run/sshd \
- && echo 'root:root' |chpasswd \
- && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
- && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config \
- && adduser --ingroup docker --disabled-password --gecos '' docker \
- && adduser docker sudo \
- && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+# finally create docker user for minikube ssh. to match VM using "docker" as username
+RUN echo "disable non-docker runtimes ..." \
+    && systemctl disable containerd && systemctl disable crio \
+    && systemctl enable docker \
+ && echo "making SSH work for docker ..." \
+    && mkdir /var/run/sshd \
+    && echo 'root:root' |chpasswd \
+    && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
+    && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config \
+ && echo "create docker user for minikube ssh ..." \
+    && adduser --ingroup docker --disabled-password --gecos '' docker \
+    && adduser docker sudo \
+    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 EXPOSE 22
-# create docker user for minikube ssh. to match VM using "docker" as username
 
-USER docker
-RUN mkdir /home/docker/.ssh
-USER root
-# kind base-image entry-point expects a "kind" folder for product_name,product_uuid
-# https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
-RUN mkdir -p /kind
 # Deleting leftovers
-RUN apt-get clean -y && rm -rf \
-  /var/cache/debconf/* \
-  /var/lib/apt/lists/* \
-  /var/log/* \
-  /tmp/* \
-  /var/tmp/* \
-  /usr/share/doc/* \
-  /usr/share/man/* \
-  /usr/share/local/* \
-RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file
+RUN echo "creating docker folders && delete leftovers ..." \
+ && mkdir /home/docker/.ssh \
+    && mkdir -p /kind \
+ && apt-get clean -y && rm -rf \
+    /var/cache/debconf/* \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/doc/* \
+    /usr/share/man/* \
+    /usr/share/local/* \
+ && echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file

From 4186feb15458eb6fce17214e6e54fccb9bd6d495 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 00:04:35 -0500
Subject: [PATCH 07/21] combining more steps

---
 hack/images/kicbase.Dockerfile | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 3710d51b8d0f..4ed27bbbde03 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -75,7 +75,7 @@ RUN echo "Ensuring scripts are executable ..." \
     && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
         curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
         apt-key add - < Release.key && apt-get update && \
-        apt-get install -y --no-install-recommends cri-o-1.17 podman \
+        apt-get install -y --no-install-recommends cri-o-1.17 podman lz4 sudo docker.io dnsutils \
     && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
     && rm -f /lib/systemd/system/multi-user.target.wants/* \
     && rm -f /etc/systemd/system/*.wants/* \
@@ -129,14 +129,14 @@ ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 ARG COMMIT_SHA
 
 # specify version of everything explicitly using 'apt-cache policy'
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    lz4 \
-    sudo \
-    docker.io \
-    openssh-server\
-    dnsutils \
+# RUN apt-get update && apt-get install -y --no-install-recommends \
+    # lz4 \
+    # sudo \
+    # docker.io \
+    # openssh-server\
+    # dnsutils \
     # libglib2.0-0 is required for conmon, which is required for podman
-    libglib2.0-0
+    # libglib2.0-0
 
 # In this step we First disable non-docker runtimes by default
 # then enable docker which is default
@@ -147,6 +147,7 @@ RUN echo "disable non-docker runtimes ..." \
     && systemctl disable containerd && systemctl disable crio \
     && systemctl enable docker \
  && echo "making SSH work for docker ..." \
+    && apt-get install -y --no-install-recommends openssh-server \
     && mkdir /var/run/sshd \
     && echo 'root:root' |chpasswd \
     && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \

From 758193c6a22469817bc294a4908b21a6fd22d873 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 19:59:39 -0500
Subject: [PATCH 08/21] give some love to Dockerfile

---
 hack/images/kicbase.Dockerfile | 36 ++++++++++++----------------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 4ed27bbbde03..ee0a02176b4f 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -23,11 +23,11 @@ FROM ubuntu:focal-20200319 as base
 
 # Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
 # The repository contains latest stable releases and nightlies built for multiple architectures
-ARG CONTAINERD_VERSION="v1.3.3-14-g449e9269"
+# ARG CONTAINERD_VERSION="v1.3.3-14-g449e9269"
 # Configure CNI binaries from upstream
-ARG CNI_VERSION="v0.8.5"
+# ARG CNI_VERSION="v0.8.5"
 # Configure crictl binary from upstream
-ARG CRICTL_VERSION="v1.17.0"
+# ARG CRICTL_VERSION="v1.17.0"
 
 # copy in static files (configs, scripts)
 COPY files/ /usr/local/bin/
@@ -69,9 +69,10 @@ RUN echo "Ensuring scripts are executable ..." \
  && echo "Installing Packages ..." \
     && DEBIAN_FRONTEND=noninteractive clean-install \
       systemd \
-      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod gnupg libglib2.0-0\
+      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod gnupg libglib2.0-0 \
       libseccomp2 \
       bash ca-certificates curl rsync \
+      && rm -rf /var/lib/apt/lists/* \
     && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
         curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
         apt-key add - < Release.key && apt-get update && \
@@ -96,10 +97,11 @@ RUN echo "Ensuring scripts are executable ..." \
     && chmod 755 /usr/local/sbin/runc \
     && containerd --version \
  && echo "Installing crictl ..." \
-    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
+    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.17.0/crictl-v1.17.0-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
+    && rm -rf crictl-v1.17.0-linux-${ARCH}.tar.gz \
  && echo "Installing CNI binaries ..." \
     && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
+    && export CNI_TARBALL="v0.8.5/cni-plugins-linux-${ARCH}-v0.8.5.tgz" \
     && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
     && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
     && mkdir -p /opt/cni/bin \
@@ -128,26 +130,18 @@ ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 
 ARG COMMIT_SHA
 
-# specify version of everything explicitly using 'apt-cache policy'
-# RUN apt-get update && apt-get install -y --no-install-recommends \
-    # lz4 \
-    # sudo \
-    # docker.io \
-    # openssh-server\
-    # dnsutils \
-    # libglib2.0-0 is required for conmon, which is required for podman
-    # libglib2.0-0
-
 # In this step we First disable non-docker runtimes by default
 # then enable docker which is default
 # next making SSH work for docker container
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
-# finally create docker user for minikube ssh. to match VM using "docker" as username
+# next create docker user for minikube ssh. to match VM using "docker" as username
+# finally deleting leftovers
 RUN echo "disable non-docker runtimes ..." \
     && systemctl disable containerd && systemctl disable crio \
     && systemctl enable docker \
  && echo "making SSH work for docker ..." \
     && apt-get install -y --no-install-recommends openssh-server \
+    && rm -rf /var/lib/apt/lists/* \
     && mkdir /var/run/sshd \
     && echo 'root:root' |chpasswd \
     && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
@@ -155,12 +149,8 @@ RUN echo "disable non-docker runtimes ..." \
  && echo "create docker user for minikube ssh ..." \
     && adduser --ingroup docker --disabled-password --gecos '' docker \
     && adduser docker sudo \
-    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-
-EXPOSE 22
-
-# Deleting leftovers
-RUN echo "creating docker folders && delete leftovers ..." \
+    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+ && echo "creating docker folders && delete leftovers ..." \
  && mkdir /home/docker/.ssh \
     && mkdir -p /kind \
  && apt-get clean -y && rm -rf \

From d22268fdd9e2ab0dd3e8ad178217ce990750e905 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 20:05:47 -0500
Subject: [PATCH 09/21] restore tests files

---
 cmd/minikube/cmd/root_test.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/cmd/minikube/cmd/root_test.go b/cmd/minikube/cmd/root_test.go
index 32df7d09d626..b0c29ca4b14b 100644
--- a/cmd/minikube/cmd/root_test.go
+++ b/cmd/minikube/cmd/root_test.go
@@ -115,12 +115,7 @@ func hideEnv(t *testing.T) func(t *testing.T) {
 func TestPreRunDirectories(t *testing.T) {
 	// Make sure we create the required directories.
 	tempDir := tests.MakeTempDir()
-	defer func() { //clean up tempdir
-		err := os.RemoveAll(tempDir)
-		if err != nil {
-			t.Errorf("failed to clean up temp folder  %q", tempDir)
-		}
-	}()
+	defer os.RemoveAll(tempDir)
 
 	runCommand(RootCmd.PersistentPreRun)
 

From 1bf8d4db73ab5e67d028d3cbeb8d757661dc693d Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 23:31:02 -0500
Subject: [PATCH 10/21] remove unnecesary file

---
 hack/images/files/clean-install | 19 -------------------
 pkg/drivers/kic/types.go        |  3 ++-
 2 files changed, 2 insertions(+), 20 deletions(-)
 delete mode 100644 hack/images/files/clean-install

diff --git a/hack/images/files/clean-install b/hack/images/files/clean-install
deleted file mode 100644
index dd769ff57dba..000000000000
--- a/hack/images/files/clean-install
+++ /dev/null
@@ -1,19 +0,0 @@
-set -o errexit
-
-if [ $# = 0 ]; then
-  echo >&2 "No packages specified"
-  exit 1
-fi
-
-apt-get update
-apt-get install -y --no-install-recommends "$@"
-apt-get clean -y
-rm -rf \
-   /var/cache/debconf/* \
-   /var/lib/apt/lists/* \
-   /var/log/* \
-   /tmp/* \
-   /var/tmp/* \
-   /usr/share/doc/* \
-   /usr/share/man/* \
-   /usr/share/local/*
\ No newline at end of file
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 2b01019b2ed8..83fec3be1fd3 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -40,7 +40,8 @@ const (
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
+	//BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
+	BaseImage = fmt.Sprintf("kicbase-ruben:v3.3.3")
 )
 
 // Config is configuration for the kic driver used by registry

From 7cc5de1a0778fbc53871ac71f9c1df0c6e6e6186 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 23:31:28 -0500
Subject: [PATCH 11/21] give more love to Dockerfile

---
 hack/images/kicbase.Dockerfile | 66 ++++++++++++++++++++++------------
 1 file changed, 43 insertions(+), 23 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index ee0a02176b4f..9b25ee3aace9 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -21,18 +21,12 @@
 # for a kubernetes node image, it doesn't contain much we don't need
 FROM ubuntu:focal-20200319 as base
 
-# Configure containerd and runc binaries from kind-ci/containerd-nightlies repository
-# The repository contains latest stable releases and nightlies built for multiple architectures
-# ARG CONTAINERD_VERSION="v1.3.3-14-g449e9269"
-# Configure CNI binaries from upstream
-# ARG CNI_VERSION="v0.8.5"
-# Configure crictl binary from upstream
-# ARG CRICTL_VERSION="v1.17.0"
-
 # copy in static files (configs, scripts)
 COPY files/ /usr/local/bin/
 
-# Install dependencies, first from apt, then from release tarballs.
+ARG COMMIT_SHA
+
+# Set environment variable along with Install dependencies, first from apt, then from release tarballs.
 # NOTE: we use one RUN to minimize layers.
 #
 # First we must ensure that our util scripts are executable.
@@ -64,19 +58,47 @@ COPY files/ /usr/local/bin/
 # This is plenty after we've done initial setup for a node, but before we are
 # likely to try to export logs etc.
 
-RUN echo "Ensuring scripts are executable ..." \
-    && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \
+RUN echo "set ENV variables ..." \
+ && export SYSTEM_VERSION="245.4-4ubuntu3" \
+    CONNTRACK_VERSION="1:1.4.5-2" \
+    IPTABLES_VERSION="1.8.4-3ubuntu2" \
+    IPROUTE2_VERSION="5.5.0-1ubuntu1" \
+    ETHTOOL_VERSION="1:5.4-1" \
+    SOCAT_VERSION="1.7.3.3-2" \
+    UTIL_LINUX_VERSION="2.34-0.1ubuntu9" \
+    MOUNT_VERSION="2.34-0.1ubuntu9" \
+    EBTABLES_VERSION="2.0.11-3build1" \
+    UDEV_VERSION="245.4-4ubuntu3" \
+    KMOD_VERSION="27-1ubuntu2" \
+    GNUPG_VERSION="2.2.19-3ubuntu2" \
+    LIBGLIB2_VERSION="2.64.2-1~fakesync1" \
+    LIBSECCOMP2_VERSION="2.4.3-1ubuntu1" \
+    CA_CERTIFICATES_VERSION="20190110ubuntu1" \
+    CURL_VERSION="7.68.0-1ubuntu2" \
+    RSYNC_VERSION="3.1.3-8" \
+    CRIO_VERSION="1.17.3~2" \
+    PODMAN_VERSION="1.9.0~2" \
+    LZ4_VERSION="1.9.2-2" \
+    SUDO_VERSION="1.8.31-1ubuntu1" \
+    DOCKER_VERSION="19.03.8-0ubuntu1" \
+    DNSUTILS_VERSION="1:9.16.1-0ubuntu2" \
+    CNI_VERSION="v0.8.5" \
+    CRICTL_VERSION="v1.17.0" \
+ && echo "Ensuring scripts are executable ..." \
+    && chmod +x /usr/local/bin/entrypoint \
  && echo "Installing Packages ..." \
-    && DEBIAN_FRONTEND=noninteractive clean-install \
-      systemd \
-      conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod gnupg libglib2.0-0 \
-      libseccomp2 \
-      bash ca-certificates curl rsync \
+    && DEBIAN_FRONTEND=noninteractive apt-get update && \
+      apt-get install -y --no-install-recommends \
+      systemd=${SYSTEM_VERSION} conntrack=${CONNTRACK_VERSION} iptables=${IPTABLES_VERSION} iproute2=${IPROUTE2_VERSION} ethtool=${ETHTOOL_VERSION} \
+      socat=${SOCAT_VERSION} util-linux=${UTIL_LINUX_VERSION} mount=${MOUNT_VERSION} ebtables=${EBTABLES_VERSION} udev=${UDEV_VERSION} kmod=${KMOD_VERSION} \
+      gnupg=${GNUPG_VERSION} libglib2.0-0=${LIBGLIB2_VERSION} libseccomp2=${LIBSECCOMP2_VERSION} ca-certificates=${CA_CERTIFICATES_VERSION} \
+      curl=${CURL_VERSION} rsync=${RSYNC_VERSION} \
       && rm -rf /var/lib/apt/lists/* \
     && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
         curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
         apt-key add - < Release.key && apt-get update && \
-        apt-get install -y --no-install-recommends cri-o-1.17 podman lz4 sudo docker.io dnsutils \
+        apt-get install -y --no-install-recommends cri-o-1.17=${CRIO_VERSION} podman=${PODMAN_VERSION} lz4=${LZ4_VERSION} sudo=${SUDO_VERSION} \
+        docker.io=${DOCKER_VERSION} dnsutils=${DNSUTILS_VERSION} \
     && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
     && rm -f /lib/systemd/system/multi-user.target.wants/* \
     && rm -f /etc/systemd/system/*.wants/* \
@@ -97,11 +119,11 @@ RUN echo "Ensuring scripts are executable ..." \
     && chmod 755 /usr/local/sbin/runc \
     && containerd --version \
  && echo "Installing crictl ..." \
-    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.17.0/crictl-v1.17.0-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
-    && rm -rf crictl-v1.17.0-linux-${ARCH}.tar.gz \
+    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
+    && rm -rf crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz \
  && echo "Installing CNI binaries ..." \
     && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CNI_TARBALL="v0.8.5/cni-plugins-linux-${ARCH}-v0.8.5.tgz" \
+    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
     && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
     && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
     && mkdir -p /opt/cni/bin \
@@ -117,7 +139,7 @@ RUN echo "Ensuring scripts are executable ..." \
  && echo "Ensuring /etc/kubernetes/manifests" \
     && mkdir -p /etc/kubernetes/manifests \
  && echo "Adjusting systemd-tmpfiles timer" \
-    && sed -i /usr/lib/systemd/system/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#'
+    && sed -i /usr/lib/systemd/user/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#'
 
 # tell systemd that it is in docker (it will check for the container env)
 # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
@@ -128,8 +150,6 @@ STOPSIGNAL SIGRTMIN+3
 # NOTE: this is *only* for documentation, the entrypoint is overridden later
 ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
 
-ARG COMMIT_SHA
-
 # In this step we First disable non-docker runtimes by default
 # then enable docker which is default
 # next making SSH work for docker container

From 2d49c3b6111c038cf7dcd5f3d60220074b03956b Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 23:35:57 -0500
Subject: [PATCH 12/21] restore types.go

---
 pkg/drivers/kic/types.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 83fec3be1fd3..b71e4a937be6 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -18,7 +18,6 @@ package kic
 
 import (
 	"fmt"
-
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 )
 
@@ -40,8 +39,7 @@ const (
 
 var (
 	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
-	//BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
-	BaseImage = fmt.Sprintf("kicbase-ruben:v3.3.3")
+	BaseImage = fmt.Sprintf("gcr.io/k8s-minikube/kicbase:%s@sha256:%s", Version, baseImageSHA)
 )
 
 // Config is configuration for the kic driver used by registry

From 536de08307938c89119eb87c6406ac84971f2655 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Thu, 23 Apr 2020 23:37:55 -0500
Subject: [PATCH 13/21] fix lint

---
 pkg/drivers/kic/types.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index b71e4a937be6..2b01019b2ed8 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -18,6 +18,7 @@ package kic
 
 import (
 	"fmt"
+
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 )
 

From 7d39857039233539b40ec5d786b412aea27ca39c Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 00:11:32 -0500
Subject: [PATCH 14/21] add containerd version

---
 hack/images/kicbase.Dockerfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 9b25ee3aace9..44d1ff3c8e34 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -17,7 +17,7 @@
 # For systemd + docker configuration used below, see the following references:
 # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 
-# start from ubuntu 20.04, this image is reasonably small as a starting point
+# start from ubuntu 20.04(focal), this image is reasonably small as a starting point
 # for a kubernetes node image, it doesn't contain much we don't need
 FROM ubuntu:focal-20200319 as base
 
@@ -84,6 +84,7 @@ RUN echo "set ENV variables ..." \
     DNSUTILS_VERSION="1:9.16.1-0ubuntu2" \
     CNI_VERSION="v0.8.5" \
     CRICTL_VERSION="v1.17.0" \
+    CONTAINERD_VERSION="1.3.3-61-g60bc1282" \
  && echo "Ensuring scripts are executable ..." \
     && chmod +x /usr/local/bin/entrypoint \
  && echo "Installing Packages ..." \
@@ -110,8 +111,8 @@ RUN echo "set ENV variables ..." \
     && ln -s "$(which systemd)" /sbin/init \
  && echo "Installing containerd ..." \
     && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-1.3.3-61-g60bc1282" \
-    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-1.3.3-61-g60bc1282.linux-amd64.tar.gz" \
+    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION}" \
+    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz" \
     && tar -C /usr/local -xzvf /tmp/containerd.tgz \
     && rm -rf /tmp/containerd.tgz \
     && rm -f /usr/local/bin/containerd-stress /usr/local/bin/containerd-shim-runc-v1 \

From 7f3465371efd54ee4d4e0ccea538fd93999f26e3 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 10:10:11 -0500
Subject: [PATCH 15/21] change Dockerfile copyright

---
 hack/images/kicbase.Dockerfile | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 44d1ff3c8e34..41756b9b1ce6 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2018 The Kubernetes Authors.
+# Copyright 2019 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# kind node base image
-#
-# For systemd + docker configuration used below, see the following references:
-# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
-
-# start from ubuntu 20.04(focal), this image is reasonably small as a starting point
-# for a kubernetes node image, it doesn't contain much we don't need
 FROM ubuntu:focal-20200319 as base
 
 # copy in static files (configs, scripts)
@@ -59,7 +52,7 @@ ARG COMMIT_SHA
 # likely to try to export logs etc.
 
 RUN echo "set ENV variables ..." \
- && export SYSTEM_VERSION="245.4-4ubuntu3" \
+ && export SYSTEMD_VERSION="245.4-4ubuntu3" \
     CONNTRACK_VERSION="1:1.4.5-2" \
     IPTABLES_VERSION="1.8.4-3ubuntu2" \
     IPROUTE2_VERSION="5.5.0-1ubuntu1" \
@@ -90,7 +83,7 @@ RUN echo "set ENV variables ..." \
  && echo "Installing Packages ..." \
     && DEBIAN_FRONTEND=noninteractive apt-get update && \
       apt-get install -y --no-install-recommends \
-      systemd=${SYSTEM_VERSION} conntrack=${CONNTRACK_VERSION} iptables=${IPTABLES_VERSION} iproute2=${IPROUTE2_VERSION} ethtool=${ETHTOOL_VERSION} \
+      systemd=${SYSTEMD_VERSION} conntrack=${CONNTRACK_VERSION} iptables=${IPTABLES_VERSION} iproute2=${IPROUTE2_VERSION} ethtool=${ETHTOOL_VERSION} \
       socat=${SOCAT_VERSION} util-linux=${UTIL_LINUX_VERSION} mount=${MOUNT_VERSION} ebtables=${EBTABLES_VERSION} udev=${UDEV_VERSION} kmod=${KMOD_VERSION} \
       gnupg=${GNUPG_VERSION} libglib2.0-0=${LIBGLIB2_VERSION} libseccomp2=${LIBSECCOMP2_VERSION} ca-certificates=${CA_CERTIFICATES_VERSION} \
       curl=${CURL_VERSION} rsync=${RSYNC_VERSION} \

From 258ebf85deaa2ce1057815603406a4ef92b4e0e8 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 19:11:28 -0500
Subject: [PATCH 16/21] remove comments

---
 hack/images/kicbase.Dockerfile | 33 ---------------------------------
 1 file changed, 33 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 41756b9b1ce6..bc20cae78401 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -14,43 +14,10 @@
 
 FROM ubuntu:focal-20200319 as base
 
-# copy in static files (configs, scripts)
 COPY files/ /usr/local/bin/
 
 ARG COMMIT_SHA
 
-# Set environment variable along with Install dependencies, first from apt, then from release tarballs.
-# NOTE: we use one RUN to minimize layers.
-#
-# First we must ensure that our util scripts are executable.
-#
-# The base image already has: ssh, apt, snapd, but we need to install more packages.
-# Packages installed are broken down into (each on a line):
-# - packages needed to run services (systemd)
-# - packages needed for kubernetes components
-# - packages needed by the container runtime
-# - misc packages kind uses itself
-# After installing packages we cleanup by:
-# - removing unwanted systemd services
-# - disabling kmsg in journald (these log entries would be confusing)
-#
-# Then we install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
-# along with podman
-#
-# Then we install containerd from our nightly build infrastructure, as this
-# build for multiple architectures and allows us to upgrade to patched releases
-# more quickly.
-#
-# Next we download and extract crictl and CNI plugin binaries from upstream.
-#
-# Next we ensure the /etc/kubernetes/manifests directory exists. Normally
-# a kubeadm debain / rpm package would ensure that this exists but we install
-# freshly built binaries directly when we build the node image.
-#
-# Finally we adjust tempfiles cleanup to be 1 minute after "boot" instead of 15m
-# This is plenty after we've done initial setup for a node, but before we are
-# likely to try to export logs etc.
-
 RUN echo "set ENV variables ..." \
  && export SYSTEMD_VERSION="245.4-4ubuntu3" \
     CONNTRACK_VERSION="1:1.4.5-2" \

From ff8e5c367e240ad63e0001eecbf492d0be2415ed Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 19:29:04 -0500
Subject: [PATCH 17/21] move to experimental folder

---
 .../files/entrypoint                          |   2 +
 .../kic-base-experimental/kicbase.Dockerfile  | 198 ++++++++++++++++++
 hack/images/kicbase.Dockerfile                | 198 +++++-------------
 3 files changed, 254 insertions(+), 144 deletions(-)
 rename hack/images/{ => kic-base-experimental}/files/entrypoint (98%)
 create mode 100644 hack/images/kic-base-experimental/kicbase.Dockerfile

diff --git a/hack/images/files/entrypoint b/hack/images/kic-base-experimental/files/entrypoint
similarity index 98%
rename from hack/images/files/entrypoint
rename to hack/images/kic-base-experimental/files/entrypoint
index ebdc61546336..663f180f49a2 100644
--- a/hack/images/files/entrypoint
+++ b/hack/images/kic-base-experimental/files/entrypoint
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# using entrypoint created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
+
 set -o errexit
 set -o nounset
 set -o pipefail
diff --git a/hack/images/kic-base-experimental/kicbase.Dockerfile b/hack/images/kic-base-experimental/kicbase.Dockerfile
new file mode 100644
index 000000000000..663f180f49a2
--- /dev/null
+++ b/hack/images/kic-base-experimental/kicbase.Dockerfile
@@ -0,0 +1,198 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# using entrypoint created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+fix_mount() {
+  echo 'INFO: ensuring we can execute /bin/mount even with userns-remap'
+  # necessary only when userns-remap is enabled on the host, but harmless
+  # The binary /bin/mount should be owned by root and have the setuid bit
+  chown root:root /bin/mount
+  chmod -s /bin/mount
+
+  # This is a workaround to an AUFS bug that might cause `Text file
+  # busy` on `mount` command below. See more details in
+  # https://github.com/moby/moby/issues/9547
+  if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
+    echo 'INFO: detected aufs, calling sync'
+    sync
+  fi
+
+  echo 'INFO: remounting /sys read-only'
+  # systemd-in-a-container should have read only /sys
+  # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+  # however, we need other things from `docker run --privileged` ...
+  # and this flag also happens to make /sys rw, amongst other things
+  mount -o remount,ro /sys
+
+  echo 'INFO: making mounts shared'
+  # for mount propagation
+  mount --make-rshared /
+}
+
+fix_cgroup() {
+  echo 'INFO: fix cgroup mounts for all subsystems'
+  # For each cgroup subsystem, Docker does a bind mount from the current
+  # cgroup to the root of the cgroup subsystem. For instance:
+  #   /sys/fs/cgroup/memory/docker/<cid> -> /sys/fs/cgroup/memory
+  #
+  # This will confuse Kubelet and cadvisor and will dump the following error
+  # messages in kubelet log:
+  #   `summary_sys_containers.go:47] Failed to get system container stats for ".../kubelet.service"`
+  #
+  # This is because `/proc/<pid>/cgroup` is not affected by the bind mount.
+  # The following is a workaround to recreate the original cgroup
+  # environment by doing another bind mount for each subsystem.
+  local docker_cgroup_mounts
+  docker_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep docker || true)
+  if [[ -n "${docker_cgroup_mounts}" ]]; then
+    local docker_cgroup cgroup_subsystems subsystem
+    docker_cgroup=$(echo "${docker_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
+    cgroup_subsystems=$(echo "${docker_cgroup_mounts}" | cut -d' ' -f 5)
+    echo "${cgroup_subsystems}" |
+    while IFS= read -r subsystem; do
+      mkdir -p "${subsystem}${docker_cgroup}"
+      mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
+    done
+  fi
+}
+
+fix_machine_id() {
+  # Deletes the machine-id embedded in the node image and generates a new one.
+  # This is necessary because both kubelet and other components like weave net
+  # use machine-id internally to distinguish nodes.
+  echo 'INFO: clearing and regenerating /etc/machine-id'
+  rm -f /etc/machine-id
+  systemd-machine-id-setup
+}
+
+fix_product_name() {
+  # this is a small fix to hide the underlying hardware and fix issue #426
+  # https://github.com/kubernetes-sigs/kind/issues/426
+  if [[ -f /sys/class/dmi/id/product_name ]]; then
+    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"'
+    echo 'kind' > /kind/product_name
+    mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
+  fi
+}
+
+fix_product_uuid() {
+  # The system UUID is usually read from DMI via sysfs, the problem is that
+  # in the kind case this means that all (container) nodes share the same
+  # system/product uuid, as they share the same DMI.
+  # Note: The UUID is read from DMI, this tool is overwriting the sysfs files
+  # which should fix the attached issue, but this workaround does not address
+  # the issue if a tool is reading directly from DMI.
+  # https://github.com/kubernetes-sigs/kind/issues/1027
+  [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid
+  if [[ -f /sys/class/dmi/id/product_uuid ]]; then
+    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random'
+    mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid
+  fi
+  if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then
+    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well'
+    mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid
+  fi
+}
+
+fix_kmsg() {
+  # In environments where /dev/kmsg is not available, the kubelet (1.15+) won't
+  # start because it cannot open /dev/kmsg when starting the kmsgparser in the
+  # OOM parser.
+  # To support those environments, we link /dev/kmsg to /dev/console.
+  # https://github.com/kubernetes-sigs/kind/issues/662
+  if [[ ! -e /dev/kmsg ]]; then
+    if [[ -e /dev/console ]]; then
+      echo 'WARN: /dev/kmsg does not exist, symlinking /dev/console' >&2
+      ln -s /dev/console /dev/kmsg
+    else
+      echo 'WARN: /dev/kmsg does not exist, nor does /dev/console!' >&2
+    fi
+  fi
+}
+
+configure_proxy() {
+  # ensure all processes receive the proxy settings by default
+  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
+  mkdir -p /etc/systemd/system.conf.d/
+  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
+[Manager]
+DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
+EOF
+}
+
+select_iptables() {
+  # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper
+  local mode=nft
+  num_legacy_lines=$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l || true)
+  if [ "${num_legacy_lines}" -ge 10 ]; then
+    mode=legacy
+  else
+    num_nft_lines=$( (timeout 5 sh -c "iptables-nft-save; ip6tables-nft-save" || true) 2>/dev/null | grep '^-' | wc -l || true)
+    if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
+      mode=legacy
+    fi
+  fi
+
+  echo "INFO: setting iptables to detected mode: ${mode}"
+  update-alternatives --set iptables "/usr/sbin/iptables-${mode}" > /dev/null
+  update-alternatives --set ip6tables "/usr/sbin/ip6tables-${mode}" > /dev/null
+}
+
+enable_network_magic(){
+  # well-known docker embedded DNS is at 127.0.0.11:53
+  local docker_embedded_dns_ip='127.0.0.11'
+
+  # first we need to detect an IP to use for reaching the docker host
+  local docker_host_ip
+  docker_host_ip="$( (getent ahostsv4 'host.docker.internal' | head -n1 | cut -d' ' -f1) || true)"
+  if [[ -z "${docker_host_ip}" ]]; then
+    docker_host_ip=$(ip -4 route show default | cut -d' ' -f3)
+  fi
+
+  # patch docker's iptables rules to switch out the DNS IP
+  iptables-save \
+    | sed \
+      `# switch docker DNS DNAT rules to our chosen IP` \
+      -e "s/-d ${docker_embedded_dns_ip}/-d ${docker_host_ip}/g" \
+      `# we need to also apply these rules to non-local traffic (from pods)` \
+      -e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
+      `# switch docker DNS SNAT rules rules to our chosen IP` \
+      -e "s/--to-source :53/--to-source ${docker_host_ip}:53/g"\
+    | iptables-restore
+
+  # now we can ensure that DNS is configured to use our IP
+  cp /etc/resolv.conf /etc/resolv.conf.original
+  sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf
+}
+
+# run pre-init fixups
+fix_kmsg
+fix_mount
+fix_cgroup
+fix_machine_id
+fix_product_name
+fix_product_uuid
+configure_proxy
+select_iptables
+enable_network_magic
+
+# we want the command (expected to be systemd) to be PID1, so exec to it
+exec "$@"
\ No newline at end of file
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index bc20cae78401..1dde011a91f1 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -1,146 +1,56 @@
-# Copyright 2019 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:focal-20200319 as base
-
-COPY files/ /usr/local/bin/
-
 ARG COMMIT_SHA
-
-RUN echo "set ENV variables ..." \
- && export SYSTEMD_VERSION="245.4-4ubuntu3" \
-    CONNTRACK_VERSION="1:1.4.5-2" \
-    IPTABLES_VERSION="1.8.4-3ubuntu2" \
-    IPROUTE2_VERSION="5.5.0-1ubuntu1" \
-    ETHTOOL_VERSION="1:5.4-1" \
-    SOCAT_VERSION="1.7.3.3-2" \
-    UTIL_LINUX_VERSION="2.34-0.1ubuntu9" \
-    MOUNT_VERSION="2.34-0.1ubuntu9" \
-    EBTABLES_VERSION="2.0.11-3build1" \
-    UDEV_VERSION="245.4-4ubuntu3" \
-    KMOD_VERSION="27-1ubuntu2" \
-    GNUPG_VERSION="2.2.19-3ubuntu2" \
-    LIBGLIB2_VERSION="2.64.2-1~fakesync1" \
-    LIBSECCOMP2_VERSION="2.4.3-1ubuntu1" \
-    CA_CERTIFICATES_VERSION="20190110ubuntu1" \
-    CURL_VERSION="7.68.0-1ubuntu2" \
-    RSYNC_VERSION="3.1.3-8" \
-    CRIO_VERSION="1.17.3~2" \
-    PODMAN_VERSION="1.9.0~2" \
-    LZ4_VERSION="1.9.2-2" \
-    SUDO_VERSION="1.8.31-1ubuntu1" \
-    DOCKER_VERSION="19.03.8-0ubuntu1" \
-    DNSUTILS_VERSION="1:9.16.1-0ubuntu2" \
-    CNI_VERSION="v0.8.5" \
-    CRICTL_VERSION="v1.17.0" \
-    CONTAINERD_VERSION="1.3.3-61-g60bc1282" \
- && echo "Ensuring scripts are executable ..." \
-    && chmod +x /usr/local/bin/entrypoint \
- && echo "Installing Packages ..." \
-    && DEBIAN_FRONTEND=noninteractive apt-get update && \
-      apt-get install -y --no-install-recommends \
-      systemd=${SYSTEMD_VERSION} conntrack=${CONNTRACK_VERSION} iptables=${IPTABLES_VERSION} iproute2=${IPROUTE2_VERSION} ethtool=${ETHTOOL_VERSION} \
-      socat=${SOCAT_VERSION} util-linux=${UTIL_LINUX_VERSION} mount=${MOUNT_VERSION} ebtables=${EBTABLES_VERSION} udev=${UDEV_VERSION} kmod=${KMOD_VERSION} \
-      gnupg=${GNUPG_VERSION} libglib2.0-0=${LIBGLIB2_VERSION} libseccomp2=${LIBSECCOMP2_VERSION} ca-certificates=${CA_CERTIFICATES_VERSION} \
-      curl=${CURL_VERSION} rsync=${RSYNC_VERSION} \
-      && rm -rf /var/lib/apt/lists/* \
-    && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
-        curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
-        apt-key add - < Release.key && apt-get update && \
-        apt-get install -y --no-install-recommends cri-o-1.17=${CRIO_VERSION} podman=${PODMAN_VERSION} lz4=${LZ4_VERSION} sudo=${SUDO_VERSION} \
-        docker.io=${DOCKER_VERSION} dnsutils=${DNSUTILS_VERSION} \
-    && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
-    && rm -f /lib/systemd/system/multi-user.target.wants/* \
-    && rm -f /etc/systemd/system/*.wants/* \
-    && rm -f /lib/systemd/system/local-fs.target.wants/* \
-    && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
-    && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
-    && rm -f /lib/systemd/system/basic.target.wants/* \
-    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
-    && ln -s "$(which systemd)" /sbin/init \
- && echo "Installing containerd ..." \
-    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION}" \
-    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz" \
-    && tar -C /usr/local -xzvf /tmp/containerd.tgz \
-    && rm -rf /tmp/containerd.tgz \
-    && rm -f /usr/local/bin/containerd-stress /usr/local/bin/containerd-shim-runc-v1 \
-    && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \
-    && chmod 755 /usr/local/sbin/runc \
-    && containerd --version \
- && echo "Installing crictl ..." \
-    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
-    && rm -rf crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz \
- && echo "Installing CNI binaries ..." \
-    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
-    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
-    && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
-    && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
-    && mkdir -p /opt/cni/bin \
-    && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \
-    && rm -rf /tmp/cni.tgz \
-    && find /opt/cni/bin -type f -not \( \
-         -iname host-local \
-         -o -iname ptp \
-         -o -iname portmap \
-         -o -iname loopback \
-      \) \
-      -delete \
- && echo "Ensuring /etc/kubernetes/manifests" \
-    && mkdir -p /etc/kubernetes/manifests \
- && echo "Adjusting systemd-tmpfiles timer" \
-    && sed -i /usr/lib/systemd/user/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#'
-
-# tell systemd that it is in docker (it will check for the container env)
-# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
-ENV container docker
-# systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it)
-# https://bugzilla.redhat.com/show_bug.cgi?id=1201657
-STOPSIGNAL SIGRTMIN+3
-# NOTE: this is *only* for documentation, the entrypoint is overridden later
-ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
-
-# In this step we First disable non-docker runtimes by default
-# then enable docker which is default
-# next making SSH work for docker container
+# using base image created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/Dockerfile
+# which is an ubuntu 19.10 with an entry-point that helps running systemd
+# could be changed to any debian that can run systemd
+FROM kindest/base:v20200317-92225082 as base
+USER root
+# specify version of everything explicitly using 'apt-cache policy'
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    lz4=1.9.1-1 \
+    gnupg=2.2.12-1ubuntu3 \
+    sudo=1.8.27-1ubuntu4.1 \
+    docker.io=19.03.2-0ubuntu1 \
+    openssh-server=1:8.0p1-6build1 \
+    dnsutils=1:9.11.5.P4+dfsg-5.1ubuntu2.1 \
+    # libglib2.0-0 is required for conmon, which is required for podman
+    libglib2.0-0=2.62.1-1 \
+    && rm /etc/crictl.yaml
+# install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
+RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
+    curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
+    apt-key add - < Release.key && apt-get update && \
+    apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
+# install podman
+RUN apt-get install -y --no-install-recommends podman=1.8.2~144
+# disable non-docker runtimes by default
+RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml
+# enable docker which is default
+RUN systemctl enable docker
+# making SSH work for docker container
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
-# next create docker user for minikube ssh. to match VM using "docker" as username
-# finally deleting leftovers
-RUN echo "disable non-docker runtimes ..." \
-    && systemctl disable containerd && systemctl disable crio \
-    && systemctl enable docker \
- && echo "making SSH work for docker ..." \
-    && apt-get install -y --no-install-recommends openssh-server \
-    && rm -rf /var/lib/apt/lists/* \
-    && mkdir /var/run/sshd \
-    && echo 'root:root' |chpasswd \
-    && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
-    && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config \
- && echo "create docker user for minikube ssh ..." \
-    && adduser --ingroup docker --disabled-password --gecos '' docker \
-    && adduser docker sudo \
-    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
- && echo "creating docker folders && delete leftovers ..." \
- && mkdir /home/docker/.ssh \
-    && mkdir -p /kind \
- && apt-get clean -y && rm -rf \
-    /var/cache/debconf/* \
-    /var/lib/apt/lists/* \
-    /var/log/* \
-    /tmp/* \
-    /var/tmp/* \
-    /usr/share/doc/* \
-    /usr/share/man/* \
-    /usr/share/local/* \
- && echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file
+RUN mkdir /var/run/sshd
+RUN echo 'root:root' |chpasswd
+RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+EXPOSE 22
+# create docker user for minikube ssh. to match VM using "docker" as username
+RUN adduser --ingroup docker --disabled-password --gecos '' docker
+RUN adduser docker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER docker
+RUN mkdir /home/docker/.ssh
+USER root
+# kind base-image entry-point expects a "kind" folder for product_name,product_uuid
+# https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
+RUN mkdir -p /kind
+# Deleting leftovers
+RUN apt-get clean -y && rm -rf \
+  /var/cache/debconf/* \
+  /var/lib/apt/lists/* \
+  /var/log/* \
+  /tmp/* \
+  /var/tmp/* \
+  /usr/share/doc/* \
+  /usr/share/man/* \
+  /usr/share/local/* \
+  RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file

From a7b78beee94ca504b49a5ec176a109360165cea0 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 19:37:14 -0500
Subject: [PATCH 18/21] restore Makefile

---
 Makefile                       | 2 +-
 hack/images/kicbase.Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c0dfe62c0fc7..6989593454c8 100755
--- a/Makefile
+++ b/Makefile
@@ -525,7 +525,7 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base ./hack/images/
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base .
 
 .PHONY: upload-preloaded-images-tar
 upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 1dde011a91f1..444c20dcf352 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -53,4 +53,4 @@ RUN apt-get clean -y && rm -rf \
   /usr/share/doc/* \
   /usr/share/man/* \
   /usr/share/local/* \
-  RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file
+  RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"

From 7f5aaec16521cc598ccb662c0542bc45ab5dfee8 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 19:50:29 -0500
Subject: [PATCH 19/21] nobody saw that :)

---
 .../kic-base-experimental/kicbase.Dockerfile  | 318 ++++++++----------
 1 file changed, 133 insertions(+), 185 deletions(-)

diff --git a/hack/images/kic-base-experimental/kicbase.Dockerfile b/hack/images/kic-base-experimental/kicbase.Dockerfile
index 663f180f49a2..bc20cae78401 100644
--- a/hack/images/kic-base-experimental/kicbase.Dockerfile
+++ b/hack/images/kic-base-experimental/kicbase.Dockerfile
@@ -1,6 +1,4 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2019 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,185 +12,135 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# using entrypoint created by kind https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-fix_mount() {
-  echo 'INFO: ensuring we can execute /bin/mount even with userns-remap'
-  # necessary only when userns-remap is enabled on the host, but harmless
-  # The binary /bin/mount should be owned by root and have the setuid bit
-  chown root:root /bin/mount
-  chmod -s /bin/mount
-
-  # This is a workaround to an AUFS bug that might cause `Text file
-  # busy` on `mount` command below. See more details in
-  # https://github.com/moby/moby/issues/9547
-  if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
-    echo 'INFO: detected aufs, calling sync'
-    sync
-  fi
-
-  echo 'INFO: remounting /sys read-only'
-  # systemd-in-a-container should have read only /sys
-  # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
-  # however, we need other things from `docker run --privileged` ...
-  # and this flag also happens to make /sys rw, amongst other things
-  mount -o remount,ro /sys
-
-  echo 'INFO: making mounts shared'
-  # for mount propagation
-  mount --make-rshared /
-}
-
-fix_cgroup() {
-  echo 'INFO: fix cgroup mounts for all subsystems'
-  # For each cgroup subsystem, Docker does a bind mount from the current
-  # cgroup to the root of the cgroup subsystem. For instance:
-  #   /sys/fs/cgroup/memory/docker/<cid> -> /sys/fs/cgroup/memory
-  #
-  # This will confuse Kubelet and cadvisor and will dump the following error
-  # messages in kubelet log:
-  #   `summary_sys_containers.go:47] Failed to get system container stats for ".../kubelet.service"`
-  #
-  # This is because `/proc/<pid>/cgroup` is not affected by the bind mount.
-  # The following is a workaround to recreate the original cgroup
-  # environment by doing another bind mount for each subsystem.
-  local docker_cgroup_mounts
-  docker_cgroup_mounts=$(grep /sys/fs/cgroup /proc/self/mountinfo | grep docker || true)
-  if [[ -n "${docker_cgroup_mounts}" ]]; then
-    local docker_cgroup cgroup_subsystems subsystem
-    docker_cgroup=$(echo "${docker_cgroup_mounts}" | head -n 1 | cut -d' ' -f 4)
-    cgroup_subsystems=$(echo "${docker_cgroup_mounts}" | cut -d' ' -f 5)
-    echo "${cgroup_subsystems}" |
-    while IFS= read -r subsystem; do
-      mkdir -p "${subsystem}${docker_cgroup}"
-      mount --bind "${subsystem}" "${subsystem}${docker_cgroup}"
-    done
-  fi
-}
-
-fix_machine_id() {
-  # Deletes the machine-id embedded in the node image and generates a new one.
-  # This is necessary because both kubelet and other components like weave net
-  # use machine-id internally to distinguish nodes.
-  echo 'INFO: clearing and regenerating /etc/machine-id'
-  rm -f /etc/machine-id
-  systemd-machine-id-setup
-}
-
-fix_product_name() {
-  # this is a small fix to hide the underlying hardware and fix issue #426
-  # https://github.com/kubernetes-sigs/kind/issues/426
-  if [[ -f /sys/class/dmi/id/product_name ]]; then
-    echo 'INFO: faking /sys/class/dmi/id/product_name to be "kind"'
-    echo 'kind' > /kind/product_name
-    mount -o ro,bind /kind/product_name /sys/class/dmi/id/product_name
-  fi
-}
-
-fix_product_uuid() {
-  # The system UUID is usually read from DMI via sysfs, the problem is that
-  # in the kind case this means that all (container) nodes share the same
-  # system/product uuid, as they share the same DMI.
-  # Note: The UUID is read from DMI, this tool is overwriting the sysfs files
-  # which should fix the attached issue, but this workaround does not address
-  # the issue if a tool is reading directly from DMI.
-  # https://github.com/kubernetes-sigs/kind/issues/1027
-  [[ ! -f /kind/product_uuid ]] && cat /proc/sys/kernel/random/uuid > /kind/product_uuid
-  if [[ -f /sys/class/dmi/id/product_uuid ]]; then
-    echo 'INFO: faking /sys/class/dmi/id/product_uuid to be random'
-    mount -o ro,bind /kind/product_uuid /sys/class/dmi/id/product_uuid
-  fi
-  if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then
-    echo 'INFO: faking /sys/devices/virtual/dmi/id/product_uuid as well'
-    mount -o ro,bind /kind/product_uuid /sys/devices/virtual/dmi/id/product_uuid
-  fi
-}
-
-fix_kmsg() {
-  # In environments where /dev/kmsg is not available, the kubelet (1.15+) won't
-  # start because it cannot open /dev/kmsg when starting the kmsgparser in the
-  # OOM parser.
-  # To support those environments, we link /dev/kmsg to /dev/console.
-  # https://github.com/kubernetes-sigs/kind/issues/662
-  if [[ ! -e /dev/kmsg ]]; then
-    if [[ -e /dev/console ]]; then
-      echo 'WARN: /dev/kmsg does not exist, symlinking /dev/console' >&2
-      ln -s /dev/console /dev/kmsg
-    else
-      echo 'WARN: /dev/kmsg does not exist, nor does /dev/console!' >&2
-    fi
-  fi
-}
-
-configure_proxy() {
-  # ensure all processes receive the proxy settings by default
-  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
-  mkdir -p /etc/systemd/system.conf.d/
-  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
-[Manager]
-DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
-EOF
-}
-
-select_iptables() {
-  # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper
-  local mode=nft
-  num_legacy_lines=$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l || true)
-  if [ "${num_legacy_lines}" -ge 10 ]; then
-    mode=legacy
-  else
-    num_nft_lines=$( (timeout 5 sh -c "iptables-nft-save; ip6tables-nft-save" || true) 2>/dev/null | grep '^-' | wc -l || true)
-    if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then
-      mode=legacy
-    fi
-  fi
-
-  echo "INFO: setting iptables to detected mode: ${mode}"
-  update-alternatives --set iptables "/usr/sbin/iptables-${mode}" > /dev/null
-  update-alternatives --set ip6tables "/usr/sbin/ip6tables-${mode}" > /dev/null
-}
-
-enable_network_magic(){
-  # well-known docker embedded DNS is at 127.0.0.11:53
-  local docker_embedded_dns_ip='127.0.0.11'
-
-  # first we need to detect an IP to use for reaching the docker host
-  local docker_host_ip
-  docker_host_ip="$( (getent ahostsv4 'host.docker.internal' | head -n1 | cut -d' ' -f1) || true)"
-  if [[ -z "${docker_host_ip}" ]]; then
-    docker_host_ip=$(ip -4 route show default | cut -d' ' -f3)
-  fi
-
-  # patch docker's iptables rules to switch out the DNS IP
-  iptables-save \
-    | sed \
-      `# switch docker DNS DNAT rules to our chosen IP` \
-      -e "s/-d ${docker_embedded_dns_ip}/-d ${docker_host_ip}/g" \
-      `# we need to also apply these rules to non-local traffic (from pods)` \
-      -e 's/-A OUTPUT \(.*\) -j DOCKER_OUTPUT/\0\n-A PREROUTING \1 -j DOCKER_OUTPUT/' \
-      `# switch docker DNS SNAT rules rules to our chosen IP` \
-      -e "s/--to-source :53/--to-source ${docker_host_ip}:53/g"\
-    | iptables-restore
-
-  # now we can ensure that DNS is configured to use our IP
-  cp /etc/resolv.conf /etc/resolv.conf.original
-  sed -e "s/${docker_embedded_dns_ip}/${docker_host_ip}/g" /etc/resolv.conf.original >/etc/resolv.conf
-}
-
-# run pre-init fixups
-fix_kmsg
-fix_mount
-fix_cgroup
-fix_machine_id
-fix_product_name
-fix_product_uuid
-configure_proxy
-select_iptables
-enable_network_magic
-
-# we want the command (expected to be systemd) to be PID1, so exec to it
-exec "$@"
\ No newline at end of file
+FROM ubuntu:focal-20200319 as base
+
+COPY files/ /usr/local/bin/
+
+ARG COMMIT_SHA
+
+RUN echo "set ENV variables ..." \
+ && export SYSTEMD_VERSION="245.4-4ubuntu3" \
+    CONNTRACK_VERSION="1:1.4.5-2" \
+    IPTABLES_VERSION="1.8.4-3ubuntu2" \
+    IPROUTE2_VERSION="5.5.0-1ubuntu1" \
+    ETHTOOL_VERSION="1:5.4-1" \
+    SOCAT_VERSION="1.7.3.3-2" \
+    UTIL_LINUX_VERSION="2.34-0.1ubuntu9" \
+    MOUNT_VERSION="2.34-0.1ubuntu9" \
+    EBTABLES_VERSION="2.0.11-3build1" \
+    UDEV_VERSION="245.4-4ubuntu3" \
+    KMOD_VERSION="27-1ubuntu2" \
+    GNUPG_VERSION="2.2.19-3ubuntu2" \
+    LIBGLIB2_VERSION="2.64.2-1~fakesync1" \
+    LIBSECCOMP2_VERSION="2.4.3-1ubuntu1" \
+    CA_CERTIFICATES_VERSION="20190110ubuntu1" \
+    CURL_VERSION="7.68.0-1ubuntu2" \
+    RSYNC_VERSION="3.1.3-8" \
+    CRIO_VERSION="1.17.3~2" \
+    PODMAN_VERSION="1.9.0~2" \
+    LZ4_VERSION="1.9.2-2" \
+    SUDO_VERSION="1.8.31-1ubuntu1" \
+    DOCKER_VERSION="19.03.8-0ubuntu1" \
+    DNSUTILS_VERSION="1:9.16.1-0ubuntu2" \
+    CNI_VERSION="v0.8.5" \
+    CRICTL_VERSION="v1.17.0" \
+    CONTAINERD_VERSION="1.3.3-61-g60bc1282" \
+ && echo "Ensuring scripts are executable ..." \
+    && chmod +x /usr/local/bin/entrypoint \
+ && echo "Installing Packages ..." \
+    && DEBIAN_FRONTEND=noninteractive apt-get update && \
+      apt-get install -y --no-install-recommends \
+      systemd=${SYSTEMD_VERSION} conntrack=${CONNTRACK_VERSION} iptables=${IPTABLES_VERSION} iproute2=${IPROUTE2_VERSION} ethtool=${ETHTOOL_VERSION} \
+      socat=${SOCAT_VERSION} util-linux=${UTIL_LINUX_VERSION} mount=${MOUNT_VERSION} ebtables=${EBTABLES_VERSION} udev=${UDEV_VERSION} kmod=${KMOD_VERSION} \
+      gnupg=${GNUPG_VERSION} libglib2.0-0=${LIBGLIB2_VERSION} libseccomp2=${LIBSECCOMP2_VERSION} ca-certificates=${CA_CERTIFICATES_VERSION} \
+      curl=${CURL_VERSION} rsync=${RSYNC_VERSION} \
+      && rm -rf /var/lib/apt/lists/* \
+    && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
+        curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
+        apt-key add - < Release.key && apt-get update && \
+        apt-get install -y --no-install-recommends cri-o-1.17=${CRIO_VERSION} podman=${PODMAN_VERSION} lz4=${LZ4_VERSION} sudo=${SUDO_VERSION} \
+        docker.io=${DOCKER_VERSION} dnsutils=${DNSUTILS_VERSION} \
+    && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
+    && rm -f /lib/systemd/system/multi-user.target.wants/* \
+    && rm -f /etc/systemd/system/*.wants/* \
+    && rm -f /lib/systemd/system/local-fs.target.wants/* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*udev* \
+    && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \
+    && rm -f /lib/systemd/system/basic.target.wants/* \
+    && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \
+    && ln -s "$(which systemd)" /sbin/init \
+ && echo "Installing containerd ..." \
+    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
+    && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION}" \
+    && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz" \
+    && tar -C /usr/local -xzvf /tmp/containerd.tgz \
+    && rm -rf /tmp/containerd.tgz \
+    && rm -f /usr/local/bin/containerd-stress /usr/local/bin/containerd-shim-runc-v1 \
+    && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \
+    && chmod 755 /usr/local/sbin/runc \
+    && containerd --version \
+ && echo "Installing crictl ..." \
+    && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \
+    && rm -rf crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz \
+ && echo "Installing CNI binaries ..." \
+    && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \
+    && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \
+    && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \
+    && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \
+    && mkdir -p /opt/cni/bin \
+    && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \
+    && rm -rf /tmp/cni.tgz \
+    && find /opt/cni/bin -type f -not \( \
+         -iname host-local \
+         -o -iname ptp \
+         -o -iname portmap \
+         -o -iname loopback \
+      \) \
+      -delete \
+ && echo "Ensuring /etc/kubernetes/manifests" \
+    && mkdir -p /etc/kubernetes/manifests \
+ && echo "Adjusting systemd-tmpfiles timer" \
+    && sed -i /usr/lib/systemd/user/systemd-tmpfiles-clean.timer -e 's#OnBootSec=.*#OnBootSec=1min#'
+
+# tell systemd that it is in docker (it will check for the container env)
+# https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
+ENV container docker
+# systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it)
+# https://bugzilla.redhat.com/show_bug.cgi?id=1201657
+STOPSIGNAL SIGRTMIN+3
+# NOTE: this is *only* for documentation, the entrypoint is overridden later
+ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]
+
+# In this step we First disable non-docker runtimes by default
+# then enable docker which is default
+# next making SSH work for docker container
+# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
+# next create docker user for minikube ssh. to match VM using "docker" as username
+# finally deleting leftovers
+RUN echo "disable non-docker runtimes ..." \
+    && systemctl disable containerd && systemctl disable crio \
+    && systemctl enable docker \
+ && echo "making SSH work for docker ..." \
+    && apt-get install -y --no-install-recommends openssh-server \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir /var/run/sshd \
+    && echo 'root:root' |chpasswd \
+    && sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config \
+    && sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config \
+ && echo "create docker user for minikube ssh ..." \
+    && adduser --ingroup docker --disabled-password --gecos '' docker \
+    && adduser docker sudo \
+    && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \
+ && echo "creating docker folders && delete leftovers ..." \
+ && mkdir /home/docker/.ssh \
+    && mkdir -p /kind \
+ && apt-get clean -y && rm -rf \
+    /var/cache/debconf/* \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/doc/* \
+    /usr/share/man/* \
+    /usr/share/local/* \
+ && echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
\ No newline at end of file

From f2df642fa1f8b0bb9200ef8ab8f2fd4e111030f0 Mon Sep 17 00:00:00 2001
From: Alonyb <rubenbaez1995@gmail.com>
Date: Fri, 24 Apr 2020 19:53:19 -0500
Subject: [PATCH 20/21] restore original Dockerfile

---
 hack/images/kicbase.Dockerfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 444c20dcf352..97a29b73710b 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -7,7 +7,7 @@ USER root
 # specify version of everything explicitly using 'apt-cache policy'
 RUN apt-get update && apt-get install -y --no-install-recommends \
     lz4=1.9.1-1 \
-    gnupg=2.2.12-1ubuntu3 \
+    gnupg=2.2.12-1ubuntu3 \ 
     sudo=1.8.27-1ubuntu4.1 \
     docker.io=19.03.2-0ubuntu1 \
     openssh-server=1:8.0p1-6build1 \
@@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libglib2.0-0=2.62.1-1 \
     && rm /etc/crictl.yaml
 # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
-RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
+RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \    
     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
     apt-key add - < Release.key && apt-get update && \
     apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
@@ -26,7 +26,7 @@ RUN apt-get install -y --no-install-recommends podman=1.8.2~144
 RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml
 # enable docker which is default
 RUN systemctl enable docker
-# making SSH work for docker container
+# making SSH work for docker container 
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
 RUN mkdir /var/run/sshd
 RUN echo 'root:root' |chpasswd
@@ -34,7 +34,7 @@ RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_confi
 RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
 EXPOSE 22
 # create docker user for minikube ssh. to match VM using "docker" as username
-RUN adduser --ingroup docker --disabled-password --gecos '' docker
+RUN adduser --ingroup docker --disabled-password --gecos '' docker 
 RUN adduser docker sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER docker

From 2ed3df3f6c4bc64e0392a4c672d3194188250bfb Mon Sep 17 00:00:00 2001
From: Mili <rdbaez@pichincha.com>
Date: Wed, 10 Jun 2020 00:02:48 -0500
Subject: [PATCH 21/21] remove version for non main packages

---
 .../kic-base-experimental/kicbase.Dockerfile  | 38 +++++--------------
 1 file changed, 9 insertions(+), 29 deletions(-)

diff --git a/hack/images/kic-base-experimental/kicbase.Dockerfile b/hack/images/kic-base-experimental/kicbase.Dockerfile
index bc20cae78401..84cc91ab330e 100644
--- a/hack/images/kic-base-experimental/kicbase.Dockerfile
+++ b/hack/images/kic-base-experimental/kicbase.Dockerfile
@@ -19,30 +19,10 @@ COPY files/ /usr/local/bin/
 ARG COMMIT_SHA
 
 RUN echo "set ENV variables ..." \
- && export SYSTEMD_VERSION="245.4-4ubuntu3" \
-    CONNTRACK_VERSION="1:1.4.5-2" \
-    IPTABLES_VERSION="1.8.4-3ubuntu2" \
-    IPROUTE2_VERSION="5.5.0-1ubuntu1" \
-    ETHTOOL_VERSION="1:5.4-1" \
-    SOCAT_VERSION="1.7.3.3-2" \
-    UTIL_LINUX_VERSION="2.34-0.1ubuntu9" \
-    MOUNT_VERSION="2.34-0.1ubuntu9" \
-    EBTABLES_VERSION="2.0.11-3build1" \
-    UDEV_VERSION="245.4-4ubuntu3" \
-    KMOD_VERSION="27-1ubuntu2" \
-    GNUPG_VERSION="2.2.19-3ubuntu2" \
-    LIBGLIB2_VERSION="2.64.2-1~fakesync1" \
-    LIBSECCOMP2_VERSION="2.4.3-1ubuntu1" \
-    CA_CERTIFICATES_VERSION="20190110ubuntu1" \
-    CURL_VERSION="7.68.0-1ubuntu2" \
-    RSYNC_VERSION="3.1.3-8" \
-    CRIO_VERSION="1.17.3~2" \
-    PODMAN_VERSION="1.9.0~2" \
-    LZ4_VERSION="1.9.2-2" \
-    SUDO_VERSION="1.8.31-1ubuntu1" \
-    DOCKER_VERSION="19.03.8-0ubuntu1" \
-    DNSUTILS_VERSION="1:9.16.1-0ubuntu2" \
+ && export CRIO_VERSION="1.17.4~1" \
+    PODMAN_VERSION="1.9.3~1" \
     CNI_VERSION="v0.8.5" \
+    DOCKER_VERSION="19.03.8-0ubuntu1" \
     CRICTL_VERSION="v1.17.0" \
     CONTAINERD_VERSION="1.3.3-61-g60bc1282" \
  && echo "Ensuring scripts are executable ..." \
@@ -50,16 +30,16 @@ RUN echo "set ENV variables ..." \
  && echo "Installing Packages ..." \
     && DEBIAN_FRONTEND=noninteractive apt-get update && \
       apt-get install -y --no-install-recommends \
-      systemd=${SYSTEMD_VERSION} conntrack=${CONNTRACK_VERSION} iptables=${IPTABLES_VERSION} iproute2=${IPROUTE2_VERSION} ethtool=${ETHTOOL_VERSION} \
-      socat=${SOCAT_VERSION} util-linux=${UTIL_LINUX_VERSION} mount=${MOUNT_VERSION} ebtables=${EBTABLES_VERSION} udev=${UDEV_VERSION} kmod=${KMOD_VERSION} \
-      gnupg=${GNUPG_VERSION} libglib2.0-0=${LIBGLIB2_VERSION} libseccomp2=${LIBSECCOMP2_VERSION} ca-certificates=${CA_CERTIFICATES_VERSION} \
-      curl=${CURL_VERSION} rsync=${RSYNC_VERSION} \
+      systemd conntrack iptables iproute2 ethtool \
+      socat util-linux mount ebtables udev kmod \
+      gnupg libglib2.0-0 libseccomp2 ca-certificates \
+      curl rsync \
       && rm -rf /var/lib/apt/lists/* \
     && sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \
         curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
         apt-key add - < Release.key && apt-get update && \
-        apt-get install -y --no-install-recommends cri-o-1.17=${CRIO_VERSION} podman=${PODMAN_VERSION} lz4=${LZ4_VERSION} sudo=${SUDO_VERSION} \
-        docker.io=${DOCKER_VERSION} dnsutils=${DNSUTILS_VERSION} \
+        apt-get install -y --no-install-recommends cri-o-1.17=${CRIO_VERSION} podman=${PODMAN_VERSION} lz4 sudo \
+        docker.io=${DOCKER_VERSION} dnsutils \
     && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
     && rm -f /lib/systemd/system/multi-user.target.wants/* \
     && rm -f /etc/systemd/system/*.wants/* \