Add e2e package

.circleci/config.yml

@@ -194,9 +194,11 @@ jobs:
                 exit 1
             fi
 
+  pkg-kubelet:
+    <<: *linuxkit_pkg_build
   pkg-cri-containerd:
     <<: *linuxkit_pkg_build
-  pkg-kubelet:
+  pkg-kube-e2e-test:
     <<: *linuxkit_pkg_build
 
   pkg-kubernetes-docker-image-cache-common:
@@ -272,8 +274,9 @@ jobs:
             mkdir -p ~/.docker/trust/private
             cp .circleci/content-trust.key ~/.docker/trust/private/b056f84873aa0be205dfe826afa6e7458120c9569dd19a2a84154498fb1165d5.key
 
-            linuxkit pkg push --nobuild pkg/cri-containerd
             linuxkit pkg push --nobuild pkg/kubelet
+            linuxkit pkg push --nobuild pkg/cri-containerd
+            linuxkit pkg push --nobuild pkg/kube-e2e-test
             linuxkit pkg push --nobuild pkg/kubernetes-docker-image-cache-common
             linuxkit pkg push --nobuild pkg/kubernetes-docker-image-cache-control-plane
 
@@ -292,6 +295,9 @@ workflows:
       - pkg-cri-containerd:
           requires:
             - dependencies
+      - pkg-kube-e2e-test:
+          requires:
+            - dependencies
       - pkg-kubernetes-docker-image-cache-common:
           requires:
             - dependencies
@@ -330,6 +336,7 @@ workflows:
             - lint
             - pkg-kubelet
             - pkg-cri-containerd
+            - pkg-kube-e2e-test
             - pkg-kubernetes-docker-image-cache-common
             - pkg-kubernetes-docker-image-cache-control-plane
             - image-docker-weave

pkg/kube-e2e-test/Dockerfile

@@ -0,0 +1,66 @@
+FROM linuxkit/alpine:07f7d136e427dc68154cd5edbb2b9576f9ac5213 AS build
+
+# When changing kubernetes_version remember to also update:
+# - scripts/mk-image-cache-lst and run `make refresh-image-caches` from top-level
+# - pkg/kubelet/Dockerfile
+ENV kubernetes_version v1.9.0
+
+RUN apk add -U --no-cache \
+  bash \
+  coreutils \
+  curl \
+  findutils \
+  git \
+  go \
+  grep \
+  libc-dev \
+  linux-headers \
+  make \
+  rsync \
+  && true
+
+ENV GOPATH=/go PATH=$PATH:/go/bin
+
+ENV KUBERNETES_URL https://github.com/kubernetes/kubernetes.git
+#ENV KUBERNETES_BRANCH pull/NNN/head
+ENV KUBERNETES_COMMIT ${kubernetes_version}
+RUN mkdir -p $GOPATH/src/github.com/kubernetes && \
+    cd $GOPATH/src/github.com/kubernetes && \
+    git clone $KUBERNETES_URL kubernetes
+WORKDIR $GOPATH/src/github.com/kubernetes/kubernetes
+RUN set -e; \
+    if [ -n "$KUBERNETES_BRANCH" ] ; then \
+        git fetch origin "$KUBERNETES_BRANCH"; \
+    fi; \
+    git checkout -q $KUBERNETES_COMMIT
+
+RUN make WHAT="cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo"
+
+RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/
+RUN apk add --no-cache --initdb -p /out \
+    alpine-baselayout \
+    bash \
+    ca-certificates \
+    curl \
+    musl \
+    socat \
+    util-linux \
+    && true
+
+RUN cp _output/bin/kubectl /out/usr/bin/kubectl
+RUN cp _output/bin/ginkgo /out/usr/bin/ginkgo
+RUN cp _output/bin/e2e.test /out/usr/bin/e2e.test
+
+# Remove apk residuals. We have a read-only rootfs, so apk is of no use.
+RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache
+
+RUN rmdir /out/var/run && ln -nfs /run /out/var/run
+
+ADD in-cluster-config.yaml /out/etc/in-cluster-config.yaml
+ADD e2e.sh /out/usr/bin/e2e.sh
+
+FROM scratch
+WORKDIR /
+ENV KUBECONFIG /etc/in-cluster-config.yaml
+ENTRYPOINT ["/usr/bin/e2e.sh"]
+COPY --from=build /out /

pkg/kube-e2e-test/README.md

@@ -0,0 +1,34 @@
+# Kubernetes end-to-end test suite (e2e)
+
+In this package provides Kubernetes e2e test suite, it is simple and convient to use within LinuxKit CI.
+
+> There are other ways to run e2e tests, however there are downsides to each of those approaches
+> and maintaining this package is not seen as a major downside at present (however, things may
+> change in the future).
+> For example, [kubetest][] attempts to setup cluster by itself, and also has various modes which make
+> it more complex to use, additionally it download release tarballs each time you run it.
+> And [sonobuoy][] appears to have dependencies on a service, which is not desirable for LinuxKit use-case.
+
+[kubetest]: https://github.com/kubernetes/test-infra/tree/master/kubetest
+[sonobuoy]: https://github.com/heptio/sonobuoy
+
+## Building the package
+
+```
+linuxkit pkg build pkg/kube-e2e-test
+```
+
+This will result in `linuxkit/kube-e2e-test:<hash>` image that can be use with `scripts/run-e2e-test.sh`.
+
+## Running as Job on any cluster
+
+Start the test suite:
+```
+scripts/run-e2e-test.sh
+```
+
+After the script exits, you can find `e2e.log` in the current directory.
+
+Please consult [Kubernetes documentation for more information][e2e-docs].
+
+[e2e-docs]: https://github.com/kubernetes/community/blob/master/contributors/devel/e2e-tests.md

pkg/kube-e2e-test/build.yml

@@ -0,0 +1,5 @@
+org: linuxkit
+image: kube-e2e-test
+network: true
+arches:
+  - amd64

pkg/kube-e2e-test/e2e.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -eu
+
+# cleanup resources created by previous runs
+kubectl get namespaces \
+  --output="jsonpath={range .items[?(.status.phase == \"Active\")]}{.metadata.name}{\"\n\"}{end}" \
+  | grep '^e2e-.*' \
+  | xargs -r kubectl delete namespaces
+
+# execute the test suite
+exec /usr/bin/ginkgo \
+  -progress \
+  -nodes="${E2E_PARALLEL}" \
+  -flakeAttempts="${E2E_FLAKE_ATTEMPTS}" \
+  -skip="${E2E_SKIP}" \
+  -focus="${E2E_FOCUS}" \
+  /usr/bin/e2e.test -- \
+    -provider="${E2E_CLOUD_PROVIDER}" \
+    -host="https://kubernetes.default.svc:443" \
+    -kubeconfig="${KUBECONFIG}" \
+    -test.short \
+    -test.v

pkg/kube-e2e-test/in-cluster-config.yaml

@@ -0,0 +1,19 @@
+# most tests generate `kubeconfig` on the fly, but some depend on `-host`,
+# so we must provide `-kubeconfig` and `-host` flags until that changes
+apiVersion: v1
+kind: Config
+clusters:
+  - name: kube-e2e-test-cluster
+    cluster:
+      server: https://kubernetes.default.svc:443
+      certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+users:
+  - name: kube-e2e-test-user
+    user:
+      tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+contexts:
+  - name: kube-e2e-test-context
+    context:
+      cluster: kube-e2e-test-cluster
+      user: kube-e2e-test-user
+current-context: kube-e2e-test-context

pkg/kubelet/Dockerfile

@@ -2,6 +2,7 @@ FROM linuxkit/alpine:07f7d136e427dc68154cd5edbb2b9576f9ac5213 AS build
 
 # When changing kubernetes_version remember to also update:
 # - scripts/mk-image-cache-lst and run `make refresh-image-caches` from top-level
+# - pkg/e2e-test/Dockerfile
 ENV kubernetes_version v1.9.0
 ENV cni_version        v0.6.0
 ENV critools_version   v1.0.0-alpha.0

scripts/mk-image-cache-lst

@@ -2,6 +2,7 @@
 repo=gcr.io/google_containers
 # When changing kubernetes_version remember to also update:
 # - pkg/kubelet/Dockerfile
+# - pkg/e2e-test/Dockerfile
 kubernetes_version=v1.9.0
 kube_dns_version=1.14.7
 pause_version=3.0

scripts/run-e2e-test.sh

@@ -0,0 +1,146 @@
+#!/bin/sh
+
+set -eu
+
+E2E_CLOUD_PROVIDER="local"
+E2E_PARALLEL="4"
+E2E_FLAKE_ATTEMPTS="2"
+
+E2E_FOCUS=''
+E2E_SKIP='\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[HPA\\]|Dashboard|Services.*functioning.*NodePort|.*NFS.*|.*Volume.*|\\[sig-storage\\]|.*StatefulSet.*|should\\ proxy\\ to\\ cadvisor\\ using\\ proxy\\ subresource'
+
+## To see this fail quickly try:
+#E2E_FOCUS='should\ handle\ in-cluster\ config'
+#E2E_SKIP=''
+## To see this pass quickly try:
+#E2E_FOCUS='Simple\ pod'
+#E2E_SKIP='should\ handle\ in-cluster\ config'
+
+namespace="kube-system"
+name="kube-e2e-test"
+
+cleanup() {
+  ## we only cleanup control resources, the resources created by the
+  ## test suite itself are cleaned up by pkg/kube-e2e-test/e2e.sh, as
+  ## those can be useful for investigation of why something fails
+  kubectl delete --namespace "${namespace}" \
+    "Job/${name}" \
+    "ServiceAccount/${name}" \
+    "ClusterRole/${name}" \
+    "ClusterRoleBinding/${name}"
+}
+
+get_pods() {
+  kubectl get pods --namespace "${namespace}" --selector job-name="${name}" "$@"
+}
+
+one_pod_running() {
+  test "$(get_pods --output "jsonpath={range .items[?(.status.phase == \"Running\")]}{.metadata.name}{\"\n\"}{end}" | wc -l)" -eq 1
+}
+
+all_pods_absent() {
+  test "$(get_pods --output "jsonpath={range .items[*]}{.metadata.name}{\"\n\"}{end}" | wc -l)" -eq 0
+}
+
+get_logs() {
+  kubectl logs --namespace "${namespace}" "Job/${name}" "$@" || true
+}
+
+echo "$0: deleting any old resources left over from the previous run..."
+cleanup 2> /dev/null || true
+echo "$0: waiting until old pods are absent..."
+until all_pods_absent ; do sleep 0.5 ; done
+
+echo "$0: creating resources to run the suite..."
+kubectl create --namespace "${namespace}" --filename - <<EOF
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: "${name}"
+      namespace: "${namespace}"
+      labels:
+        name: "${name}"
+  - apiVersion: rbac.authorization.k8s.io/v1beta1
+    kind: ClusterRole
+    metadata:
+      name: "${name}"
+      labels:
+        name: "${name}"
+    rules:
+      - apiGroups: [ '*' ]
+        resources: [ '*' ]
+        verbs: [ '*' ]
+      - nonResourceURLs: [ '*' ]
+        verbs: [ '*' ]
+  - apiVersion: rbac.authorization.k8s.io/v1beta1
+    kind: ClusterRoleBinding
+    metadata:
+      name: "${name}"
+      namespace: "${namespace}"
+      labels:
+        name: "${name}"
+    roleRef:
+      kind: ClusterRole
+      name: cluster-admin
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: "${name}"
+        namespace: "${namespace}"
+  - apiVersion: batch/v1
+    kind: Job
+    metadata:
+      name: "${name}"
+      namespace: "${namespace}"
+      labels:
+        name: "${name}"
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            name: "${name}"
+        spec:
+          serviceAccount: "${name}"
+          tolerations:
+            - effect: NoSchedule
+              operator: Exists
+          restartPolicy: Never
+          containers:
+            - name: "${name}"
+              image: "linuxkit/${name}:260c4467ab0a752b5369e73d4f2da13eb2da7bbf"
+              imagePullPolicy: IfNotPresent
+              env:
+              - name: E2E_CLOUD_PROVIDER
+                value: "${E2E_CLOUD_PROVIDER}"
+              - name: E2E_PARALLEL
+                value: "${E2E_PARALLEL}"
+              - name: E2E_FLAKE_ATTEMPTS
+                value: "${E2E_FLAKE_ATTEMPTS}"
+              - name: E2E_FOCUS
+                value: "${E2E_FOCUS}"
+              - name: E2E_SKIP
+                value: "${E2E_SKIP}"
+EOF
+
+echo "$0: waiting until 1 pod is running..."
+until one_pod_running ; do sleep 0.5 ; done
+get_logs --follow
+echo "$0: kubectl logs terminated, waiting until no pods are running..."
+## sometimes kubectl logs terminates abruptly, and even in theory
+## we cannot trust it to indicated overall status, so we wait, and
+## dump the logs once again after the pod has exited
+until all_pods_absent ; do sleep 1.5 ; done
+get_logs > e2e.log
+echo "$0: log saved in ${PWD}/e2e.log, cleaning up the resources..."
+cleanup 2> /dev/null || true
+if grep -q '^Test Suite Passed$' e2e.log ; then
+  echo "$0: test suite passed, exiting"
+  exit 0
+else
+  echo "$0: test suite failed, exiting"
+  exit 1
+fi

yml/kube.yml

@@ -36,7 +36,7 @@ services:
   - name: sshd
     image: linuxkit/sshd:ac5e8364e2e9aa8717a3295c51eb60b8c57373d5
   - name: kubelet
-    image: linuxkit/kubelet:03205e3daddfeedeb64d4e023b42c225c8e00945
+    image: linuxkit/kubelet:2a685aa89b2bd9023b60d1ec64baec8727449c5a
 files:
   - path: etc/linuxkit.yml
     metadata: yaml