diff --git a/.circleci/config.yml b/.circleci/config.yml index 77dc08c..23306ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -194,9 +194,11 @@ jobs: exit 1 fi + pkg-kubelet: + <<: *linuxkit_pkg_build pkg-cri-containerd: <<: *linuxkit_pkg_build - pkg-kubelet: + pkg-kube-e2e-test: <<: *linuxkit_pkg_build pkg-kubernetes-docker-image-cache-common: @@ -272,8 +274,9 @@ jobs: mkdir -p ~/.docker/trust/private cp .circleci/content-trust.key ~/.docker/trust/private/b056f84873aa0be205dfe826afa6e7458120c9569dd19a2a84154498fb1165d5.key - linuxkit pkg push --nobuild pkg/cri-containerd linuxkit pkg push --nobuild pkg/kubelet + linuxkit pkg push --nobuild pkg/cri-containerd + linuxkit pkg push --nobuild pkg/kube-e2e-test linuxkit pkg push --nobuild pkg/kubernetes-docker-image-cache-common linuxkit pkg push --nobuild pkg/kubernetes-docker-image-cache-control-plane @@ -292,6 +295,9 @@ workflows: - pkg-cri-containerd: requires: - dependencies + - pkg-kube-e2e-test: + requires: + - dependencies - pkg-kubernetes-docker-image-cache-common: requires: - dependencies @@ -330,6 +336,7 @@ workflows: - lint - pkg-kubelet - pkg-cri-containerd + - pkg-kube-e2e-test - pkg-kubernetes-docker-image-cache-common - pkg-kubernetes-docker-image-cache-control-plane - image-docker-weave diff --git a/pkg/kube-e2e-test/Dockerfile b/pkg/kube-e2e-test/Dockerfile new file mode 100644 index 0000000..6092b17 --- /dev/null +++ b/pkg/kube-e2e-test/Dockerfile @@ -0,0 +1,66 @@ +FROM linuxkit/alpine:07f7d136e427dc68154cd5edbb2b9576f9ac5213 AS build + +# When changing kubernetes_version remember to also update: +# - scripts/mk-image-cache-lst and run `make refresh-image-caches` from top-level +# - pkg/kubelet/Dockerfile +ENV kubernetes_version v1.9.0 + +RUN apk add -U --no-cache \ + bash \ + coreutils \ + curl \ + findutils \ + git \ + go \ + grep \ + libc-dev \ + linux-headers \ + make \ + rsync \ + && true + +ENV GOPATH=/go PATH=$PATH:/go/bin + +ENV KUBERNETES_URL https://github.com/kubernetes/kubernetes.git +#ENV KUBERNETES_BRANCH pull/NNN/head +ENV KUBERNETES_COMMIT ${kubernetes_version} +RUN mkdir -p $GOPATH/src/github.com/kubernetes && \ + cd $GOPATH/src/github.com/kubernetes && \ + git clone $KUBERNETES_URL kubernetes +WORKDIR $GOPATH/src/github.com/kubernetes/kubernetes +RUN set -e; \ + if [ -n "$KUBERNETES_BRANCH" ] ; then \ + git fetch origin "$KUBERNETES_BRANCH"; \ + fi; \ + git checkout -q $KUBERNETES_COMMIT + +RUN make WHAT="cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo" + +RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +RUN apk add --no-cache --initdb -p /out \ + alpine-baselayout \ + bash \ + ca-certificates \ + curl \ + musl \ + socat \ + util-linux \ + && true + +RUN cp _output/bin/kubectl /out/usr/bin/kubectl +RUN cp _output/bin/ginkgo /out/usr/bin/ginkgo +RUN cp _output/bin/e2e.test /out/usr/bin/e2e.test + +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + +RUN rmdir /out/var/run && ln -nfs /run /out/var/run + +ADD in-cluster-config.yaml /out/etc/in-cluster-config.yaml +ADD e2e.sh /out/usr/bin/e2e.sh + +FROM scratch +WORKDIR / +ENV KUBECONFIG /etc/in-cluster-config.yaml +ENTRYPOINT ["/usr/bin/e2e.sh"] +COPY --from=build /out / diff --git a/pkg/kube-e2e-test/README.md b/pkg/kube-e2e-test/README.md new file mode 100644 index 0000000..f68c858 --- /dev/null +++ b/pkg/kube-e2e-test/README.md @@ -0,0 +1,34 @@ +# Kubernetes end-to-end test suite (e2e) + +In this package provides Kubernetes e2e test suite, it is simple and convient to use within LinuxKit CI. + +> There are other ways to run e2e tests, however there are downsides to each of those approaches +> and maintaining this package is not seen as a major downside at present (however, things may +> change in the future). +> For example, [kubetest][] attempts to setup cluster by itself, and also has various modes which make +> it more complex to use, additionally it download release tarballs each time you run it. +> And [sonobuoy][] appears to have dependencies on a service, which is not desirable for LinuxKit use-case. + +[kubetest]: https://github.com/kubernetes/test-infra/tree/master/kubetest +[sonobuoy]: https://github.com/heptio/sonobuoy + +## Building the package + +``` +linuxkit pkg build pkg/kube-e2e-test +``` + +This will result in `linuxkit/kube-e2e-test:` image that can be use with `scripts/run-e2e-test.sh`. + +## Running as Job on any cluster + +Start the test suite: +``` +scripts/run-e2e-test.sh +``` + +After the script exits, you can find `e2e.log` in the current directory. + +Please consult [Kubernetes documentation for more information][e2e-docs]. + +[e2e-docs]: https://github.com/kubernetes/community/blob/master/contributors/devel/e2e-tests.md diff --git a/pkg/kube-e2e-test/build.yml b/pkg/kube-e2e-test/build.yml new file mode 100644 index 0000000..be040da --- /dev/null +++ b/pkg/kube-e2e-test/build.yml @@ -0,0 +1,5 @@ +org: linuxkit +image: kube-e2e-test +network: true +arches: + - amd64 diff --git a/pkg/kube-e2e-test/e2e.sh b/pkg/kube-e2e-test/e2e.sh new file mode 100755 index 0000000..192090d --- /dev/null +++ b/pkg/kube-e2e-test/e2e.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -eu + +# cleanup resources created by previous runs +kubectl get namespaces \ + --output="jsonpath={range .items[?(.status.phase == \"Active\")]}{.metadata.name}{\"\n\"}{end}" \ + | grep '^e2e-.*' \ + | xargs -r kubectl delete namespaces + +# execute the test suite +exec /usr/bin/ginkgo \ + -progress \ + -nodes="${E2E_PARALLEL}" \ + -flakeAttempts="${E2E_FLAKE_ATTEMPTS}" \ + -skip="${E2E_SKIP}" \ + -focus="${E2E_FOCUS}" \ + /usr/bin/e2e.test -- \ + -provider="${E2E_CLOUD_PROVIDER}" \ + -host="https://kubernetes.default.svc:443" \ + -kubeconfig="${KUBECONFIG}" \ + -test.short \ + -test.v diff --git a/pkg/kube-e2e-test/in-cluster-config.yaml b/pkg/kube-e2e-test/in-cluster-config.yaml new file mode 100644 index 0000000..3e0fde4 --- /dev/null +++ b/pkg/kube-e2e-test/in-cluster-config.yaml @@ -0,0 +1,19 @@ +# most tests generate `kubeconfig` on the fly, but some depend on `-host`, +# so we must provide `-kubeconfig` and `-host` flags until that changes +apiVersion: v1 +kind: Config +clusters: + - name: kube-e2e-test-cluster + cluster: + server: https://kubernetes.default.svc:443 + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt +users: + - name: kube-e2e-test-user + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +contexts: + - name: kube-e2e-test-context + context: + cluster: kube-e2e-test-cluster + user: kube-e2e-test-user +current-context: kube-e2e-test-context diff --git a/pkg/kubelet/Dockerfile b/pkg/kubelet/Dockerfile index 2e95d7a..02298f2 100644 --- a/pkg/kubelet/Dockerfile +++ b/pkg/kubelet/Dockerfile @@ -2,6 +2,7 @@ FROM linuxkit/alpine:07f7d136e427dc68154cd5edbb2b9576f9ac5213 AS build # When changing kubernetes_version remember to also update: # - scripts/mk-image-cache-lst and run `make refresh-image-caches` from top-level +# - pkg/e2e-test/Dockerfile ENV kubernetes_version v1.9.0 ENV cni_version v0.6.0 ENV critools_version v1.0.0-alpha.0 diff --git a/scripts/mk-image-cache-lst b/scripts/mk-image-cache-lst index f8a70fc..1424b44 100755 --- a/scripts/mk-image-cache-lst +++ b/scripts/mk-image-cache-lst @@ -2,6 +2,7 @@ repo=gcr.io/google_containers # When changing kubernetes_version remember to also update: # - pkg/kubelet/Dockerfile +# - pkg/e2e-test/Dockerfile kubernetes_version=v1.9.0 kube_dns_version=1.14.7 pause_version=3.0 diff --git a/scripts/run-e2e-test.sh b/scripts/run-e2e-test.sh new file mode 100755 index 0000000..4c323ad --- /dev/null +++ b/scripts/run-e2e-test.sh @@ -0,0 +1,146 @@ +#!/bin/sh + +set -eu + +E2E_CLOUD_PROVIDER="local" +E2E_PARALLEL="4" +E2E_FLAKE_ATTEMPTS="2" + +E2E_FOCUS='' +E2E_SKIP='\\[Slow\\]|\\[Serial\\]|\\[Disruptive\\]|\\[Flaky\\]|\\[Feature:.+\\]|\\[HPA\\]|Dashboard|Services.*functioning.*NodePort|.*NFS.*|.*Volume.*|\\[sig-storage\\]|.*StatefulSet.*|should\\ proxy\\ to\\ cadvisor\\ using\\ proxy\\ subresource' + +## To see this fail quickly try: +#E2E_FOCUS='should\ handle\ in-cluster\ config' +#E2E_SKIP='' +## To see this pass quickly try: +#E2E_FOCUS='Simple\ pod' +#E2E_SKIP='should\ handle\ in-cluster\ config' + +namespace="kube-system" +name="kube-e2e-test" + +cleanup() { + ## we only cleanup control resources, the resources created by the + ## test suite itself are cleaned up by pkg/kube-e2e-test/e2e.sh, as + ## those can be useful for investigation of why something fails + kubectl delete --namespace "${namespace}" \ + "Job/${name}" \ + "ServiceAccount/${name}" \ + "ClusterRole/${name}" \ + "ClusterRoleBinding/${name}" +} + +get_pods() { + kubectl get pods --namespace "${namespace}" --selector job-name="${name}" "$@" +} + +one_pod_running() { + test "$(get_pods --output "jsonpath={range .items[?(.status.phase == \"Running\")]}{.metadata.name}{\"\n\"}{end}" | wc -l)" -eq 1 +} + +all_pods_absent() { + test "$(get_pods --output "jsonpath={range .items[*]}{.metadata.name}{\"\n\"}{end}" | wc -l)" -eq 0 +} + +get_logs() { + kubectl logs --namespace "${namespace}" "Job/${name}" "$@" || true +} + +echo "$0: deleting any old resources left over from the previous run..." +cleanup 2> /dev/null || true +echo "$0: waiting until old pods are absent..." +until all_pods_absent ; do sleep 0.5 ; done + +echo "$0: creating resources to run the suite..." +kubectl create --namespace "${namespace}" --filename - < e2e.log +echo "$0: log saved in ${PWD}/e2e.log, cleaning up the resources..." +cleanup 2> /dev/null || true +if grep -q '^Test Suite Passed$' e2e.log ; then + echo "$0: test suite passed, exiting" + exit 0 +else + echo "$0: test suite failed, exiting" + exit 1 +fi diff --git a/yml/kube.yml b/yml/kube.yml index b053eb8..1164407 100644 --- a/yml/kube.yml +++ b/yml/kube.yml @@ -36,7 +36,7 @@ services: - name: sshd image: linuxkit/sshd:ac5e8364e2e9aa8717a3295c51eb60b8c57373d5 - name: kubelet - image: linuxkit/kubelet:03205e3daddfeedeb64d4e023b42c225c8e00945 + image: linuxkit/kubelet:2a685aa89b2bd9023b60d1ec64baec8727449c5a files: - path: etc/linuxkit.yml metadata: yaml