From 57516514e2131c421ea313a48d912a737ac05b39 Mon Sep 17 00:00:00 2001 From: Daz Wilkin Date: Thu, 18 Feb 2021 10:33:26 -0800 Subject: [PATCH] Simplify matrix; Use all versions 1.16+ for all distros (#244) * Simplify matrix; Use all versions 1.16+ for all distros * Remove non-existent webhook test * Remove `microk8s.enable helm3` appears redundant and fails * Link Kubernetes and crictl for K3s|MicroK8s * Per: https://github.com/deislabs/akri/pull/206#issuecomment-778459680 * Typo * fix matrix and crictl path * Consistency and double-quote to expand `${PWD}` * Temporarily (!) block K3s 1.16 due to hard-coded device plugs (https://github.com/k3s-io/k3s/issues/1390) * Test mapping K3s 1.16 device plugins to default K8s location * Create kubectl path in expected location if not present * Consistent w/ MicroK8s step * Wait until cluster has stabilized before wrangling device-plugins * Initial documentation for `run-test-cases.yml` * Correct indentation Co-authored-by: bfjelds --- .github/workflows/run-test-cases.yml | 525 ++++++++++++++------------- docs/workflows-run-test-cases.md | 120 ++++++ 2 files changed, 390 insertions(+), 255 deletions(-) create mode 100644 docs/workflows-run-test-cases.md diff --git a/.github/workflows/run-test-cases.yml b/.github/workflows/run-test-cases.yml index 9b58979fd..a7eebccca 100644 --- a/.github/workflows/run-test-cases.yml +++ b/.github/workflows/run-test-cases.yml @@ -4,81 +4,81 @@ on: workflow_dispatch: inputs: pull_request: - branches: [ main ] + branches: [main] paths: - - test/run-end-to-end.py - - test/run-conservation-of-broker-pod.py - - test/run-helm-install-delete.py - - test/shared_test_code.py - - .github/workflows/run-test-cases.yml - - build/containers/Dockerfile.agent - - build/containers/Dockerfile.controller - - deployment/helm/** - - agent/** - - controller/** - - shared/** - - version.txt - - build/akri-containers.mk - - Makefile + - test/run-end-to-end.py + - test/run-conservation-of-broker-pod.py + - test/run-helm-install-delete.py + - test/shared_test_code.py + - .github/workflows/run-test-cases.yml + - build/containers/Dockerfile.agent + - build/containers/Dockerfile.controller + - deployment/helm/** + - agent/** + - controller/** + - shared/** + - version.txt + - build/akri-containers.mk + - Makefile push: - branches: [ main ] + branches: [main] paths: - - test/run-end-to-end.py - - test/run-conservation-of-broker-pod.py - - test/run-helm-install-delete.py - - test/shared_test_code.py - - .github/workflows/run-test-cases.yml - - build/containers/Dockerfile.agent - - build/containers/Dockerfile.controller - - deployment/helm/** - - agent/** - - controller/** - - shared/** - - version.txt - - build/akri-containers.mk - - Makefile + - test/run-end-to-end.py + - test/run-conservation-of-broker-pod.py + - test/run-helm-install-delete.py + - test/shared_test_code.py + - .github/workflows/run-test-cases.yml + - build/containers/Dockerfile.agent + - build/containers/Dockerfile.controller + - deployment/helm/** + - agent/** + - controller/** + - shared/** + - version.txt + - build/akri-containers.mk + - Makefile release: types: - published - + jobs: build-containers: runs-on: ubuntu-18.04 timeout-minutes: 35 steps: - - name: Checkout the head commit of the branch - uses: actions/checkout@v2 - with: - persist-credentials: false + - name: Checkout the head commit of the branch + uses: actions/checkout@v2 + with: + persist-credentials: false - - name: Build local containers for PR tests - if: startsWith(github.event_name, 'pull_request') - env: - BUILD_AMD64: 1 - BUILD_ARM32: 0 - BUILD_ARM64: 0 - PREFIX: ghcr.io/deislabs/akri - LABEL_PREFIX: pr - run: | - make akri-build - make controller-build-amd64 - make agent-build-amd64 - docker save ${PREFIX}/agent:${LABEL_PREFIX}-amd64 > agent.tar - docker save ${PREFIX}/controller:${LABEL_PREFIX}-amd64 > controller.tar + - name: Build local containers for PR tests + if: startsWith(github.event_name, 'pull_request') + env: + BUILD_AMD64: 1 + BUILD_ARM32: 0 + BUILD_ARM64: 0 + PREFIX: ghcr.io/deislabs/akri + LABEL_PREFIX: pr + run: | + make akri-build + make controller-build-amd64 + make agent-build-amd64 + docker save ${PREFIX}/agent:${LABEL_PREFIX}-amd64 > agent.tar + docker save ${PREFIX}/controller:${LABEL_PREFIX}-amd64 > controller.tar - - name: Upload Agent container as artifact - if: startsWith(github.event_name, 'pull_request') - uses: actions/upload-artifact@v2 - with: - name: agent.tar - path: agent.tar - - name: Upload Controller container as artifact - if: startsWith(github.event_name, 'pull_request') - uses: actions/upload-artifact@v2 - with: - name: controller.tar - path: controller.tar + - name: Upload Agent container as artifact + if: startsWith(github.event_name, 'pull_request') + uses: actions/upload-artifact@v2 + with: + name: agent.tar + path: agent.tar + - name: Upload Controller container as artifact + if: startsWith(github.event_name, 'pull_request') + uses: actions/upload-artifact@v2 + with: + name: controller.tar + path: controller.tar test-cases: needs: build-containers @@ -88,211 +88,226 @@ jobs: strategy: fail-fast: false matrix: - kube-runtime: - - K3s-1.18 - - K3s-1.19 - - K3s-1.20 - - MicroK8s-1.18 - - MicroK8s-1.19 - - MicroK8s-1.20 - - Kubernetes-1.16 - - Kubernetes-1.17 - - Kubernetes-1.18 - - Kubernetes-1.19 - - Kubernetes-1.20 - test-case: - - end-to-end - include: - - kube-runtime: MicroK8s-1.18 - kube-version: 1.18/stable - - kube-runtime: MicroK8s-1.19 - kube-version: 1.19/stable - - kube-runtime: MicroK8s-1.20 - kube-version: 1.20/stable - - kube-runtime: K3s-1.18 - kube-version: v1.18.9+k3s1 - - kube-runtime: K3s-1.19 - kube-version: v1.19.4+k3s1 - - kube-runtime: K3s-1.20 - kube-version: v1.20.0+k3s2 - - kube-runtime: Kubernetes-1.16 - kube-version: 1.16.15-00 - - kube-runtime: Kubernetes-1.17 - kube-version: 1.17.14-00 - - kube-runtime: Kubernetes-1.18 - kube-version: 1.18.12-00 - - kube-runtime: Kubernetes-1.19 - kube-version: 1.19.4-00 - - kube-runtime: Kubernetes-1.20 - kube-version: 1.20.1-00 - - test-case: end-to-end - test-file: test/run-end-to-end.py + kube: + - runtime: MicroK8s-1.16 + version: 1.16/stable + crictl: v1.16.0 + - runtime: MicroK8s-1.17 + version: 1.17/stable + crictl: v1.16.0 + - runtime: MicroK8s-1.18 + version: 1.18/stable + crictl: v1.17.0 + - runtime: MicroK8s-1.19 + version: 1.19/stable + crictl: v1.17.0 + - runtime: MicroK8s-1.20 + version: 1.20/stable + crictl: v1.17.0 + - runtime: K3s-1.16 + version: v1.16.14+k3s1 + crictl: v1.16.0 + - runtime: K3s-1.17 + version: v1.17.17+k3s1 + crictl: v1.16.0 + - runtime: K3s-1.18 + version: v1.18.9+k3s1 + crictl: v1.17.0 + - runtime: K3s-1.19 + version: v1.19.4+k3s1 + crictl: v1.17.0 + - runtime: K3s-1.20 + version: v1.20.0+k3s2 + crictl: v1.17.0 + - runtime: Kubernetes-1.16 + version: 1.16.15-00 + crictl: UNUSED + - runtime: Kubernetes-1.17 + version: 1.17.14-00 + crictl: UNUSED + - runtime: Kubernetes-1.18 + version: 1.18.12-00 + crictl: UNUSED + - runtime: Kubernetes-1.19 + version: 1.19.4-00 + crictl: UNUSED + - runtime: Kubernetes-1.20 + version: 1.20.1-00 + crictl: UNUSED + test: + - case: end-to-end + file: test/run-end-to-end.py steps: - - name: Checkout the head commit of the branch - uses: actions/checkout@v2 - with: - persist-credentials: false + - name: Checkout the head commit of the branch + uses: actions/checkout@v2 + with: + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Python kubernetes dependency + run: | + python -m pip install --upgrade pip + pip install kubernetes - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Python kubernetes dependency - run: | - python -m pip install --upgrade pip - pip install kubernetes + - name: Download Agent container artifact + if: startsWith(github.event_name, 'pull_request') + uses: actions/download-artifact@v2 + with: + name: agent.tar + - name: Download Controller container artifact + if: startsWith(github.event_name, 'pull_request') + uses: actions/download-artifact@v2 + with: + name: controller.tar - - name: Download Agent container artifact - if: startsWith(github.event_name, 'pull_request') - uses: actions/download-artifact@v2 - with: - name: agent.tar - - name: Download Controller container artifact - if: startsWith(github.event_name, 'pull_request') - uses: actions/download-artifact@v2 - with: - name: controller.tar + - if: startsWith(matrix.kube.runtime, 'K3s') + name: Install K3s + env: + INSTALL_K3S_VERSION: ${{ matrix.kube.version }} + run: | + sudo curl -sfL https://get.k3s.io | sh - + sudo addgroup k3s-admin + sudo adduser $USER k3s-admin + sudo usermod -a -G k3s-admin $USER + sudo chgrp k3s-admin /etc/rancher/k3s/k3s.yaml + sudo chmod g+r /etc/rancher/k3s/k3s.yaml + sudo chmod 666 /etc/rancher/k3s/* + mkdir -p ~/.kube/ && cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/${{ matrix.kube.crictl }}/crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz --output crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz + sudo tar zxvf crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz -C /usr/local/bin + rm -f crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz + echo "--set agent.host.crictl=/usr/local/bin/crictl --set agent.host.dockerShimSock=/run/k3s/containerd/containerd.sock" > /tmp/cri_args_to_test.txt + echo 'kubectl' > /tmp/runtime_cmd_to_test.txt + echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt + until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for k3s to become ready"; sleep 10; done + if [ "${{ matrix.kube.runtime }}" == "K3s-1.16" ]; then + mkdir -p /var/lib/kubelet + if [ -d /var/lib/kubelet/device-plugins ]; then + sudo rm -rf /var/lib/kubelet/device-plugins + fi + sudo ln -s /var/lib/rancher/k3s/agent/kubelet/device-plugins /var/lib/kubelet/device-plugins + fi - - if: startsWith(matrix.kube-runtime, 'K3s') - name: Install K3s - env: - INSTALL_K3S_VERSION: ${{ matrix.kube-version }} - run: | - sudo curl -sfL https://get.k3s.io | sh - - sudo addgroup k3s-admin - sudo adduser $USER k3s-admin - sudo usermod -a -G k3s-admin $USER - sudo chgrp k3s-admin /etc/rancher/k3s/k3s.yaml - sudo chmod g+r /etc/rancher/k3s/k3s.yaml - sudo chmod 666 /etc/rancher/k3s/* - mkdir -p ~/.kube/ && cp /etc/rancher/k3s/k3s.yaml ~/.kube/config - VERSION="v1.17.0" - curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz - sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C $(pwd) - rm -f crictl-$VERSION-linux-amd64.tar.gz - echo '--set agent.host.crictl=$(pwd)/crictl --set agent.host.dockerShimSock=/run/k3s/containerd/containerd.sock' > /tmp/cri_args_to_test.txt - echo 'kubectl' > /tmp/runtime_cmd_to_test.txt - echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt - until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for k3s to become ready"; sleep 10; done + - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'K3s')) + name: Import local agent and controller to K3s + run: | + sudo k3s ctr image import agent.tar + sudo k3s ctr image import controller.tar - - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube-runtime, 'K3s')) - name: Import local agent and controller to K3s - run: | - sudo k3s ctr image import agent.tar - sudo k3s ctr image import controller.tar + - if: startsWith(matrix.kube.runtime, 'Kubernetes') + name: Install Kubernetes + run: | + sudo apt-get update -y + sudo apt-get install -o Dpkg::Options::="--force-overwrite" -y --allow-downgrades kubelet=${{ matrix.kube.version }} kubeadm=${{ matrix.kube.version }} kubectl=${{ matrix.kube.version }} + kubectl version && echo "kubectl return code: $?" || echo "kubectl return code: $?" + kubeadm version && echo "kubeadm return code: $?" || echo "kubeadm return code: $?" + kubelet --version && echo "kubelet return code: $?" || echo "kubelet return code: $?" + sudo swapoff -a + sudo kubeadm init + sudo mkdir -p $HOME/.kube + sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config + sudo chown $(id -u):$(id -g) $HOME/.kube/config + kubectl taint nodes --all node-role.kubernetes.io/master- + echo '--set agent.host.crictl=/usr/bin/crictl --set agent.host.dockerShimSock=/var/run/dockershim.sock' > /tmp/cri_args_to_test.txt + echo 'kubectl' > /tmp/runtime_cmd_to_test.txt + echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt + until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for kubernetes to become ready"; sleep 10; done - - if: startsWith(matrix.kube-runtime, 'Kubernetes') - name: Install Kubernetes - run: | - sudo apt-get update -y - sudo apt-get install -o Dpkg::Options::="--force-overwrite" -y --allow-downgrades kubelet=${{ matrix.kube-version }} kubeadm=${{ matrix.kube-version }} kubectl=${{ matrix.kube-version }} - kubectl version && echo "kubectl return code: $?" || echo "kubectl return code: $?" - kubeadm version && echo "kubeadm return code: $?" || echo "kubeadm return code: $?" - kubelet --version && echo "kubelet return code: $?" || echo "kubelet return code: $?" - sudo swapoff -a - sudo kubeadm init - sudo mkdir -p $HOME/.kube - sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config - sudo chown $(id -u):$(id -g) $HOME/.kube/config - kubectl taint nodes --all node-role.kubernetes.io/master- - echo '--set agent.host.crictl=/usr/bin/crictl --set agent.host.dockerShimSock=/var/run/dockershim.sock' > /tmp/cri_args_to_test.txt - echo 'kubectl' > /tmp/runtime_cmd_to_test.txt - echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt - until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for kubernetes to become ready"; sleep 10; done + - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'Kubernetes')) + name: Import local agent and controller to Kubernetes + run: | + sudo docker load --input agent.tar + sudo docker load --input controller.tar - - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube-runtime, 'Kubernetes')) - name: Import local agent and controller to Kubernetes - run: | - sudo docker load --input agent.tar - sudo docker load --input controller.tar + - if: startsWith(matrix.kube.runtime, 'MicroK8s') + name: Install MicroK8s + run: | + sudo snap install microk8s --classic --channel=${{ matrix.kube.version }} + sudo microk8s status --wait-ready + sudo usermod -a -G microk8s $USER + sudo ls -la $HOME/.kube + echo sudo chown $(id -u):$(id -g) $HOME/.kube + sudo chown -f -R $USER $HOME/.kube --verbose + sudo sh -c "microk8s.kubectl config view --raw >~/.kube/config" + sudo cat ~/.kube/config + # sudo microk8s.enable helm3 + sudo microk8s.enable rbac + sudo microk8s.enable dns + sudo microk8s.status --wait-ready + curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/${{ matrix.kube.crictl }}/crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz --output crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz + sudo tar zxvf crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz -C /usr/local/bin + rm -f crictl-${{ matrix.kube.crictl }}-linux-amd64.tar.gz + echo '--set agent.host.crictl=/usr/local/bin/crictl --set agent.host.dockerShimSock=/var/snap/microk8s/common/run/containerd.sock' > /tmp/cri_args_to_test.txt + echo 'microk8s kubectl' > /tmp/runtime_cmd_to_test.txt + echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt - - if: startsWith(matrix.kube-runtime, 'MicroK8s') - name: Install MicroK8s - run: | - sudo snap install microk8s --classic --channel=${{ matrix.kube-version }} - sudo microk8s status --wait-ready - sudo usermod -a -G microk8s $USER - sudo ls -la $HOME/.kube - echo sudo chown $(id -u):$(id -g) $HOME/.kube - sudo chown -f -R $USER $HOME/.kube --verbose - sudo sh -c "microk8s.kubectl config view --raw >~/.kube/config" - sudo cat ~/.kube/config - sudo microk8s.enable helm3 - sudo microk8s.enable rbac - sudo microk8s.enable dns - sudo microk8s.status --wait-ready - VERSION="v1.17.0" - curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz - sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin - rm -f crictl-$VERSION-linux-amd64.tar.gz - echo '--set agent.host.crictl=/usr/local/bin/crictl --set agent.host.dockerShimSock=/var/snap/microk8s/common/run/containerd.sock' > /tmp/cri_args_to_test.txt - echo 'microk8s kubectl' > /tmp/runtime_cmd_to_test.txt - echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt + - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'MicroK8s')) + name: Import local agent and controller to MicroK8s + run: | + sudo microk8s.status --wait-ready + until sudo microk8s ctr images ls; do sleep 5s; echo "Try again"; done + sudo microk8s ctr images ls + sudo microk8s ctr --debug --timeout 10s images import agent.tar + sudo microk8s ctr --debug --timeout 10s images import controller.tar + sudo microk8s ctr images ls - - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube-runtime, 'MicroK8s')) - name: Import local agent and controller to MicroK8s - run: | - sudo microk8s.status --wait-ready - until sudo microk8s ctr images ls; do sleep 5s; echo "Try again"; done - sudo microk8s ctr images ls - sudo microk8s ctr --debug --timeout 10s images import agent.tar - sudo microk8s ctr --debug --timeout 10s images import controller.tar - sudo microk8s ctr images ls + - name: Add Akri Helm Chart + run: helm repo add akri-helm-charts https://deislabs.github.io/akri/ - - name: Add Akri Helm Chart - run: helm repo add akri-helm-charts https://deislabs.github.io/akri/ - - # For push and release, we need to wait for the Helm chart and - # associated containers to build. - - if: github.event_name == 'push' || github.event_name == 'release' - name: Set sleep duration before running script to 1500 - run: echo 1500 > /tmp/sleep_duration.txt + # For push and release, we need to wait for the Helm chart and + # associated containers to build. + - if: github.event_name == 'push' || github.event_name == 'release' + name: Set sleep duration before running script to 1500 + run: echo 1500 > /tmp/sleep_duration.txt - # For pull_request, use the locally built containers. - - if: startsWith(github.event_name, 'pull_request') - name: Tell Helm to use the 'local' labels for container images - run: | - git fetch origin main - git show origin/main:version.txt > /tmp/version_to_test.txt - echo '--set agent.image.pullPolicy=Never,agent.image.tag=pr-amd64,controller.image.pullPolicy=Never,controller.image.tag=pr-amd64' > /tmp/extra_helm_args.txt - # For non-PR (i.e. push, release, manual), version.txt is corresponds - # to an existing Helm chart. - - if: (!(startsWith(github.event_name, 'pull_request'))) - name: Use current version for push - run: cat version.txt > /tmp/version_to_test.txt + # For pull_request, use the locally built containers. + - if: startsWith(github.event_name, 'pull_request') + name: Tell Helm to use the 'local' labels for container images + run: | + git fetch origin main + git show origin/main:version.txt > /tmp/version_to_test.txt + echo '--set agent.image.pullPolicy=Never,agent.image.tag=pr-amd64,controller.image.pullPolicy=Never,controller.image.tag=pr-amd64' > /tmp/extra_helm_args.txt + # For non-PR (i.e. push, release, manual), version.txt is corresponds + # to an existing Helm chart. + - if: (!(startsWith(github.event_name, 'pull_request'))) + name: Use current version for push + run: cat version.txt > /tmp/version_to_test.txt - # For workflow_dispatch and pull_request, use the files in deployment/helm - # as basis for helm install ... this enables us to test any changes made to - # the helm chart files in a PR (where no helm chart is published) - - if: github.event_name != 'push' && github.event_name != 'release' - name: Tell Helm to use the files in deployment/helm to build chart - run: | - echo './deployment/helm' > /tmp/helm_chart_location.txt - # For push, use a specific version of the `akri-dev` charts that are built and - # published by the helm workflow. - - if: github.event_name == 'push' - name: Tell Helm to use the `akri-dev` published charts - run: | - echo "akri-helm-charts/akri-dev --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt - # For release, use a specific version of the `akri` charts that are built and - # published by the helm workflow. - - if: github.event_name == 'release' - name: Tell Helm to use the `akri` published charts - run: | - echo "akri-helm-charts/akri --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt + # For workflow_dispatch and pull_request, use the files in deployment/helm + # as basis for helm install ... this enables us to test any changes made to + # the helm chart files in a PR (where no helm chart is published) + - if: github.event_name != 'push' && github.event_name != 'release' + name: Tell Helm to use the files in deployment/helm to build chart + run: | + echo './deployment/helm' > /tmp/helm_chart_location.txt + # For push, use a specific version of the `akri-dev` charts that are built and + # published by the helm workflow. + - if: github.event_name == 'push' + name: Tell Helm to use the `akri-dev` published charts + run: | + echo "akri-helm-charts/akri-dev --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt + # For release, use a specific version of the `akri` charts that are built and + # published by the helm workflow. + - if: github.event_name == 'release' + name: Tell Helm to use the `akri` published charts + run: | + echo "akri-helm-charts/akri --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt - - name: Execute test script ${{ matrix.test-file }} - run: python ${{ matrix.test-file }} - - name: Upload Agent log as artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.kube-runtime }}-${{ matrix.test-case }}-agent-log - path: /tmp/agent_log.txt - - name: Upload controller log as artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.kube-runtime }}-${{ matrix.test-case }}-controller-log - path: /tmp/controller_log.txt + - name: Execute test script ${{ matrix.test.file }} + run: python ${{ matrix.test.file }} + - name: Upload Agent log as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-agent-log + path: /tmp/agent_log.txt + - name: Upload controller log as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-controller-log + path: /tmp/controller_log.txt diff --git a/docs/workflows-run-test-cases.md b/docs/workflows-run-test-cases.md new file mode 100644 index 000000000..c8707c0e9 --- /dev/null +++ b/docs/workflows-run-test-cases.md @@ -0,0 +1,120 @@ +# Test K3s, Kubernetes (Kubeadm) and MicroK8s + +File: `/.github/workflows/run-test-cases.yml` + +A GitHub workflow that: + ++ runs Python-based end-to-end [tests](#Tests); ++ through 5 different Kubernetes [versions](#Versions): 1.16, 1.17, 1.18, 1.19, 1.20; ++ on 3 different Kubernetes distros: [K3s](https://k3s.io), [Kubernetes (Kubeadm)](https://kubernetes.io/docs/reference/setup-tools/kubeadm/), [MicroK8s](https://microk8s.io). + +## Tests + +|Name|File|Documentation| +|----|----|-----------| +|end-to-end|`/test/run-end-to-end.py`|TBD| + +## Versions + +Distro K3s Version 1.16 creates [Device Plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/) sockets at `/var/lib/rancher/k3s/agent/kubelet/device-plugins` whereas Kubernetes expects these sockets to be created at `/var/lib/kubelet/device-plugins`. + +See K3s issue: [Compatibility with Device Plugins #1390](https://github.com/k3s-io/k3s/issues/1390) + +The fix for K3s 1.16 is to create a symbolic link from the K3s location to the Kubernetes-expected location. This is added as an exception to the workflow step for K3s: + +```bash +if [ "${{ matrix.kube.runtime }}" == "K3s-1.16" ]; then + mkdir -p /var/lib/kubelet + if [ -d /var/lib/kubelet/device-plugins ]; then + sudo rm -rf /var/lib/kubelet/device-plugins + fi + sudo ln -s /var/lib/rancher/k3s/agent/kubelet/device-plugins /var/lib/kubelet/device-plugins +fi +``` + +This issue was addressed in K3s version 1.17. + +## Jobs|Steps + +The workflow comprises two jobs (`build-containers` and `test-cases`). + +## `build-containers` + +`build-containers` builds container images for Akri 'controller' and 'agent' based upon the commit that triggers the workflow. Once build, these iamges are shared across the `test-cases` job, using GitHub Action [upload-artifact](https://github.com/actions/upload-artifact). + +## `test-cases` + +`test-cases` uses a GitHub [strategy](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstrategy) to run its steps across the different Kubernetes distros and versions summarized at the top of this document. + +New Kubernetes distro versions may be added to the job by adding entries to `jobs.test-cases.strategy.matrix.kube`. Each array entry must include: + +|Property|Description| +|--------|-----------| +|`runtime`|A unique identifier for this distro-version pair| +|`version`|A distro-specific unique identifier for the Kubernetes version| +|`crictl`|A reference to the release of [`cri-tools`](https://github.com/kubernetes-sigs/cri-tools) including `crictl` that will be used| + +Notes: + ++ `runtime` is used by subsequent steps as a way to determine the distro, e.g. `startsWith(matrix.kube.runtime, 'K3s')` ++ `version` is used by each distro to determine which binary, snap etc. to install. Refer to each distro's documentation to determine the value required ++ `crictl` is used by `K3s` and `MicroK8s` to determine which version of `crictl` (sic.) is must be installed. `Kubeadm` includes `crictl` and so this variable is left as `UNUSED` for this distro. + +### Distro installation and Akri container images + +Each distro has an installation step and a step to import the Akri `controller` and `agent` images created by the `build-containers` job. + +The installation steps are identified by: + +```YAML +if: startsWith(matrix.kube.runtime, ${DISTRO}) +``` + +The installation instructions map closely with the installation instructions provided for the distro. For `K3s` and `MicroK8s`, the step includes installation of `cri-tools` so that `crictl` is available. + +The container image import steps are identified by: + +```YAML +if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, ${DISTRO})) +``` + +### Helm and state + +In order to pass state between the workflow and the Python end-to-end test scripts, temporary (`/tmp`) files are used: + +|File|Description| +|----|-----------| +|`agent_log.txt`|Filename used by workflow to persist the Agent's log| +|`controller_log.txt`|Filename used by workflow to persist the Controller's log| +|`cri_args_to_test.txt`|`crictcl` configuration that is passed to `helm install` command| +|`extra_helm_args.txt`|Additionl configuration that is passed to `helm install` command| +|`helm_chart_location.txt`|Path to the Helm Chart| +|`kubeconfig_path_to_test.txt`|Path to `kubectl` cluster configuration file| +|`runtime_cmd_to_test`|Location of `kubectl` binary| +|`sleep_duration.txt`|Optional: contains the number of seconds to pause| +|`version_to_test.txt`|Akri version to test| + + +If you review `/test/run-end-to-end.py` and `/test/shared_test_code.py`, you will see these files referenced. + +```Python3 +AGENT_LOG_PATH = "/tmp/agent_log.txt" +CONTROLLER_LOG_PATH = "/tmp/controller_log.txt" +KUBE_CONFIG_PATH_FILE = "/tmp/kubeconfig_path_to_test.txt" +RUNTIME_COMMAND_FILE = "/tmp/runtime_cmd_to_test.txt" +HELM_CRI_ARGS_FILE = "/tmp/cri_args_to_test.txt" +VERSION_FILE = "/tmp/version_to_test.txt" +SLEEP_DURATION_FILE = "/tmp/sleep_duration.txt" +EXTRA_HELM_ARGS_FILE = "/tmp/extra_helm_args.txt" +HELM_CHART_LOCATION = "/tmp/helm_chart_location.txt" +``` + +### Tests + +Of all the steps, only one is needed to run the Python end-to-end script. + +stdout|stderr from the script can be logged to the workflow. + +### Upload + +Once the end-to-end script is complete, the workflow uses the GitHub Action [upload-artifact](https://github.com/actions/upload-artifact) again to upload `/tmp/agent_log.txt` and `/tmp/controller_log.txt` so that these remain available (for download) once the workflow completes. \ No newline at end of file