Add rerty, VM log capture on failure

Signed-off-by: Derek Nola <[email protected]>
k3s-io · Oct 31, 2023 · ed8b27a · ed8b27a
1 parent 7819163
commit ed8b27a
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 15 deletions.
diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml
@@ -31,7 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         vm: [centos-7, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2004, ubuntu-2204]
-      max-parallel: 2
+      max-parallel: 1
     defaults:
       run:
         working-directory: tests/install/${{ matrix.vm }}
@@ -58,7 +58,26 @@ jobs:
           name: k3s
           path: tests/install/${{ matrix.vm }}
       - name: "Vagrant Up"
-        run: vagrant up --no-provision
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          # Workaround for https://github.com/nick-fields/retry/issues/89
+          command: cd tests/install/${{ matrix.vm }} && vagrant up --no-provision
+          on_retry_command: cd tests/install/${{ matrix.vm }} && vagrant destroy -f
+          retry_wait_seconds: 30
+      - name: On Failure, Debug VM status, capture logs
+        if: ${{ failure() || cancelled() }}
+        run: |
+          logsDir=$(vboxmanage list vms --long | grep Logs | awk '{print $3" "$4}')
+          cp "$logsDir"/VBox.log /tmp/${{ matrix.vm }}.log
+          ls /tmp/${{ matrix.vm }}.log
+      - name: On Failure, Upload VM logs
+        if: ${{ failure() || cancelled() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.vm }}.log
+          path: /tmp/${{ matrix.vm }}.log
       - name: "Upload k3s binary"
         run: |
           chmod +x k3s
@@ -90,3 +109,11 @@ jobs:
         run: vagrant provision --provision-with=k3s-status
       - name: "k3s-procps"
         run: vagrant provision --provision-with=k3s-procps
+      - name: Cleanup VM
+        run: vagrant destroy -f
+      - name: On Failure, launch debug session
+        uses: lhotari/action-upterm@v1
+        if: ${{ failure() }}
+        with:
+          ## If no one connects after 5 minutes, shut down server.
+          wait-timeout-minutes: 5
diff --git a/.github/workflows/snapshotter.yaml b/.github/workflows/snapshotter.yaml
@@ -47,18 +47,6 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v4
         with: { fetch-depth: 1 }
-      - name: Set up Homebrew
-        uses: Homebrew/actions/setup-homebrew@master
-      - name: "Install libvirt"
-        run: |
-          brew install libvirt
-          brew services start libvirt
-      - name: On Failure, launch debug session
-        uses: lhotari/action-upterm@v1
-        if: ${{ failure() }}
-        with:
-          ## If no one connects after 5 minutes, shut down server.
-          wait-timeout-minutes: 5
       - name: "Download Binary"
         uses: actions/download-artifact@v3
         with: { name: k3s, path: dist/artifacts/ }
@@ -75,11 +63,23 @@ jobs:
         run: vagrant plugin install vagrant-k3s
       - name: "Vagrant Up ⏩ Install K3s"
         run: vagrant up
+      - name: On Failure, Debug VM status, capture logs
+        if: ${{ failure() || cancelled() }}
+        run: |
+          logsDir=$(vboxmanage list vms --long | grep Logs | awk '{print $3" "$4}')
+          cp "$logsDir"/VBox.log /tmp/${{ matrix.vm }}.log
+          ls /tmp/${{ matrix.vm }}.log
+      - name: On Failure, Upload VM logs
+        if: ${{ failure() || cancelled() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.vm }}.log
+          path: /tmp/${{ matrix.vm }}.log
       - name: "⏳ Node"
         run: vagrant provision --provision-with=k3s-wait-for-node
       - name: "⏳ CoreDNS"
         run: vagrant provision --provision-with=k3s-wait-for-coredns
       - name: "k3s-status" # kubectl get node,all -A -o wide
         run: vagrant provision --provision-with=k3s-status
       - name: "k3s-snapshots" # if no snapshots then we fail
-        run: vagrant provision --provision-with=k3s-snapshots
+        run: vagrant provision --provision-with=k3s-snapshots