Add e2e distribution tests (#160)

* assume yes for yum installs

* initial e2e script

* Initial gh action test

* chdir before box update

* test nested virtualization

* test cache

* Test matrix

* Fix matrix test

* github action update

* Test to ensure masking of token with fake token

* Test masking with fake token

* Switch role and test masking with fake key

* Fix env name and test with fake token

* Passing test_api_token

* First full e2e test with fake token

* sanitize dist and update cache key

* pass dist to vagrant env

* change cache key

* Tail logs during 45s wait

* Trap exits

* Trap exits

* kill journalctl tail

* Remove background journalctl job / restore mem/cpu

* sleep required

* journalctl required only once

* Add schedule. 5pm UTC every day

* split steps

* consistent casing for vars

* Test CI build

* Removed on push gh action trigger

.github/workflows/dist-e2e.yml

@@ -0,0 +1,76 @@
+on:
+  schedule:
+    - cron: '0 17 * * *' # every day at 5pm UTC
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      all-dists: ${{ steps.get-all-dists.outputs.dists }}
+    steps:
+      - uses: actions/checkout@v3
+      - id: get-all-dists
+        name: Get all distributions
+        run: |
+          cd dist
+          echo "dists=$(ls -ld */* | grep ^d | awk '{print $9}' | jq -cnMR '[inputs | select(length>0)]')" >> $GITHUB_OUTPUT
+  e2e:
+    # We use this runner because it is currently the only runner that supports nested virtualization.
+    # See https://github.com/actions/virtual-environments/issues/433 for more
+    # information
+    runs-on: macos-12
+    timeout-minutes: 30 # If something hangs, timeout the job in 30 so we aren't billed for 6h * distributions * 10 (mac-os minute multiplier)
+    needs:
+      - setup
+    permissions:
+      id-token: write
+      contents: read      
+    strategy:
+      fail-fast: false
+      matrix: 
+        dist: ${{ fromJSON(needs.setup.outputs.all-dists) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::147803588724:role/github-action
+          aws-region: us-west-2
+
+      - name: Import e2e API TOKEN
+        run: |
+          e2e_api_token=$(aws secretsmanager get-secret-value --secret-id /prod/gh-actions/orchestrator-e2e-token --region us-west-2 | jq -r '.SecretString')
+          echo "::add-mask::${e2e_api_token}"
+          echo "TEST_API_TOKEN=${e2e_api_token}" >> $GITHUB_ENV
+
+      - name: Cache Vagrant box
+        # Caches that are not accessed within the last week will be evicted
+        # If any or all of the vagrant files change a lot in a short period of time
+        # the cache may hit its 10GB limit at which point it will just start trimming
+        # cached items in order of oldest to newest. Worst case it will re-download a vagrant box
+        # (see https://github.com/actions/cache#cache-limits)
+        uses: actions/cache@v3
+        with:
+          path: ~/.vagrant.d/boxes
+          key: ${{ runner.os }}-vagrant-${{ matrix.dist }}-${{ hashFiles('**/Vagrantfile') }}
+
+      - name: Provision e2e environment
+        run: |
+          cd ./dist/${{ matrix.dist }}
+          vagrant box update
+          vagrant up --provision
+
+      - name: Run e2e
+        env:
+          DIST: ${{ matrix.dist }}
+        run: |
+          cd ./dist/${{ matrix.dist }}
+          vagrant upload ../../e2e.sh
+          vagrant ssh -c "TEST_API_TOKEN=$TEST_API_TOKEN DIST=$DIST /bin/bash /home/vagrant/e2e.sh"
+
+      - name: Destroy e2e environment
+        run: |
+          cd ./dist/${{ matrix.dist }}
+          vagrant destroy -f

dist/amazon-linux/2/Vagrantfile

@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   # commercial use we need to sort out first.
   config.vm.provider "libvirt" do |vm|
     vm.memory = "8192"
-    vm.cpus = 8
+    vm.cpus = 2
     vm.memorybacking :access, :mode => "shared"
   end
   config.vm.synced_folder "../../../", "/vagrant" #, type: "virtiofs"

dist/e2e.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+#  This script is primarily intended to be used with the Vagrant definitions in each dist folder
+#  A monitor should be setup in the account for which $TEST_API_TOKEN belongs to (testsignal is a good option)
+#
+
+# Oddly enough even after yum remove or apt remove the orchestrator is still running. Clean that up
+StopAndDisableOrchestrator() {
+    sudo systemctl stop metrist-orchestrator
+    sudo systemctl disable metrist-orchestrator
+    sudo systemctl daemon-reload
+}
+
+RemoveOrchestrator(){
+    StopAndDisableOrchestrator
+
+    if type apt >/dev/null; then
+        sudo apt purge -y metrist-orchestrator
+    elif type yum >/dev/null; then
+        sudo yum remove -y metrist-orchestrator
+    fi
+}
+
+Main() {
+
+# Sanitize DIST env var before using it for instance id - remove forward slashes, hyphens, and periods
+DIST=${DIST//[\/\.\-]/}
+
+curl https://dist.metrist.io/install.sh >/tmp/install.sh
+
+cat <<EOF | bash /tmp/install.sh
+$TEST_API_TOKEN
+e2e_test_$DIST
+EOF
+
+sleep 45
+
+LOGS=$(sudo journalctl --unit metrist-orchestrator --since "1m ago" --no-pager)
+echo "$LOGS"
+SUCCESS_COUNT=$(echo "$LOGS" | grep -c "All steps done, asking monitor to exit")
+
+if [ $SUCCESS_COUNT -gt 0 ]; then
+    RemoveOrchestrator
+    echo "e2e successful" 
+    exit 0
+else
+    RemoveOrchestrator
+    echo "Error during e2e, can't find 'All steps done'"
+    exit 1
+fi
+
+}
+
+Main

dist/interactive/install.sh

@@ -287,7 +287,7 @@ InstallYum() {
     cd /tmp
     latest=$($CURL https://dist.metrist.io/orchestrator/$OS/$VERSION.$ARCH.latest.txt)
     $CURL "https://dist.metrist.io/orchestrator/$OS/$latest" >$latest
-    $SUDO yum localinstall ./$latest
+    $SUDO yum localinstall -y ./$latest
     cat <<EOF | $SUDO tee -a /etc/default/metrist-orchestrator >/dev/null
 
 # Added by installation script.

dist/ubuntu/20.04/Vagrantfile

@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   # commercial use we need to sort out first.
   config.vm.provider "libvirt" do |vm|
     vm.memory = "8192"
-    vm.cpus = 8
+    vm.cpus = 2
     vm.memorybacking :access, :mode => "shared"
   end
   config.vm.synced_folder "../../../", "/vagrant" #, type: "virtiofs"

dist/ubuntu/22.04/Vagrantfile

@@ -10,7 +10,7 @@ Vagrant.configure("2") do |config|
   # commercial use we need to sort out first.
   config.vm.provider "libvirt" do |vm|
     vm.memory = "8192"
-    vm.cpus = 8
+    vm.cpus = 2
     vm.memorybacking :access, :mode => "shared"
   end
   config.vm.synced_folder "../../../", "/vagrant" #, type: "virtiofs"