VM multi-kernel testing using little-vm-helper

.dockerignore

@@ -20,3 +20,6 @@ contrib/tester-progs/fork-tester
 contrib/tester-progs/namespace-tester
 contrib/tester-progs/sigkill-tester
 contrib/tester-progs/trigger-test-events
+# ignore files downloaded by tests/vmtests/fetch-data.sh
+tests/vmtests/test-data
+tests/vmtests/vmtests-results-*

.github/workflows/vmtests.yml

@@ -0,0 +1,143 @@
+name: vmtests
+on:
+  pull_request:
+   paths-ignore:
+   - "**.md"
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    name: Build tetragon
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: '1.18.3'
+
+    - name: Install docker
+      uses: docker-practice/actions-setup-docker@master
+
+    - name: Checkout code
+      uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      with:
+        path: go/src/github.com/cilium/tetragon/
+
+    - name: Install build dependencies
+      run: |
+        sudo apt install libelf-dev netcat-traditional libcap-dev gcc
+        echo `which clang`
+        echo `which llc`
+        echo `clang --version`
+    - name: Build
+      env:
+        GOPATH: /home/runner/work/tetragon/tetragon/go
+      run: |
+        cd go/src/github.com/cilium/tetragon/
+        make tetragon-bpf tester-progs test-compile
+        make -C tests/vmtests
+
+    - name: Split tests
+      run: |
+        #  see testfile below
+        cd go/src/github.com/cilium/tetragon/
+        go run ./tools/split-tetragon-gotests -ci-run 3
+
+    - name: tar build
+      run: |
+        cd go/src/github.com/cilium/
+        tar cz --exclude='tetragon/.git' -f /tmp/tetragon.tar ./tetragon
+
+    - name: upload build
+      uses: actions/upload-artifact@v3
+      with:
+         name: tetragon-build
+         path: /tmp/tetragon.tar
+         retention-days: 5
+  test:
+    strategy:
+        matrix:
+           kernel:
+              - '5.10'
+              - '5.4'
+              - '4.19'
+           group:
+              - 0
+              - 1
+              - 2
+    needs: build
+    name: Test kernel ${{ matrix.kernel }} / test group ${{ matrix.group }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+    - name: Install docker
+      uses: docker-practice/actions-setup-docker@master
+    - name: Install VM test dependencies
+      run: |
+        sudo apt-get -qy update
+        sudo apt-cache search qemu
+        sudo apt-get -qy install mmdebstrap libguestfs-tools qemu-utils qemu-system-x86 docker
+
+    - name: Make kernel accessible
+      run: |
+        sudo chmod go+rX -R /boot/
+
+    - name: set panic on RCU stall
+      run: |
+        sudo sh -c 'echo 1 > /proc/sys/kernel/panic_on_rcu_stall'
+
+    - name: download build data
+      uses: actions/download-artifact@v3
+      with:
+         name: tetragon-build
+
+    - name: extract build data
+      # NB: currently, due to how tests work, we need to extract to the same path.
+      run: |
+        mkdir -p go/src/github.com/cilium/
+        tar xf tetragon.tar -C go/src/github.com/cilium
+
+    - name: test kernel ${{ matrix.kernel }}
+      if: ${{ matrix.kernel  != '4.19' }}
+      run: |
+        cd go/src/github.com/cilium/tetragon
+        ./tests/vmtests/fetch-data.sh ${{ matrix.kernel }}
+        ./tests/vmtests/tetragon-vmtests-run \
+                --kernel tests/vmtests/test-data/kernels/${{ matrix.kernel }}/boot/vmlinuz* \
+                --base tests/vmtests/test-data/images/base.qcow2 \
+                --testsfile ./tests/vmtests/test-group-${{ matrix.group }}
+
+    - name: test kernel ${{ matrix.kernel }} with btf file
+      if: ${{ matrix.kernel  == '4.19' }}
+      run: |
+        cd go/src/github.com/cilium/tetragon
+        ./tests/vmtests/fetch-data.sh ${{ matrix.kernel }}
+        ./tests/vmtests/tetragon-vmtests-run \
+                --kernel tests/vmtests/test-data/kernels/${{ matrix.kernel }}/boot/vmlinuz* \
+                --btf-file tests/vmtests/test-data/kernels/${{ matrix.kernel }}/boot/btf-* \
+                --base tests/vmtests/test-data/images/base.qcow2 \
+                --testsfile ./tests/vmtests/test-group-${{ matrix.group }}
+
+    - name: Upload test results on failure or cancelation
+      if: failure() || cancelled()
+      uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+      with:
+        name: tetragon-vmtests-${{ matrix.kernel }}-${{ matrix.group }}-results
+        path: go/src/github.com/cilium/tetragon/tests/vmtests/vmtests-results-*
+        retention-days: 5
+  post-test:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: success()
+    steps:
+    - name: Create truncated build file
+      run: |
+        touch /tmp/tetragon.tar
+    - name: Upload truncated file
+      uses: actions/upload-artifact@v3
+      with:
+         name: tetragon-build
+         path: /tmp/tetragon.tar
+         retention-days: 1

cmd/protoc-gen-go-tetragon/eventchecker/multichecker.go

@@ -140,33 +140,38 @@ func generateUnorderedEventChecker(g *protogen.GeneratedFile) error {
             return true, nil
         }
 
+        totalMatched := checker.totalChecks - pending
         if logger != nil {
-            logger.Infof("UnorderedEventChecker: %d/%d matched", checker.totalChecks-pending, checker.totalChecks)
+            logger.Infof("UnorderedEventChecker: checking event with %d/%d total matched", totalMatched, checker.totalChecks)
         }
         idx := 1
 
         for e := checker.pendingChecks.Front(); e != nil; e = e.Next() {
             check := e.Value.(EventChecker)
             err := check.CheckEvent(event)
             if err == nil {
+                totalMatched++
+                if logger != nil {
+                        logger.Infof("UnorderedEventChecker: successfully matched %d/%d", totalMatched, checker.totalChecks)
+                }
                 checker.pendingChecks.Remove(e)
                 pending--
                 if pending > 0 {
                     return false, nil
                 }
 
                 if logger != nil {
-                    logger.Infof("UnorderedEventChecker: all %d checks matched", checker.totalChecks)
+                    logger.Infof("UnorderedEventChecker: all %d check(s) matched", checker.totalChecks)
                 }
                 return true, nil
             }
             if logger != nil {
-                logger.Infof("UnorderedEventChecker: checking %d/%d: failure: %s", idx, pending, err)
+                logger.Infof("UnorderedEventChecker: checking pending %d/%d: failure: %s", idx, pending, err)
             }
             idx++
         }
 
-        return false, ` + common.FmtErrorf(g, "UnorderedEventChecker: all %d checks failed", "pending") + `
+        return false, ` + common.FmtErrorf(g, "UnorderedEventChecker: all %d check(s) failed", "pending") + `
     }`)
 
 	g.P(`// FinalCheck implements the MultiEventChecker interface

cmd/tetragon-tester/README.md

@@ -0,0 +1 @@
+a program to perform tests inside a VM

cmd/tetragon-tester/init.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/cilium/tetragon/pkg/vmtests"
+	"golang.org/x/sys/unix"
+)
+
+// NB(kkourt): this is meant for running this program as init. It kinda works,
+// but I think creates more trouble that being useful at this point. I'll leave
+// the code for now, just in case we want to actually use it at some point. If
+// it keeping the code creates issues, let's just delete it.
+func doPID0() {
+	// mount proc and other filesystems that we need
+	if err := unix.Mount("none", "/proc", "proc", 0, ""); err != nil {
+		fmt.Printf("failed to mount proc: %v\n", err)
+	}
+
+	if err := unix.Mount("none", "/sys", "sysfs", 0, ""); err != nil {
+		fmt.Printf("failed to mount sysfs: %v\n", err)
+	}
+
+	if err := unix.Mount("none", "/sys/kernel/debug", "debugfs", 0, ""); err != nil {
+		fmt.Printf("failed to mount debugfs: %v\n", err)
+	}
+
+	if err := unix.Mount("none", "/sys/kernel/debug", "debugfs", 0, ""); err != nil {
+		fmt.Printf("failed to mount debugfs: %v\n", err)
+	}
+
+	if err := unix.Mount("/dev/root", "/", "", unix.MS_REMOUNT, ""); err != nil {
+		fmt.Printf("failed to remount /: %v\n", err)
+	}
+
+	// TODO: do mount -a, to mount everything in /etc/fstab
+
+}
+
+// https://github.com/aisola/go-coreutils/blob/6eb4c2d5305ac4795a562573fbd9b39d6cbdfc10/uname/uname.go#L98
+// utsnameToString converts the utsname to a string and returns it.
+func utsnameToString(unameArray [65]byte) string {
+	var byteString [65]byte
+	var indexLength int
+	for ; unameArray[indexLength] != 0; indexLength++ {
+		byteString[indexLength] = uint8(unameArray[indexLength])
+	}
+	return string(byteString[:indexLength])
+}
+
+// printInfo prints a short message similar to unmame
+func printInfo() {
+	var uname unix.Utsname
+	if err := unix.Uname(&uname); err != nil {
+		fmt.Printf("failed to execute Uname: %v", err)
+	}
+
+	fmt.Printf("%s %s %s %s\n", utsnameToString(uname.Sysname), utsnameToString(uname.Nodename), utsnameToString(uname.Release), utsnameToString(uname.Version))
+}
+
+// tester performs the following
+//  - reads the configuration from the configuration file
+//  - runs the tests
+//  - marshalls the results in results.json
+func tester() int {
+	var err error
+	var data []byte
+	var conf vmtests.Conf
+	if data, err = os.ReadFile(vmtests.ConfFile); err == nil {
+		if err = json.Unmarshal(data, &conf); err != nil {
+			fmt.Printf("failed to load config file %s: %v", vmtests.ConfFile, err)
+			return 1
+		}
+	} else {
+		// TODO: add proper parameters to handle local execution use-case
+		tetragonDir := "."
+		resultsDir, err := os.MkdirTemp(tetragonDir, "tetragon-tester-")
+		if err != nil {
+			fmt.Printf("failed to create results directory: %v\n", err)
+			return 1
+		}
+
+		conf = vmtests.Conf{
+			NoPowerOff:  true,
+			TetragonDir: tetragonDir,
+			ResultsDir:  resultsDir,
+		}
+		fmt.Printf("no configuration found: %v. Using default configuration for local execution (resultsdir:%s)\n", err, resultsDir)
+	}
+
+	if !conf.NoPowerOff {
+		defer func() {
+			fmt.Println("tetragon-tester shutting down the machine...")
+			os.Stdout.Sync()
+			time.Sleep(100 * time.Millisecond)
+			if err := os.WriteFile("/proc/sysrq-trigger", []byte("o"), 0777); err != nil {
+				fmt.Printf("failed to use syseq-trigger to shutdown machine")
+				return
+			}
+			// wait for the powerdown before init is killed to avoid confusing messages from the kernel
+			for {
+				time.Sleep(1 * time.Second)
+			}
+		}()
+	}
+
+	if os.Getpid() == 1 {
+		doPID0()
+	}
+
+	printInfo()
+	err = vmtests.Run(&conf)
+	if err != nil {
+		fmt.Printf("error running vmtests: %v\n", err)
+		return 1
+	}
+
+	return 0
+}
+
+func main() {
+	os.Exit(tester())
+}

cmd/tetragon-vmtests-run/conf.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/cilium/tetragon/pkg/vmtests"
+)
+
+type RunConf struct {
+	testImage             string
+	baseFname             string
+	kernelFname           string
+	dontRebuildImage      bool
+	useTetragonTesterInit bool
+	testerOut             string
+	qemuPrint             bool
+	justBoot              bool
+	disableKVM            bool
+	btfFile               string
+	testerConf            vmtests.Conf
+
+	filesystems []QemuFS
+}
+
+func (rc *RunConf) testImageFname() string {
+	imagesDir := filepath.Dir(rc.baseFname)
+	return filepath.Join(imagesDir, rc.testImage)
+}