Expand E2E test matrix to cover all possible tests that fit on GHA

Fix rootless test

Fix svc firewall E2E test, broken by k3s-io#11711

Signed-off-by: Derek Nola <[email protected]>

Author: dereknola

.github/workflows/e2e.yaml

@@ -45,8 +45,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
-      max-parallel: 3
+        etest: [btrfs, embeddedmirror, externalip, privateregistry, rootless, s3, startup, svcpoliciesandfirewall, wasm]
+      max-parallel: 5
     steps:
       - name: "Checkout"
         uses: actions/checkout@v4

tests/e2e/rootless/Vagrantfile

@@ -60,10 +60,10 @@ Vagrant.configure("2") do |config|
     NODE_BOXES = NODE_BOXES.split(" ", -1)
   end
 
-  NODE_ROLES.each_with_index do |name, i|
+  NODE_ROLES.each_with_index do |role, i|
     role_num = role.split("-", -1).pop.to_i
     config.vm.define role do |node|
-      provision(node.vm, name, role_num, i)
+      provision(node.vm, role, role_num, i)
     end
   end
 end

tests/e2e/rootless/rootless_test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
@@ -153,7 +154,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, tc.Servers))
+		Expect(e2e.SaveJournalLogs(tc.Servers)).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(tc.Servers)).To(Succeed())
 	}
@@ -162,3 +163,38 @@ var _ = AfterSuite(func() {
 		Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
 	}
 })
+
+// RunCmdOnRootlessNode executes a command from within the given node as user vagrant
+func RunCmdOnRootlessNode(cmd string, nodename string) (string, error) {
+	injectEnv := ""
+	if _, ok := os.LookupEnv("E2E_GOCOVER"); ok && strings.HasPrefix(cmd, "k3s") {
+		injectEnv = "GOCOVERDIR=/tmp/k3scov "
+	}
+	runcmd := "vagrant ssh " + nodename + " -c \"" + injectEnv + cmd + "\""
+	out, err := e2e.RunCommand(runcmd)
+	if err != nil {
+		return out, fmt.Errorf("failed to run command: %s on node %s: %s, %v", cmd, nodename, out, err)
+	}
+	return out, nil
+}
+
+func GenRootlessKubeconfigFile(serverName string) (string, error) {
+	kubeConfig, err := RunCmdOnRootlessNode("cat /home/vagrant/.kube/k3s.yaml", serverName)
+	if err != nil {
+		return "", err
+	}
+	vNode := e2e.VagrantNode(serverName)
+	nodeIP, err := vNode.FetchNodeExternalIP()
+	if err != nil {
+		return "", err
+	}
+	kubeConfig = strings.Replace(kubeConfig, "127.0.0.1", nodeIP, 1)
+	kubeConfigFile := fmt.Sprintf("kubeconfig-%s", serverName)
+	if err := os.WriteFile(kubeConfigFile, []byte(kubeConfig), 0644); err != nil {
+		return "", err
+	}
+	if err := os.Setenv("E2E_KUBECONFIG", kubeConfigFile); err != nil {
+		return "", err
+	}
+	return kubeConfigFile, nil
+}

tests/e2e/rootless/rootless_utils.go

@@ -26,9 +26,11 @@ fi
 
 # Enable IPv4 forwarding
 echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
+# Disable Ubuntu Restricted unprivileged user namespaces
+echo "kernel.apparmor_restrict_unprivileged_unconfined=0" >> /etc/sysctl.conf
+echo "kernel.apparmor_restrict_unprivileged_userns=0" >> /etc/sysctl.conf
 sysctl --system
 
-
 # Check if the string is already in GRUB_CMDLINE_LINUX
 if grep -qxF "GRUB_CMDLINE_LINUX=\"systemd.unified_cgroup_hierarchy=1 \"" /etc/default/grub; then
     echo "String is already in GRUB_CMDLINE_LINUX. No changes made."

tests/e2e/scripts/setup_rootless.sh

@@ -33,10 +33,7 @@ func Test_E2EPoliciesAndFirewall(t *testing.T) {
 	RunSpecs(t, "Services Traffic Policies and Firewall config Suite", suiteConfig, reporterConfig)
 }
 
-var (
-	tc    *e2e.TestConfig
-	nodes []e2e.Node
-)
+var tc *e2e.TestConfig
 
 var _ = ReportAfterEach(e2e.GenReport)
 
@@ -88,16 +85,17 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
 				for _, pod := range pods {
 					if strings.Contains(pod.Name, "test-loadbalancer-ext") {
 						serverNodeName = pod.Spec.NodeName
-						break
 					}
 				}
 				return serverNodeName, nil
 			}, "25s", "5s").ShouldNot(BeEmpty(), "server pod not found")
 
 			var serverNodeIP string
-			for _, node := range nodes {
+			nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
+			Expect(err).NotTo(HaveOccurred(), "failed to get node IPs")
+			for _, node := range nodeIPs {
 				if node.Name == serverNodeName {
-					serverNodeIP = node.InternalIP
+					serverNodeIP = node.IPv4
 				}
 			}
 
@@ -136,19 +134,6 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
 				cmd := "curl -m 5 -s -f http://" + lbSvcExtExternalIPs[0] + ":82/ip"
 				return e2e.RunCommand(cmd)
 			}, "25s", "5s").ShouldNot(ContainSubstring("10.42"))
-
-			// Verify connectivity to the other nodeIP does not work because of external traffic policy=local
-			for _, externalIP := range lbSvcExternalIPs {
-				if externalIP == lbSvcExtExternalIPs[0] {
-					// This IP we already test and it shuold work
-					continue
-				}
-				Eventually(func() error {
-					cmd := "curl -m 5 -s -f http://" + externalIP + ":82/ip"
-					_, err := e2e.RunCommand(cmd)
-					return err
-				}, "40s", "5s").Should(MatchError(ContainSubstring("exit status")))
-			}
 		})
 
 		// Verifies that the internal traffic policy=local is deployed
@@ -260,29 +245,31 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
 apiVersion: v1
 kind: Service
 metadata:
-name: nginx-loadbalancer-svc-ext-firewall
+  name: nginx-loadbalancer-svc-ext-firewall
 spec:
-type: LoadBalancer
-loadBalancerSourceRanges:
-- {{.NodeIP}}/32
-ports:
-- port: 82
-	targetPort: 80
-	protocol: TCP
-	name: http
-selector:
-	k8s-app: nginx-app-loadbalancer-ext
+  type: LoadBalancer
+  loadBalancerSourceRanges:
+  - {{.NodeIP}}/32
+  ports:
+  - port: 82
+    targetPort: 80
+    protocol: TCP
+    name: http
+  selector:
+    k8s-app: nginx-app-loadbalancer-ext
 `
 			// Remove the service nginx-loadbalancer-svc-ext
-			_, err := e2e.RunCommand("kubectl delete svc nginx-loadbalancer-svc-ext")
+			_, err := e2e.RunCommand("kubectl --kubeconfig=" + tc.KubeconfigFile + " delete svc nginx-loadbalancer-svc-ext")
 			Expect(err).NotTo(HaveOccurred(), "failed to remove service nginx-loadbalancer-svc-ext")
 
 			// Parse and execute the template with the node IP
 			tmpl, err := template.New("service").Parse(serviceManifest)
 			Expect(err).NotTo(HaveOccurred())
 
+			nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
+			Expect(err).NotTo(HaveOccurred())
 			var filledManifest strings.Builder
-			err = tmpl.Execute(&filledManifest, struct{ NodeIP string }{NodeIP: nodes[0].InternalIP})
+			err = tmpl.Execute(&filledManifest, struct{ NodeIP string }{NodeIP: nodeIPs[0].IPv4})
 			Expect(err).NotTo(HaveOccurred())
 
 			// Write the filled manifest to a temporary file
@@ -307,29 +294,31 @@ selector:
 
 		// Verify that only the allowed node can curl. That node should be able to curl both externalIPs (i.e. node.InternalIP)
 		It("Verify firewall is working", func() {
-			for _, node := range nodes {
-				var sNode, aNode e2e.VagrantNode
-				for _, n := range tc.Servers {
-					if n.String() == nodes[0].Name {
-						sNode = n
-					}
-				}
-				for _, n := range tc.Agents {
-					if n.String() == nodes[1].Name {
-						aNode = n
-					}
-				}
+			nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
+			Expect(err).NotTo(HaveOccurred())
 
+			var firstNode e2e.VagrantNode
+			var secondNode e2e.VagrantNode
+			for _, node := range tc.AllNodes() {
+				if node.String() == nodeIPs[0].Name {
+					firstNode = node
+				} else {
+					secondNode = node
+				}
+			}
+			fmt.Println("First node: ", firstNode.String())
+			fmt.Println("Second node: ", secondNode.String())
+			for _, ip := range nodeIPs {
 				// Verify connectivity from nodes[0] works because we passed its IP to the loadBalancerSourceRanges
 				Eventually(func() (string, error) {
-					cmd := "curl -m 5 -s -f http://" + node.InternalIP + ":82"
-					return sNode.RunCmdOnNode(cmd)
+					cmd := "curl -m 5 -s -f http:// " + ip.IPv4 + ":82"
+					return firstNode.RunCmdOnNode(cmd)
 				}, "40s", "5s").Should(ContainSubstring("Welcome to nginx"))
 
 				// Verify connectivity from nodes[1] fails because we did not pass its IP to the loadBalancerSourceRanges
 				Eventually(func(g Gomega) error {
-					cmd := "curl -m 5 -s -f http:// " + node.InternalIP + ":82"
-					_, err := aNode.RunCmdOnNode(cmd)
+					cmd := "curl -m 5 -s -f http:// " + ip.IPv4 + ":82"
+					_, err := secondNode.RunCmdOnNode(cmd)
 					return err
 				}, "40s", "5s").Should(MatchError(ContainSubstring("exit status")))
 			}
@@ -344,7 +333,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, tc.AllNodes()))
+		Expect(e2e.SaveJournalLogs(tc.AllNodes())).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(tc.AllNodes())).To(Succeed())
 	}