Skip to content

Commit 7bc5ee9

Browse files
dereknoladereknola
committed
Convert legacy docker tests from bash to golang (#11357)
* Convert the following Docker test from Bash to Go - basics - bootstraptoken - cacerts - compat -> skew - etcd - lazypull - upgrade Signed-off-by: Derek Nola <[email protected]> * Add Docker go tests to GHA * Prebuild K3s Go Tests * Strip go test binaries to reduce size * Handle complex branch options Signed-off-by: Derek Nola <[email protected]> * Implement basic golang tests on arm and arm64 pipelines Signed-off-by: Derek Nola <[email protected]>
1 parent 4331f45 commit 7bc5ee9

File tree

13 files changed

+1430
-1
lines changed

13 files changed

+1430
-1
lines changed

Diff for: ‎.github/workflows/e2e.yaml

+82
Original file line numberDiff line numberDiff line change
@@ -121,3 +121,85 @@ jobs:
121121
. ./tests/docker/test-helpers
122122
. ./tests/docker/test-run-${{ matrix.dtest }}
123123
echo "Did test-run-${{ matrix.dtest }} pass $?"
124+
125+
build-go-tests:
126+
name: "Build Go Tests"
127+
runs-on: ubuntu-latest
128+
outputs:
129+
branch_name: ${{ steps.branch_step.outputs.BRANCH_NAME }}
130+
steps:
131+
- name: Checkout
132+
uses: actions/checkout@v4
133+
- name: Install Go
134+
uses: ./.github/actions/setup-go
135+
- name: Build Go Tests
136+
run: |
137+
mkdir -p ./dist/artifacts
138+
go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/...
139+
- name: Upload Go Tests
140+
uses: actions/upload-artifact@v4
141+
with:
142+
name: docker-go-tests
143+
path: ./dist/artifacts/*.test
144+
compression-level: 9
145+
retention-days: 1
146+
# For upgrade and skew tests, we need to know the branch name this run is based off.
147+
# Since this is predetermined, we can run this step before the docker-go job, saving time.
148+
# For PRs we can use the base_ref (ie the target branch of the PR).
149+
# For pushes to k3s-io/k3s, the branch_name is a valid ref, master or release-x.y.
150+
# For pushes to a fork, we need to determine the branch name by finding the parent branch from git show-branch history.
151+
- name: Determine branch name
152+
id: branch_step
153+
run: |
154+
if [ ${{ github.repository }} = "k3s-io/k3s" ]; then
155+
BRANCH_NAME=$(echo ${{ github.base_ref || github.ref_name }})
156+
elif [ -z "${{ github.base_ref }}" ]; then
157+
# We are in a fork, and need some git history to determine the branch name
158+
git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
159+
BRANCH_NAME=$(git show-branch -a 2> /dev/null | grep '\*' | grep -v `git rev-parse --abbrev-ref HEAD` | head -n1 | sed 's/.*\[\(.*\/\)\(.*\)\].*/\2/' | sed 's/[\^~].*//')
160+
else
161+
BRANCH_NAME=${{ github.base_ref }}
162+
fi
163+
echo "Branch Name is $BRANCH_NAME"
164+
echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
165+
166+
docker-go:
167+
needs: [build, build-go-tests]
168+
name: Docker Tests In GO
169+
runs-on: ubuntu-latest
170+
timeout-minutes: 20
171+
strategy:
172+
fail-fast: false
173+
matrix:
174+
dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, upgrade]
175+
env:
176+
BRANCH_NAME: ${{ needs.build-go-tests.outputs.branch_name }}
177+
steps:
178+
- name: Checkout
179+
uses: actions/checkout@v4
180+
- name: "Download K3s image"
181+
uses: actions/download-artifact@v4
182+
with:
183+
name: k3s
184+
path: ./dist/artifacts
185+
- name: Load and set K3s image
186+
run: |
187+
docker image load -i ./dist/artifacts/k3s-image.tar
188+
IMAGE_TAG=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep 'rancher/k3s')
189+
echo "K3S_IMAGE=$IMAGE_TAG" >> $GITHUB_ENV
190+
- name: Download Go Tests
191+
uses: actions/download-artifact@v4
192+
with:
193+
name: docker-go-tests
194+
path: ./dist/artifacts
195+
- name: Run ${{ matrix.dtest }} Test
196+
# Put the compied test binary back in the same place as the test source
197+
run: |
198+
chmod +x ./dist/artifacts/${{ matrix.dtest }}.test
199+
mv ./dist/artifacts/${{ matrix.dtest }}.test ./tests/docker/${{ matrix.dtest }}/
200+
cd ./tests/docker/${{ matrix.dtest }}
201+
if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
202+
./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -branch=$BRANCH_NAME
203+
else
204+
./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE
205+
fi

Diff for: ‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ require (
148148
go.etcd.io/etcd/server/v3 v3.5.16
149149
go.uber.org/zap v1.27.0
150150
golang.org/x/crypto v0.27.0
151+
golang.org/x/mod v0.20.0
151152
golang.org/x/net v0.29.0
152153
golang.org/x/sync v0.8.0
153154
golang.org/x/sys v0.28.0
@@ -446,7 +447,6 @@ require (
446447
go.uber.org/mock v0.4.0 // indirect
447448
go.uber.org/multierr v1.11.0 // indirect
448449
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
449-
golang.org/x/mod v0.20.0 // indirect
450450
golang.org/x/oauth2 v0.22.0 // indirect
451451
golang.org/x/term v0.24.0 // indirect
452452
golang.org/x/text v0.18.0 // indirect

Diff for: ‎scripts/test

+4
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ docker ps
2323
# Only run basic tests on non amd64 archs, we use GitHub Actions for amd64
2424
if [ "$ARCH" != 'amd64' ]; then
2525

26+
export K3S_IMAGE="rancher/k3s:${VERSION_TAG}${SUFFIX}"
27+
go test ./tests/docker/basics/basics_test.go -k3sImage="$K3S_IMAGE"
28+
echo "Did go test basics $?"
29+
2630
. ./tests/docker/test-run-basics
2731
echo "Did test-run-basics $?"
2832

Diff for: ‎tests/docker/basics/basics_test.go

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"os"
7+
"strings"
8+
"testing"
9+
10+
tester "github.com/k3s-io/k3s/tests/docker"
11+
. "github.com/onsi/ginkgo/v2"
12+
. "github.com/onsi/gomega"
13+
)
14+
15+
var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
16+
var config *tester.TestConfig
17+
18+
func Test_DockerBasic(t *testing.T) {
19+
flag.Parse()
20+
RegisterFailHandler(Fail)
21+
RunSpecs(t, "Basic Docker Test Suite")
22+
}
23+
24+
var _ = Describe("Basic Tests", Ordered, func() {
25+
26+
Context("Setup Cluster", func() {
27+
It("should provision servers and agents", func() {
28+
var err error
29+
config, err = tester.NewTestConfig(*k3sImage)
30+
Expect(err).NotTo(HaveOccurred())
31+
Expect(config.ProvisionServers(1)).To(Succeed())
32+
Expect(config.ProvisionAgents(1)).To(Succeed())
33+
Eventually(func() error {
34+
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
35+
}, "60s", "5s").Should(Succeed())
36+
Eventually(func() error {
37+
return tester.NodesReady(config.KubeconfigFile)
38+
}, "40s", "5s").Should(Succeed())
39+
})
40+
})
41+
42+
Context("Use Local Storage Volume", func() {
43+
It("should apply local storage volume", func() {
44+
const volumeTestManifest = "../resources/volume-test.yaml"
45+
46+
// Apply the manifest
47+
cmd := fmt.Sprintf("kubectl apply -f %s --kubeconfig=%s", volumeTestManifest, config.KubeconfigFile)
48+
_, err := tester.RunCommand(cmd)
49+
Expect(err).NotTo(HaveOccurred(), "failed to apply volume test manifest")
50+
})
51+
It("should validate local storage volume", func() {
52+
Eventually(func() (bool, error) {
53+
return tester.PodReady("volume-test", "kube-system", config.KubeconfigFile)
54+
}, "20s", "5s").Should(BeTrue())
55+
})
56+
})
57+
58+
Context("Verify Binaries and Images", func() {
59+
It("has valid bundled binaries", func() {
60+
for _, server := range config.Servers {
61+
Expect(tester.VerifyValidVersion(server.Name, "kubectl")).To(Succeed())
62+
Expect(tester.VerifyValidVersion(server.Name, "ctr")).To(Succeed())
63+
Expect(tester.VerifyValidVersion(server.Name, "crictl")).To(Succeed())
64+
}
65+
})
66+
It("has valid airgap images", func() {
67+
Expect(config).To(Not(BeNil()))
68+
err := VerifyAirgapImages(config)
69+
Expect(err).NotTo(HaveOccurred())
70+
})
71+
})
72+
})
73+
74+
var failed bool
75+
var _ = AfterEach(func() {
76+
failed = failed || CurrentSpecReport().Failed()
77+
})
78+
79+
var _ = AfterSuite(func() {
80+
if config != nil && !failed {
81+
config.Cleanup()
82+
}
83+
})
84+
85+
// VerifyAirgapImages checks for changes in the airgap image list
86+
func VerifyAirgapImages(config *tester.TestConfig) error {
87+
// This file is generated during the build packaging step
88+
const airgapImageList = "../../../scripts/airgap/image-list.txt"
89+
90+
// Use a map to automatically handle duplicates
91+
imageSet := make(map[string]struct{})
92+
93+
// Collect all images from nodes
94+
for _, node := range config.GetNodeNames() {
95+
cmd := fmt.Sprintf("docker exec %s crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)'", node)
96+
output, err := tester.RunCommand(cmd)
97+
Expect(err).NotTo(HaveOccurred(), "failed to execute crictl and jq: %v", err)
98+
99+
for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") {
100+
if line != "" {
101+
imageSet[line] = struct{}{}
102+
}
103+
}
104+
}
105+
106+
// Convert map keys to slice
107+
uniqueImages := make([]string, 0, len(imageSet))
108+
for image := range imageSet {
109+
uniqueImages = append(uniqueImages, image)
110+
}
111+
112+
existing, err := os.ReadFile(airgapImageList)
113+
if err != nil && !os.IsNotExist(err) {
114+
return fmt.Errorf("failed to read airgap list file: %v", err)
115+
}
116+
117+
// Sorting doesn't matter with ConsistOf
118+
existingImages := strings.Split(strings.TrimSpace(string(existing)), "\n")
119+
Expect(existingImages).To(ConsistOf(uniqueImages))
120+
return nil
121+
}

Diff for: ‎tests/docker/bootstraptoken/bootstraptoken_test.go

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"strings"
6+
"testing"
7+
8+
tester "github.com/k3s-io/k3s/tests/docker"
9+
. "github.com/onsi/ginkgo/v2"
10+
. "github.com/onsi/gomega"
11+
)
12+
13+
var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
14+
var config *tester.TestConfig
15+
16+
func Test_DockerBootstrapToken(t *testing.T) {
17+
flag.Parse()
18+
RegisterFailHandler(Fail)
19+
RunSpecs(t, "BoostrapToken Docker Test Suite")
20+
}
21+
22+
var _ = Describe("Boostrap Token Tests", Ordered, func() {
23+
24+
Context("Setup Cluster", func() {
25+
It("should provision servers", func() {
26+
var err error
27+
config, err = tester.NewTestConfig(*k3sImage)
28+
Expect(err).NotTo(HaveOccurred())
29+
Expect(config.ProvisionServers(1)).To(Succeed())
30+
Eventually(func() error {
31+
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
32+
}, "60s", "5s").Should(Succeed())
33+
})
34+
})
35+
36+
Context("Add Agent with Bootstrap token", func() {
37+
var newSecret string
38+
It("creates a bootstrap token", func() {
39+
var err error
40+
newSecret, err = tester.RunCmdOnDocker(config.Servers[0].Name, "k3s token create --ttl=5m --description=Test")
41+
Expect(err).NotTo(HaveOccurred())
42+
Expect(newSecret).NotTo(BeEmpty())
43+
})
44+
It("joins the agent with the new tokens", func() {
45+
newSecret = strings.ReplaceAll(newSecret, "\n", "")
46+
config.Secret = newSecret
47+
Expect(config.ProvisionAgents(1)).To(Succeed())
48+
Eventually(func(g Gomega) {
49+
nodes, err := tester.ParseNodes(config.KubeconfigFile)
50+
g.Expect(err).NotTo(HaveOccurred())
51+
g.Expect(nodes).To(HaveLen(2))
52+
g.Expect(tester.NodesReady(config.KubeconfigFile)).To(Succeed())
53+
}, "40s", "5s").Should(Succeed())
54+
})
55+
})
56+
})
57+
58+
var failed bool
59+
var _ = AfterEach(func() {
60+
failed = failed || CurrentSpecReport().Failed()
61+
})
62+
63+
var _ = AfterSuite(func() {
64+
if config != nil && !failed {
65+
config.Cleanup()
66+
}
67+
})

0 commit comments

Comments
 (0)