New test tooling

.github/workflows/test-canary.yml

@@ -112,5 +112,5 @@ jobs:
           ctrdVersion: ${{ env.CONTAINERD_VERSION }}
         run: powershell hack/configure-windows-ci.ps1
       - name: "Run integration tests"
-        # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
         run: go test -p 1 -v ./cmd/nerdctl/...

.github/workflows/test-kube.yml

@@ -22,7 +22,7 @@ jobs:
         with:
           fetch-depth: 1
       - name: "Run Kubernetes integration tests"
-        # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
         run: |
           ./hack/build-integration-kubernetes.sh
           sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test -p 1 ./cmd/nerdctl/... -test.only-kubernetes'

.github/workflows/test.yml

@@ -290,15 +290,15 @@ jobs:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+          # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
           command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
       - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+          # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
           command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
 
   test-integration-windows:
@@ -332,7 +332,7 @@ jobs:
         run: powershell hack/configure-windows-ci.ps1
       # TODO: Run unit tests
       - name: "Run integration tests"
-        # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
         run: go test -p 1 -v ./cmd/nerdctl/...
 
   test-integration-freebsd:

README.md

@@ -253,7 +253,7 @@ Using `go install github.com/containerd/nerdctl/v2/cmd/nerdctl` is possible, but
 
 ### Testing
 
-See [testing nerdctl](docs/dev/testing.md).
+See [testing nerdctl](docs/testing/README.md).
 
 ### Contributing to nerdctl
 

cmd/nerdctl/main_linux_test.go

@@ -20,17 +20,39 @@ import (
 	"testing"
 
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/test"
 )
 
 // TestIssue108 tests https://github.com/containerd/nerdctl/issues/108
 // ("`nerdctl run --net=host -it` fails while `nerdctl run -it --net=host` works")
 func TestIssue108(t *testing.T) {
-	base := testutil.NewBase(t)
-	// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
-	// unbuffer(1) can be installed with `apt-get install expect`.
-	unbuffer := []string{"unbuffer"}
-	base.CmdWithHelper(unbuffer, "run", "-it", "--rm", "--net=host", testutil.AlpineImage,
-		"echo", "this was always working").AssertOK()
-	base.CmdWithHelper(unbuffer, "run", "--rm", "--net=host", "-it", testutil.AlpineImage,
-		"echo", "this was not working due to issue #108").AssertOK()
+	nerdtest.Setup()
+
+	testGroup := &test.Group{
+		{
+			Description: "-it --net=host",
+			Require:     test.Binary("unbuffer"),
+			Command: func(data test.Data, helpers test.Helpers) test.Command {
+				return helpers.
+					Command("run", "-it", "--rm", "--net=host", testutil.AlpineImage, "echo", "this was always working").
+					WithWrapper("unbuffer")
+			},
+			// Note: unbuffer will merge stdout and stderr, preventing exact match here
+			Expected: test.Expects(0, nil, test.Contains("this was always working")),
+		},
+		{
+			Description: "--net=host -it",
+			Require:     test.Binary("unbuffer"),
+			Command: func(data test.Data, helpers test.Helpers) test.Command {
+				return helpers.
+					Command("run", "--rm", "--net=host", "-it", testutil.AlpineImage, "echo", "this was not working due to issue #108").
+					WithWrapper("unbuffer")
+			},
+			// Note: unbuffer will merge stdout and stderr, preventing exact match here
+			Expected: test.Expects(0, nil, test.Contains("this was not working due to issue #108")),
+		},
+	}
+
+	testGroup.Run(t)
 }

cmd/nerdctl/main_test.go

@@ -17,15 +17,14 @@
 package main
 
 import (
-	"os"
-	"path/filepath"
+	"errors"
 	"testing"
 
-	"gotest.tools/v3/assert"
-
 	"github.com/containerd/containerd/v2/defaults"
 
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/test"
 )
 
 func TestMain(m *testing.M) {
@@ -34,56 +33,100 @@ func TestMain(m *testing.M) {
 
 // TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487
 func TestUnknownCommand(t *testing.T) {
-	t.Parallel()
-	base := testutil.NewBase(t)
-	base.Cmd("non-existent-command").AssertFail()
-	base.Cmd("non-existent-command", "info").AssertFail()
-	base.Cmd("system", "non-existent-command").AssertFail()
-	base.Cmd("system", "non-existent-command", "info").AssertFail()
-	base.Cmd("system").AssertOK() // show help without error
-	base.Cmd("system", "info").AssertOutContains("Kernel Version:")
-	base.Cmd("info").AssertOutContains("Kernel Version:")
+	nerdtest.Setup()
+
+	var unknownSubCommand = errors.New("unknown subcommand")
+
+	testGroup := &test.Group{
+		{
+			Description: "non-existent-command",
+			Command:     test.RunCommand("non-existent-command"),
+			Expected:    test.Expects(1, []error{unknownSubCommand}, nil),
+		},
+		{
+			Description: "non-existent-command info",
+			Command:     test.RunCommand("non-existent-command", "info"),
+			Expected:    test.Expects(1, []error{unknownSubCommand}, nil),
+		},
+		{
+			Description: "system non-existent-command",
+			Command:     test.RunCommand("system", "non-existent-command"),
+			Expected:    test.Expects(1, []error{unknownSubCommand}, nil),
+		},
+		{
+			Description: "system non-existent-command info",
+			Command:     test.RunCommand("system", "non-existent-command", "info"),
+			Expected:    test.Expects(1, []error{unknownSubCommand}, nil),
+		},
+		{
+			Description: "system",
+			Command:     test.RunCommand("system"),
+			Expected:    test.Expects(0, nil, nil),
+		},
+		{
+			Description: "system info",
+			Command:     test.RunCommand("system", "info"),
+			Expected:    test.Expects(0, nil, test.Contains("Kernel Version:")),
+		},
+		{
+			Description: "info",
+			Command:     test.RunCommand("info"),
+			Expected:    test.Expects(0, nil, test.Contains("Kernel Version:")),
+		},
+	}
+
+	testGroup.Run(t)
 }
 
-// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default].
+// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default] and broken config rejection
 func TestNerdctlConfig(t *testing.T) {
-	testutil.DockerIncompatible(t)
-	t.Parallel()
-	tomlPath := filepath.Join(t.TempDir(), "nerdctl.toml")
-	err := os.WriteFile(tomlPath, []byte(`
-snapshotter = "dummy-snapshotter-via-toml"
-`), 0400)
-	assert.NilError(t, err)
-	base := testutil.NewBase(t)
-
-	// [Default]
-	base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly(defaults.DefaultSnapshotter + "\n")
-
-	// [TOML, Default]
-	base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath)
-	base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-toml\n")
-
-	// [CLI, TOML, Default]
-	base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n")
-
-	// [Env, TOML, Default]
-	base.Env = append(base.Env, "CONTAINERD_SNAPSHOTTER=dummy-snapshotter-via-env")
-	base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-env\n")
-
-	// [CLI, Env, TOML, Default]
-	base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n")
-}
-
-func TestNerdctlConfigBad(t *testing.T) {
-	testutil.DockerIncompatible(t)
-	t.Parallel()
-	tomlPath := filepath.Join(t.TempDir(), "config.toml")
-	err := os.WriteFile(tomlPath, []byte(`
-# containerd config, not nerdctl config
-version = 2
-`), 0400)
-	assert.NilError(t, err)
-	base := testutil.NewBase(t)
-	base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath)
-	base.Cmd("info").AssertFail()
+	nerdtest.Setup()
+
+	tc := &test.Case{
+		Description: "Nerdctl configuration",
+		// Docker does not support nerdctl.toml obviously
+		Require: test.Not(nerdtest.Docker),
+		SubTests: []*test.Case{
+			{
+				Description: "Default",
+				Command:     test.RunCommand("info", "-f", "{{.Driver}}"),
+				Expected:    test.Expects(0, nil, test.Equals(defaults.DefaultSnapshotter+"\n")),
+			},
+			{
+				Description: "TOML > Default",
+				Command:     test.RunCommand("info", "-f", "{{.Driver}}"),
+				Expected:    test.Expects(0, nil, test.Equals("dummy-snapshotter-via-toml\n")),
+				Data:        test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+			},
+			{
+				Description: "Cli > TOML > Default",
+				Command:     test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"),
+				Expected:    test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")),
+				Data:        test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+			},
+			{
+				Description: "Env > TOML > Default",
+				Command:     test.RunCommand("info", "-f", "{{.Driver}}"),
+				Env:         map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"},
+				Expected:    test.Expects(0, nil, test.Equals("dummy-snapshotter-via-env\n")),
+				Data:        test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+			},
+			{
+				Description: "Cli > Env > TOML > Default",
+				Command:     test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"),
+				Env:         map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"},
+				Expected:    test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")),
+				Data:        test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+			},
+			{
+				Description: "Broken config",
+				Command:     test.RunCommand("info"),
+				Expected:    test.Expects(1, []error{errors.New("failed to load nerdctl config")}, nil),
+				Data: test.WithConfig(nerdtest.NerdctlToml, `# containerd config, not nerdctl config
+version = 2`),
+			},
+		},
+	}
+
+	tc.Run(t)
 }