diff --git a/.github/workflows/test-canary.yml b/.github/workflows/test-canary.yml index 7c385edb236..d6ea18306fc 100644 --- a/.github/workflows/test-canary.yml +++ b/.github/workflows/test-canary.yml @@ -112,5 +112,5 @@ jobs: ctrdVersion: ${{ env.CONTAINERD_VERSION }} run: powershell hack/configure-windows-ci.ps1 - name: "Run integration tests" - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: go test -p 1 -v ./cmd/nerdctl/... diff --git a/.github/workflows/test-kube.yml b/.github/workflows/test-kube.yml index cb487898173..f42bb7945a0 100644 --- a/.github/workflows/test-kube.yml +++ b/.github/workflows/test-kube.yml @@ -22,7 +22,7 @@ jobs: with: fetch-depth: 1 - name: "Run Kubernetes integration tests" - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: | ./hack/build-integration-kubernetes.sh sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test -p 1 ./cmd/nerdctl/... -test.only-kubernetes' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f00e7c85a8d..1501b0fcebf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -290,7 +290,7 @@ jobs: timeout_minutes: 30 max_attempts: 2 retry_on: error - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon - name: "Ensure that the IPv6 integration test suite is compatible with Docker" uses: nick-fields/retry@v3 @@ -298,7 +298,7 @@ jobs: timeout_minutes: 30 max_attempts: 2 retry_on: error - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6 test-integration-windows: @@ -332,7 +332,7 @@ jobs: run: powershell hack/configure-windows-ci.ps1 # TODO: Run unit tests - name: "Run integration tests" - # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization + # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization run: go test -p 1 -v ./cmd/nerdctl/... test-integration-freebsd: diff --git a/README.md b/README.md index ef85dbd54f5..8076cd67446 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ Using `go install github.com/containerd/nerdctl/v2/cmd/nerdctl` is possible, but ### Testing -See [testing nerdctl](docs/dev/testing.md). +See [testing nerdctl](docs/testing/README.md). ### Contributing to nerdctl diff --git a/cmd/nerdctl/main_linux_test.go b/cmd/nerdctl/main_linux_test.go index a0e82861a84..274604e27b9 100644 --- a/cmd/nerdctl/main_linux_test.go +++ b/cmd/nerdctl/main_linux_test.go @@ -20,17 +20,39 @@ import ( "testing" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) // TestIssue108 tests https://github.com/containerd/nerdctl/issues/108 // ("`nerdctl run --net=host -it` fails while `nerdctl run -it --net=host` works") func TestIssue108(t *testing.T) { - base := testutil.NewBase(t) - // unbuffer(1) emulates tty, which is required by `nerdctl run -t`. - // unbuffer(1) can be installed with `apt-get install expect`. - unbuffer := []string{"unbuffer"} - base.CmdWithHelper(unbuffer, "run", "-it", "--rm", "--net=host", testutil.AlpineImage, - "echo", "this was always working").AssertOK() - base.CmdWithHelper(unbuffer, "run", "--rm", "--net=host", "-it", testutil.AlpineImage, - "echo", "this was not working due to issue #108").AssertOK() + nerdtest.Setup() + + testGroup := &test.Group{ + { + Description: "-it --net=host", + Require: test.Binary("unbuffer"), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers. + Command("run", "-it", "--rm", "--net=host", testutil.AlpineImage, "echo", "this was always working"). + WithWrapper("unbuffer") + }, + // Note: unbuffer will merge stdout and stderr, preventing exact match here + Expected: test.Expects(0, nil, test.Contains("this was always working")), + }, + { + Description: "--net=host -it", + Require: test.Binary("unbuffer"), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers. + Command("run", "--rm", "--net=host", "-it", testutil.AlpineImage, "echo", "this was not working due to issue #108"). + WithWrapper("unbuffer") + }, + // Note: unbuffer will merge stdout and stderr, preventing exact match here + Expected: test.Expects(0, nil, test.Contains("this was not working due to issue #108")), + }, + } + + testGroup.Run(t) } diff --git a/cmd/nerdctl/main_test.go b/cmd/nerdctl/main_test.go index a450c963bf5..c1e3caf94fd 100644 --- a/cmd/nerdctl/main_test.go +++ b/cmd/nerdctl/main_test.go @@ -17,15 +17,14 @@ package main import ( - "os" - "path/filepath" + "errors" "testing" - "gotest.tools/v3/assert" - "github.com/containerd/containerd/v2/defaults" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestMain(m *testing.M) { @@ -34,56 +33,100 @@ func TestMain(m *testing.M) { // TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487 func TestUnknownCommand(t *testing.T) { - t.Parallel() - base := testutil.NewBase(t) - base.Cmd("non-existent-command").AssertFail() - base.Cmd("non-existent-command", "info").AssertFail() - base.Cmd("system", "non-existent-command").AssertFail() - base.Cmd("system", "non-existent-command", "info").AssertFail() - base.Cmd("system").AssertOK() // show help without error - base.Cmd("system", "info").AssertOutContains("Kernel Version:") - base.Cmd("info").AssertOutContains("Kernel Version:") + nerdtest.Setup() + + var unknownSubCommand = errors.New("unknown subcommand") + + testGroup := &test.Group{ + { + Description: "non-existent-command", + Command: test.RunCommand("non-existent-command"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "non-existent-command info", + Command: test.RunCommand("non-existent-command", "info"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "system non-existent-command", + Command: test.RunCommand("system", "non-existent-command"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "system non-existent-command info", + Command: test.RunCommand("system", "non-existent-command", "info"), + Expected: test.Expects(1, []error{unknownSubCommand}, nil), + }, + { + Description: "system", + Command: test.RunCommand("system"), + Expected: test.Expects(0, nil, nil), + }, + { + Description: "system info", + Command: test.RunCommand("system", "info"), + Expected: test.Expects(0, nil, test.Contains("Kernel Version:")), + }, + { + Description: "info", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, test.Contains("Kernel Version:")), + }, + } + + testGroup.Run(t) } -// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default]. +// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default] and broken config rejection func TestNerdctlConfig(t *testing.T) { - testutil.DockerIncompatible(t) - t.Parallel() - tomlPath := filepath.Join(t.TempDir(), "nerdctl.toml") - err := os.WriteFile(tomlPath, []byte(` -snapshotter = "dummy-snapshotter-via-toml" -`), 0400) - assert.NilError(t, err) - base := testutil.NewBase(t) - - // [Default] - base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly(defaults.DefaultSnapshotter + "\n") - - // [TOML, Default] - base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath) - base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-toml\n") - - // [CLI, TOML, Default] - base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n") - - // [Env, TOML, Default] - base.Env = append(base.Env, "CONTAINERD_SNAPSHOTTER=dummy-snapshotter-via-env") - base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-env\n") - - // [CLI, Env, TOML, Default] - base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n") -} - -func TestNerdctlConfigBad(t *testing.T) { - testutil.DockerIncompatible(t) - t.Parallel() - tomlPath := filepath.Join(t.TempDir(), "config.toml") - err := os.WriteFile(tomlPath, []byte(` -# containerd config, not nerdctl config -version = 2 -`), 0400) - assert.NilError(t, err) - base := testutil.NewBase(t) - base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath) - base.Cmd("info").AssertFail() + nerdtest.Setup() + + tc := &test.Case{ + Description: "Nerdctl configuration", + // Docker does not support nerdctl.toml obviously + Require: test.Not(nerdtest.Docker), + SubTests: []*test.Case{ + { + Description: "Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}"), + Expected: test.Expects(0, nil, test.Equals(defaults.DefaultSnapshotter+"\n")), + }, + { + Description: "TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}"), + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-toml\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Cli > TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"), + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Env > TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}"), + Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"}, + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-env\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Cli > Env > TOML > Default", + Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"), + Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"}, + Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")), + Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`), + }, + { + Description: "Broken config", + Command: test.RunCommand("info"), + Expected: test.Expects(1, []error{errors.New("failed to load nerdctl config")}, nil), + Data: test.WithConfig(nerdtest.NerdctlToml, `# containerd config, not nerdctl config +version = 2`), + }, + }, + } + + tc.Run(t) } diff --git a/cmd/nerdctl/main_test_test.go b/cmd/nerdctl/main_test_test.go new file mode 100644 index 00000000000..a515df48536 --- /dev/null +++ b/cmd/nerdctl/main_test_test.go @@ -0,0 +1,147 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "errors" + "log" + "testing" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +// TestTest is testing the test tooling itself +func TestTest(t *testing.T) { + nerdtest.Setup() + + tg := &test.Group{ + { + Description: "failure", + Command: test.RunCommand("undefinedcommand"), + Expected: test.Expects(1, nil, nil), + }, + { + Description: "success", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, nil), + }, + { + Description: "failure with single error testing", + Command: test.RunCommand("undefinedcommand"), + Expected: test.Expects(1, []error{errors.New("unknown subcommand")}, nil), + }, + { + Description: "success with contains output testing", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, test.Contains("Kernel")), + }, + { + Description: "success with negative output testing", + Command: test.RunCommand("info"), + Expected: test.Expects(0, nil, test.DoesNotContain("foobar")), + }, + // Note that docker annoyingly returns 125 in a few conditions like this + { + Description: "failure with multiple error testing", + Command: test.RunCommand("-fail"), + Expected: test.Expects(-1, []error{errors.New("unknown"), errors.New("shorthand")}, nil), + }, + { + Description: "success with exact output testing", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.CustomCommand("echo", "foobar") + }, + Expected: test.Expects(0, nil, test.Equals("foobar\n")), + }, + { + Description: "data propagation", + Data: test.WithData("status", "uninitialized"), + Setup: func(data test.Data, helpers test.Helpers) { + data.Set("status", data.Get("status")+"-setup") + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + cmd := helpers.CustomCommand("printf", data.Get("status")) + data.Set("status", data.Get("status")+"-command") + return cmd + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + if data.Get("first-run") == "" { + data.Set("first-run", "first cleanup") + return + } + if data.Get("status") != "uninitialized-setup-command" { + log.Fatalf("unexpected status label %q", data.Get("status")) + } + data.Set("status", data.Get("status")+"-cleanup") + }, + SubTests: []*test.Case{ + { + Description: "Subtest data propagation", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.CustomCommand("printf", data.Get("status")) + }, + Expected: test.Expects(0, nil, test.Equals("uninitialized-setup-command")), + }, + }, + Expected: test.Expects(0, nil, test.Equals("uninitialized-setup")), + }, + { + Description: "env propagation and isolation", + Env: map[string]string{ + "GLOBAL_ENV": "in this test", + }, + Setup: func(data test.Data, helpers test.Helpers) { + cmd := helpers.CustomCommand("sh", "-c", "--", "printf \"$GLOBAL_ENV\"") + cmd.Run(&test.Expected{ + Output: test.Equals("in this test"), + }) + cmd.WithEnv(map[string]string{ + "GLOBAL_ENV": "overridden in setup", + }) + cmd.Run(&test.Expected{ + Output: test.Equals("overridden in setup"), + }) + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + cmd := helpers.CustomCommand("sh", "-c", "--", "printf \"$GLOBAL_ENV\"") + cmd.Run(&test.Expected{ + Output: test.Equals("in this test"), + }) + cmd.WithEnv(map[string]string{ + "GLOBAL_ENV": "overridden in command", + }) + return cmd + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + cmd := helpers.CustomCommand("sh", "-c", "--", "printf \"$GLOBAL_ENV\"") + cmd.Run(&test.Expected{ + Output: test.Equals("in this test"), + }) + cmd.WithEnv(map[string]string{ + "GLOBAL_ENV": "overridden in cleanup", + }) + cmd.Run(&test.Expected{ + Output: test.Equals("overridden in cleanup"), + }) + }, + Expected: test.Expects(0, nil, test.Equals("overridden in command")), + }, + } + + tg.Run(t) +} diff --git a/cmd/nerdctl/volume/volume_create_test.go b/cmd/nerdctl/volume/volume_create_test.go index a9126f704f7..767f7ac12be 100644 --- a/cmd/nerdctl/volume/volume_create_test.go +++ b/cmd/nerdctl/volume/volume_create_test.go @@ -17,155 +17,97 @@ package volume import ( + "errors" "testing" - "gotest.tools/v3/icmd" - "github.com/containerd/errdefs" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestVolumeCreate(t *testing.T) { - t.Parallel() - - base := testutil.NewBase(t) + nerdtest.Setup() - malformed := errdefs.ErrInvalidArgument.Error() - atMost := "at most 1 arg" - exitCodeVariant := 1 - if base.Target == testutil.Docker { - malformed = "invalid" - exitCodeVariant = 125 - } - - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ + tg := &test.Group{ { - description: "arg missing should create anonymous volume", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - } - }, + Description: "arg missing should create anonymous volume", + Command: test.RunCommand("volume", "create"), + Expected: test.Expects(0, nil, nil), }, { - description: "invalid identifier should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", "∞") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: malformed, - } - }, + Description: "invalid identifier should fail", + Command: test.RunCommand("volume", "create", "∞"), + Expected: test.Expects(1, []error{errdefs.ErrInvalidArgument}, nil), }, { - description: "too many args should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", "too", "many") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: atMost, - } - }, + Description: "too many args should fail", + Command: test.RunCommand("volume", "create", "too", "many"), + Expected: test.Expects(1, []error{errors.New("at most 1 arg")}, nil), }, { - description: "success", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", tID) + Description: "success", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, { - description: "success with labels", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "create", "--label", "foo1=baz1", "--label", "foo2=baz2", tID) + Description: "success with labels", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", "--label", "foo1=baz1", "--label", "foo2=baz2", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, { - description: "invalid labels", - command: func(tID string) *testutil.Cmd { + Description: "invalid labels", + Command: func(data test.Data, helpers test.Helpers) test.Command { // See https://github.com/containerd/nerdctl/issues/3126 - return base.Cmd("volume", "create", "--label", "a", "--label", "", tID) + return helpers.Command("volume", "create", "--label", "a", "--label", "", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: exitCodeVariant, - Err: malformed, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + // NOTE: docker returns 125 on this + ExitCode: -1, + Errors: []error{errdefs.ErrInvalidArgument}, } }, }, { - description: "creating already existing volume should succeed", - command: func(tID string) *testutil.Cmd { - base.Cmd("volume", "create", tID).AssertOK() - return base.Cmd("volume", "create", tID) + Description: "creating already existing volume should succeed", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - tt.Parallel() - - tID := testutil.Identifier(tt) - - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - cmd := currentTest.command(tID) - cmd.Assert(currentTest.expected(tID)) - }) - } + tg.Run(t) } diff --git a/cmd/nerdctl/volume/volume_inspect_test.go b/cmd/nerdctl/volume/volume_inspect_test.go index 0d4ce4e6d05..edee98be906 100644 --- a/cmd/nerdctl/volume/volume_inspect_test.go +++ b/cmd/nerdctl/volume/volume_inspect_test.go @@ -19,266 +19,185 @@ package volume import ( "crypto/rand" "encoding/json" + "errors" "fmt" "os" "path/filepath" - "strings" "testing" "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" "github.com/containerd/errdefs" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/native" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) -func createFileWithSize(base *testutil.Base, vol string, size int64) { - v := base.InspectVolume(vol) +func createFileWithSize(mountPoint string, size int64) error { token := make([]byte, size) _, _ = rand.Read(token) - err := os.WriteFile(filepath.Join(v.Mountpoint, "test-file"), token, 0644) - assert.NilError(base.T, err) + err := os.WriteFile(filepath.Join(mountPoint, "test-file"), token, 0644) + return err } func TestVolumeInspect(t *testing.T) { - t.Parallel() - - base := testutil.NewBase(t) - tID := testutil.Identifier(t) + nerdtest.Setup() var size int64 = 1028 - malformed := errdefs.ErrInvalidArgument.Error() - notFound := errdefs.ErrNotFound.Error() - requireArg := "requires at least 1 arg" - if base.Target == testutil.Docker { - malformed = "no such volume" - notFound = "no such volume" - } - - tearUp := func(t *testing.T) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("volume", "create", "--label", "foo=fooval", "--label", "bar=barval", tID+"-second").AssertOK() - - // Obviously note here that if inspect code gets totally hosed, this entire suite will - // probably fail right here on the tearUp instead of actually testing something - createFileWithSize(base, tID, size) - } - - tearDown := func(t *testing.T) { - base.Cmd("volume", "rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID+"-second").Run() - } - - tearDown(t) - t.Cleanup(func() { - tearDown(t) - }) - tearUp(t) - - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ - { - description: "arg missing should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: requireArg, - } - }, - }, - { - description: "invalid identifier should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "∞") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: malformed, - } - }, - }, - { - description: "non existent volume should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "doesnotexist") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: notFound, - } - }, - }, - { - description: "success", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", tID) - }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) - assert.Assert(t, dc[0].Name == tID, fmt.Sprintf("expected name to be %q (was %q)", tID, dc[0].Name)) - assert.Assert(t, dc[0].Labels == nil, fmt.Sprintf("expected labels to be nil and were %v", dc[0].Labels)) - }, - }, - { - description: "inspect labels", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", tID+"-second") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - - labels := *dc[0].Labels - assert.Assert(t, len(labels) == 2, fmt.Sprintf("two results, not %d", len(labels))) - assert.Assert(t, labels["foo"] == "fooval", fmt.Sprintf("label foo should be fooval, not %s", labels["foo"])) - assert.Assert(t, labels["bar"] == "barval", fmt.Sprintf("label bar should be barval, not %s", labels["bar"])) - }, + tc := &test.Case{ + Description: "Volume inspect", + Setup: func(data test.Data, helpers test.Helpers) { + data.Set("volprefix", data.Identifier()) + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("volume", "create", "--label", "foo=fooval", "--label", "bar=barval", data.Identifier()+"-second") + // Obviously note here that if inspect code gets totally hosed, this entire suite will + // probably fail right here on the Setup instead of actually testing something + vol := nerdtest.InspectVolume(helpers, data.Identifier()) + err := createFileWithSize(vol.Mountpoint, size) + assert.NilError(t, err, "File creation failed") }, - { - description: "inspect size", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "--size", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, dc[0].Size == size, fmt.Sprintf("expected size to be %d (was %d)", size, dc[0].Size)) - }, - dockerIncompatible: true, - }, - { - description: "multi success", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", tID, tID+"-second") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(dc) == 2, fmt.Sprintf("two results, not %d", len(dc))) - assert.Assert(t, dc[0].Name == tID, fmt.Sprintf("expected name to be %q (was %q)", tID, dc[0].Name)) - assert.Assert(t, dc[1].Name == tID+"-second", fmt.Sprintf("expected name to be %q (was %q)", tID+"-second", dc[1].Name)) - }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-second") }, - { - description: "part success multi", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "invalid∞", "nonexistent", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Out: tID, - Err: notFound, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, malformed)) - var dc []native.Volume - if err := json.Unmarshal([]byte(stdout), &dc); err != nil { - t.Fatal(err) - } - assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) - assert.Assert(t, dc[0].Name == tID, fmt.Sprintf("expected name to be %q (was %q)", tID, dc[0].Name)) - }, - }, - { - description: "multi failure", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "inspect", "invalid∞", "nonexistent") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - } - }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, malformed)) + SubTests: []*test.Case{ + { + Description: "arg missing should fail", + Command: test.RunCommand("volume", "inspect"), + Expected: test.Expects(1, []error{errors.New("requires at least 1 arg")}, nil), + }, + { + Description: "invalid identifier should fail", + Command: test.RunCommand("volume", "inspect", "∞"), + Expected: test.Expects(1, []error{errdefs.ErrInvalidArgument}, nil), + }, + { + Description: "non existent volume should fail", + Command: test.RunCommand("volume", "inspect", "doesnotexist"), + Expected: test.Expects(1, []error{errdefs.ErrNotFound}, nil), + }, + { + Description: "success", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("volprefix")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))+info) + assert.Assert(t, dc[0].Name == data.Get("volprefix"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix"), dc[0].Name)+info) + assert.Assert(t, dc[0].Labels == nil, fmt.Sprintf("expected labels to be nil and were %v", dc[0].Labels)+info) + }, + ), + } + }, + }, + { + Description: "inspect labels", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("volprefix")+"-second") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + labels := *dc[0].Labels + assert.Assert(t, len(labels) == 2, fmt.Sprintf("two results, not %d", len(labels))) + assert.Assert(t, labels["foo"] == "fooval", fmt.Sprintf("label foo should be fooval, not %s", labels["foo"])) + assert.Assert(t, labels["bar"] == "barval", fmt.Sprintf("label bar should be barval, not %s", labels["bar"])) + }, + ), + } + }, + }, + { + Description: "inspect size", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", "--size", data.Get("volprefix")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, dc[0].Size == size, fmt.Sprintf("expected size to be %d (was %d)", size, dc[0].Size)) + }, + ), + } + }, + }, + { + Description: "multi success", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("volprefix"), data.Get("volprefix")+"-second") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.Contains(data.Get("volprefix")), + test.Contains(data.Get("volprefix")+"-second"), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(dc) == 2, fmt.Sprintf("two results, not %d", len(dc))) + assert.Assert(t, dc[0].Name == data.Get("volprefix"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix"), dc[0].Name)) + assert.Assert(t, dc[1].Name == data.Get("volprefix")+"-second", fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix")+"-second", dc[1].Name)) + }, + ), + } + }, + }, + { + Description: "part success multi", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", "invalid∞", "nonexistent", data.Get("volprefix")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{errdefs.ErrNotFound, errdefs.ErrInvalidArgument}, + Output: test.All( + test.Contains(data.Get("volprefix")), + func(stdout string, info string, t *testing.T) { + var dc []native.Volume + if err := json.Unmarshal([]byte(stdout), &dc); err != nil { + t.Fatal(err) + } + assert.Assert(t, len(dc) == 1, fmt.Sprintf("one result, not %d", len(dc))) + assert.Assert(t, dc[0].Name == data.Get("volprefix"), fmt.Sprintf("expected name to be %q (was %q)", data.Get("volprefix"), dc[0].Name)) + }, + ), + } + }, + }, + { + Description: "multi failure", + Command: test.RunCommand("volume", "inspect", "invalid∞", "nonexistent"), + Expected: test.Expects(1, []error{errdefs.ErrNotFound, errdefs.ErrInvalidArgument}, nil), }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - tt.Parallel() - - // We use the main test tID here - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(tID) - res := cmd.Run() - cmd.Assert(currentTest.expected(tID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + tc.Run(t) } diff --git a/cmd/nerdctl/volume/volume_list_test.go b/cmd/nerdctl/volume/volume_list_test.go index 6195db2eb13..d0ad6d78463 100644 --- a/cmd/nerdctl/volume/volume_list_test.go +++ b/cmd/nerdctl/volume/volume_list_test.go @@ -17,464 +17,376 @@ package volume import ( - "errors" "fmt" "strings" "testing" + "gotest.tools/v3/assert" + "github.com/containerd/nerdctl/v2/pkg/tabutil" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) -func TestVolumeLs(t *testing.T) { - t.Parallel() - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - testutil.DockerIncompatible(t) - - var vol1, vol2, vol3 = tID + "vol-1", tID + "vol-2", tID + "empty" - - tearDown := func() { - base.Cmd("volume", "rm", "-f", vol1).Run() - base.Cmd("volume", "rm", "-f", vol2).Run() - base.Cmd("volume", "rm", "-f", vol3).Run() - } - - tearUp := func() { - base.Cmd("volume", "create", vol1).AssertOK() - base.Cmd("volume", "create", vol2).AssertOK() - base.Cmd("volume", "create", vol3).AssertOK() - createFileWithSize(base, vol1, 102400) - createFileWithSize(base, vol2, 204800) - } - - tearDown() - t.Cleanup(func() { - tearDown() - }) - tearUp() - - base.Cmd("volume", "ls", "--size").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 4 { - return errors.New("expected at least 4 lines") - } - volSizes := map[string]string{ - vol1: "100.0 KiB", - vol2: "200.0 KiB", - vol3: "0.0 B", - } - - var numMatches = 0 - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err - } - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - size, _ := tab.ReadRow(line, "SIZE") - expectSize, ok := volSizes[name] - if !ok { - continue - } - if size != expectSize { - return fmt.Errorf("expected size %s for volume %s, got %s", expectSize, name, size) - } - numMatches++ - } - if len(volSizes) != numMatches { - return fmt.Errorf("expected %d volumes, got: %d", len(volSizes), numMatches) - } - return nil - }) - -} - -func TestVolumeLsFilter(t *testing.T) { - t.Parallel() - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - - var vol1, vol2, vol3, vol4 = tID + "vol-1", tID + "vol-2", tID + "vol-3", tID + "vol-4" - var label1, label2, label3, label4 = tID + "=label-1", tID + "=label-2", tID + "=label-3", tID + "-group=label-4" - - tearDown := func() { - base.Cmd("volume", "rm", "-f", vol1).Run() - base.Cmd("volume", "rm", "-f", vol2).Run() - base.Cmd("volume", "rm", "-f", vol3).Run() - base.Cmd("volume", "rm", "-f", vol4).Run() - } - - tearUp := func() { - base.Cmd("volume", "create", "--label="+label1, "--label="+label4, vol1).AssertOK() - base.Cmd("volume", "create", "--label="+label2, "--label="+label4, vol2).AssertOK() - base.Cmd("volume", "create", "--label="+label3, vol3).AssertOK() - base.Cmd("volume", "create", vol4).AssertOK() - } - - tearDown() - t.Cleanup(func() { - tearDown() - }) - tearUp() - - testCases := []struct { - description string - command func(tID string) - }{ - { - description: "no filter", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet").AssertOutWithFunc(func(stdout string) error { +func TestVolumeLsSize(t *testing.T) { + nerdtest.Setup() + + tc := &test.Case{ + Description: "Volume ls --size", + Require: test.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()+"-1") + helpers.Ensure("volume", "create", data.Identifier()+"-2") + helpers.Ensure("volume", "create", data.Identifier()+"-empty") + vol1 := nerdtest.InspectVolume(helpers, data.Identifier()+"-1") + vol2 := nerdtest.InspectVolume(helpers, data.Identifier()+"-2") + + err := createFileWithSize(vol1.Mountpoint, 102400) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(vol2.Mountpoint, 204800) + assert.NilError(t, err, "File creation failed") + }, + Command: test.RunCommand("volume", "ls", "--size"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 4 { - return errors.New("expected at least 4 lines") - } - volNames := map[string]struct{}{ - vol1: {}, - vol2: {}, - vol3: {}, - vol4: {}, + assert.Assert(t, len(lines) >= 4, "expected at least 4 lines"+info) + volSizes := map[string]string{ + data.Identifier() + "-1": "100.0 KiB", + data.Identifier() + "-2": "200.0 KiB", + data.Identifier() + "-empty": "0.0 B", } var numMatches = 0 - for _, name := range lines { - _, ok := volNames[name] + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, info) + + for _, line := range lines { + name, _ := tab.ReadRow(line, "VOLUME NAME") + size, _ := tab.ReadRow(line, "SIZE") + expectSize, ok := volSizes[name] if !ok { continue } + assert.Assert(t, size == expectSize, fmt.Sprintf("expected size %s for volume %s, got %s", expectSize, name, size)+info) numMatches++ } - if len(volNames) != numMatches { - return fmt.Errorf("expected %d volumes, got: %d", len(volNames), numMatches) - } - return nil - }) - }, + assert.Assert(t, numMatches == len(volSizes), fmt.Sprintf("expected %d volumes, got: %d", len(volSizes), numMatches)+info) + }, + } }, - { - description: "label=" + tID, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+tID).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") - } - volNames := map[string]struct{}{ - vol1: {}, - vol2: {}, - vol3: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } - } - return nil - }) - }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-1") + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-2") + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-empty") }, - { - description: "label=" + label2, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+label2).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 1 { - return errors.New("expected at least 1 lines") - } - volNames := map[string]struct{}{ - vol2: {}, - } + } + + tc.Run(t) +} + +func TestVolumeLsFilter(t *testing.T) { + nerdtest.Setup() + + tc := &test.Case{ + Description: "Volume ls", + Setup: func(data test.Data, helpers test.Helpers) { + var vol1, vol2, vol3, vol4 = data.Identifier() + "-1", data.Identifier() + "-2", data.Identifier() + "-3", data.Identifier() + "-4" + var label1, label2, label3, label4 = data.Identifier() + "=label-1", data.Identifier() + "=label-2", data.Identifier() + "=label-3", data.Identifier() + "-group=label-4" + + helpers.Ensure("volume", "create", "--label="+label1, "--label="+label4, vol1) + helpers.Ensure("volume", "create", "--label="+label2, "--label="+label4, vol2) + helpers.Ensure("volume", "create", "--label="+label3, vol3) + helpers.Ensure("volume", "create", vol4) + + err := createFileWithSize(nerdtest.InspectVolume(helpers, vol1).Mountpoint, 409600) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(nerdtest.InspectVolume(helpers, vol2).Mountpoint, 1024000) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(nerdtest.InspectVolume(helpers, vol3).Mountpoint, 409600) + assert.NilError(t, err, "File creation failed") + err = createFileWithSize(nerdtest.InspectVolume(helpers, vol4).Mountpoint, 1024000) + assert.NilError(t, err, "File creation failed") + + data.Set("vol1", vol1) + data.Set("vol2", vol2) + data.Set("vol3", vol3) + data.Set("vol4", vol4) + data.Set("mainlabel", data.Identifier()) + data.Set("label1", label1) + data.Set("label2", label2) + data.Set("label3", label3) + data.Set("label4", label4) - for _, name := range lines { - if name == "" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } - } - return nil - }) - }, }, - { - description: "label=" + tID + "=", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+tID+"=").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) > 0 { - for _, name := range lines { - if name != "" { - return fmt.Errorf("unexpected volumes %d found", len(lines)) + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Get("vol1")) + helpers.Anyhow("volume", "rm", "-f", data.Get("vol2")) + helpers.Anyhow("volume", "rm", "-f", data.Get("vol3")) + helpers.Anyhow("volume", "rm", "-f", data.Get("vol4")) + }, + SubTests: []*test.Case{ + { + Description: "No filter", + Command: test.RunCommand("volume", "ls", "--quiet"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 4, "expected at least 4 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, + data.Get("vol3"): {}, + data.Get("vol4"): {}, } - } + var numMatches = 0 + for _, name := range lines { + _, ok := volNames[name] + if !ok { + continue + } + numMatches++ + } + assert.Assert(t, len(volNames) == numMatches, fmt.Sprintf("expected %d volumes, got: %d", len(volNames), numMatches)) + }, } - return nil - }) + }, }, - }, - { - description: "label=" + label1 + " label=" + label2, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+label1, "--filter", "label="+label2).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) > 0 { - for _, name := range lines { - if name != "" { - return fmt.Errorf("unexpected volumes %d found", len(lines)) + { + Description: "Retrieving label=mainlabel", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("mainlabel")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, + data.Get("vol3"): {}, } - } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) - + }, }, - }, - { - description: "label=" + tID + " label=" + label4, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "label="+tID, "--filter", "label="+label4).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 2 { - return errors.New("expected at least 2 lines") - } - volNames := map[string]struct{}{ - vol1: {}, - vol2: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + { + Description: "Retrieving label=mainlabel=label2", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("label2")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 1, "expected at least 1 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol2"): {}, + } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) + }, }, - }, - { - description: "name=" + vol1, - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "name="+vol1).AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 1 { - return errors.New("expected at least 1 lines") + { + Description: "Retrieving label=mainlabel=", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("mainlabel")+"=") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, strings.TrimSpace(stdout) == "", "expected no result"+info) + }, } - volNames := map[string]struct{}{ - vol1: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + }, + }, + { + Description: "Retrieving label=mainlabel=label1 and label=mainlabel=label2", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("label1"), "--filter", "label="+data.Get("label2")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, strings.TrimSpace(stdout) == "", "expected no result"+info) + }, } - return nil - }) + }, }, - }, - { - description: "name=vol-3", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "name=vol-3"). - AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 1 { - return errors.New("expected at least 1 lines") - } - volNames := map[string]struct{}{ - vol3: {}, - } - - for _, name := range lines { - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) + { + Description: "Retrieving label=mainlabel and label=grouplabel=label4", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "label="+data.Get("mainlabel"), "--filter", "label="+data.Get("label4")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 2, "expected at least 2 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, } - } - return nil - }) - }, - }, - { - description: "name=vol2 name=vol1", - command: func(tID string) { - base.Cmd("volume", "ls", "--quiet", "--filter", "name=vol2", "--filter", "name=vol1"). - AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) > 0 { for _, name := range lines { - if name != "" { - return fmt.Errorf("unexpected volumes %d found", len(lines)) - } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) } - } - return nil - }) - }, - }, - } - - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - tt.Parallel() - currentTest.command(tID) - }) - } - -} - -func TestVolumeLsFilterSize(t *testing.T) { - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - testutil.DockerIncompatible(t) - - var vol1, vol2, vol3, vol4 = tID + "volsize-1", tID + "volsize-2", tID + "volsize-3", tID + "volsize-4" - var label1, label2, label3, label4 = tID + "=label-1", tID + "=label-2", tID + "=label-3", tID + "-group=label-4" - - tearDown := func() { - base.Cmd("volume", "rm", "-f", vol1).Run() - base.Cmd("volume", "rm", "-f", vol2).Run() - base.Cmd("volume", "rm", "-f", vol3).Run() - base.Cmd("volume", "rm", "-f", vol4).Run() - } - - tearUp := func() { - base.Cmd("volume", "create", "--label="+label1, "--label="+label4, vol1).AssertOK() - base.Cmd("volume", "create", "--label="+label2, "--label="+label4, vol2).AssertOK() - base.Cmd("volume", "create", "--label="+label3, vol3).AssertOK() - base.Cmd("volume", "create", vol4).AssertOK() - - createFileWithSize(base, vol1, 409600) - createFileWithSize(base, vol2, 1024000) - createFileWithSize(base, vol3, 409600) - createFileWithSize(base, vol4, 1024000) - } - - tearDown() - t.Cleanup(func() { - tearDown() - }) - tearUp() - - testCases := []struct { - description string - command func(tID string) - }{ - { - description: "size=1024000", - command: func(tID string) { - base.Cmd("volume", "ls", "--size", "--filter", "size=1024000").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") + }, } - - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err - } - volNames := map[string]struct{}{ - vol2: {}, - vol4: {}, - } - - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - if name == "VOLUME NAME" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } - } - return nil - }) + }, }, - }, - { - description: "size>=1024000 size<=2048000", - command: func(tID string) { - base.Cmd("volume", "ls", "--size", "--filter", "size>=1024000", "--filter", "size<=2048000").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") - } - - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err - } - volNames := map[string]struct{}{ - vol2: {}, - vol4: {}, - } - - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - if name == "VOLUME NAME" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + { + Description: "Retrieving name=volume1", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "name="+data.Get("vol1")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 1, "expected at least 1 line"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) + }, }, - }, - { - description: "size>204800 size<1024000", - command: func(tID string) { - base.Cmd("volume", "ls", "--size", "--filter", "size>204800", "--filter", "size<1024000").AssertOutWithFunc(func(stdout string) error { - var lines = strings.Split(strings.TrimSpace(stdout), "\n") - if len(lines) < 3 { - return errors.New("expected at least 3 lines") + { + Description: "Retrieving name=volume1 and name=volume2", + // FIXME: https://github.com/containerd/nerdctl/issues/3452 + // Nerdctl filter behavior is broken + Require: nerdtest.Docker, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--quiet", "--filter", "name="+data.Get("vol1"), "--filter", "name="+data.Get("vol2")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 2, "expected at least 2 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol2"): {}, + } + for _, name := range lines { + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - - var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") - var err = tab.ParseHeader(lines[0]) - if err != nil { - return err + }, + }, + { + Description: "Retrieving size=1024000", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--size", "--filter", "size=1024000") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol2"): {}, + data.Get("vol4"): {}, + } + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, "Tab reader failed") + for _, line := range lines { + + name, _ := tab.ReadRow(line, "VOLUME NAME") + if name == "VOLUME NAME" { + continue + } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - volNames := map[string]struct{}{ - vol1: {}, - vol3: {}, + }, + }, + { + Description: "Retrieving size>=1024000 size<=2048000", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--size", "--filter", "size>=1024000", "--filter", "size<=2048000") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol2"): {}, + data.Get("vol4"): {}, + } + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, "Tab reader failed") + for _, line := range lines { + + name, _ := tab.ReadRow(line, "VOLUME NAME") + if name == "VOLUME NAME" { + continue + } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - - for _, line := range lines { - name, _ := tab.ReadRow(line, "VOLUME NAME") - if name == "VOLUME NAME" { - continue - } - _, ok := volNames[name] - if !ok { - return fmt.Errorf("unexpected volume %s found", name) - } + }, + }, + { + Description: "Retrieving size>204800 size<1024000", + Require: test.Not(nerdtest.Docker), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "ls", "--size", "--filter", "size>204800", "--filter", "size<1024000") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + var lines = strings.Split(strings.TrimSpace(stdout), "\n") + assert.Assert(t, len(lines) >= 3, "expected at least 3 lines"+info) + volNames := map[string]struct{}{ + data.Get("vol1"): {}, + data.Get("vol3"): {}, + } + var tab = tabutil.NewReader("VOLUME NAME\tDIRECTORY\tSIZE") + var err = tab.ParseHeader(lines[0]) + assert.NilError(t, err, "Tab reader failed") + for _, line := range lines { + + name, _ := tab.ReadRow(line, "VOLUME NAME") + if name == "VOLUME NAME" { + continue + } + _, ok := volNames[name] + assert.Assert(t, ok, fmt.Sprintf("unexpected volume %s found", name)+info) + } + }, } - return nil - }) + }, }, }, } - - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - tt.Parallel() - currentTest.command(tID) - }) - } + tc.Run(t) } diff --git a/cmd/nerdctl/volume/volume_namespace_test.go b/cmd/nerdctl/volume/volume_namespace_test.go index dcda29a3648..b20f64984dc 100644 --- a/cmd/nerdctl/volume/volume_namespace_test.go +++ b/cmd/nerdctl/volume/volume_namespace_test.go @@ -19,134 +19,78 @@ package volume import ( "testing" - "gotest.tools/v3/icmd" - "github.com/containerd/errdefs" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestVolumeNamespace(t *testing.T) { - testutil.DockerIncompatible(t) - - t.Parallel() - - base := testutil.NewBase(t) - tID := testutil.Identifier(t) - otherBase := testutil.NewBaseWithNamespace(t, tID+"-1") - thirdBase := testutil.NewBaseWithNamespace(t, tID+"-2") - - tearUp := func(t *testing.T) { - base.Cmd("volume", "create", tID).AssertOK() - } - - tearDown := func(t *testing.T) { - base.Cmd("volume", "rm", "-f", tID).Run() - otherBase.Cmd("namespace", "rm", "-f", tID+"-1").Run() - thirdBase.Cmd("namespace", "rm", "-f", tID+"-2").Run() - } - - tearDown(t) - t.Cleanup(func() { - tearDown(t) - }) - tearUp(t) - - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ - { - description: "inspect another namespace volume should fail", - command: func(tID string) *testutil.Cmd { - return otherBase.Cmd("volume", "inspect", tID) - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: errdefs.ErrNotFound.Error(), - } - }, + nerdtest.Setup() + + tg := &test.Case{ + Description: "Namespaces", + Require: test.Not(nerdtest.Docker), + Setup: func(data test.Data, helpers test.Helpers) { + data.Set("root_namespace", data.Identifier()) + data.Set("root_volume", data.Identifier()) + helpers.Ensure("--namespace", data.Identifier(), "volume", "create", data.Identifier()) }, - { - description: "remove another namespace volume should fail", - command: func(tID string) *testutil.Cmd { - return otherBase.Cmd("volume", "remove", tID) + SubTests: []*test.Case{ + { + Description: "inspect another namespace volume should fail", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "inspect", data.Get("root_volume")) + }, + Expected: test.Expects(1, []error{ + errdefs.ErrNotFound, + }, nil), }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: errdefs.ErrNotFound.Error(), - } + { + Description: "removing another namespace volume should fail", + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "remove", data.Get("root_volume")) + }, + Expected: test.Expects(1, []error{ + errdefs.ErrNotFound, + }, nil), }, - }, - { - description: "prune should leave other namespace untouched", - command: func(tID string) *testutil.Cmd { - return otherBase.Cmd("volume", "prune", "-a", "-f") - }, - tearDown: func(tID string) { - // Assert that the volume is here in the base namespace - // both before and after the prune command - base.Cmd("volume", "inspect", tID).AssertOK() - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - } - }, - }, - { - description: "create with namespace should work", - command: func(tID string) *testutil.Cmd { - return thirdBase.Cmd("volume", "create", tID) + { + Description: "prune should leave another namespace volume untouched", + NoParallel: true, + Command: test.RunCommand("volume", "prune", "-a", "-f"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.DoesNotContain(data.Get("root_volume")), + func(stdout string, info string, t *testing.T) { + helpers.Ensure("--namespace", data.Get("root_namespace"), "volume", "inspect", data.Get("root_volume")) + }, + ), + } + }, }, - tearDown: func(tID string) { - thirdBase.Cmd("volume", "remove", "-f", tID).Run() - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, - Out: tID, - } + { + Description: "create with the same name should work, then delete it", + NoParallel: true, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "create", data.Get("root_volume")) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", data.Get("root_volume")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: func(stdout string, info string, t *testing.T) { + helpers.Ensure("volume", "inspect", data.Get("root_volume")) + helpers.Ensure("volume", "rm", data.Get("root_volume")) + helpers.Ensure("--namespace", data.Get("root_namespace"), "volume", "inspect", data.Get("root_volume")) + }, + } + }, }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - tt.Parallel() - - // Note that here we are using the main test tID - // since we are testing against the volume created in it - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(tID) - res := cmd.Run() - cmd.Assert(currentTest.expected(tID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + tg.Run(t) } diff --git a/cmd/nerdctl/volume/volume_prune_linux_test.go b/cmd/nerdctl/volume/volume_prune_linux_test.go index 19632a73992..8898ad30ab4 100644 --- a/cmd/nerdctl/volume/volume_prune_linux_test.go +++ b/cmd/nerdctl/volume/volume_prune_linux_test.go @@ -20,130 +20,90 @@ import ( "strings" "testing" - "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" - "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) func TestVolumePrune(t *testing.T) { - // Volume pruning cannot be parallelized for Docker, since we need namespaces to do that in a way that does interact with other tests - if testutil.GetTarget() != testutil.Docker { - t.Parallel() - } + nerdtest.Setup() + + var setup = func(data test.Data, helpers test.Helpers) { + anonIDBusy := strings.TrimSpace(helpers.Capture("volume", "create")) + anonIDDangling := strings.TrimSpace(helpers.Capture("volume", "create")) + + namedBusy := data.Identifier() + "-busy" + namedDangling := data.Identifier() + "-free" - // FIXME: for an unknown reason, when testing ipv6, calling NewBaseWithNamespace per sub-test, in the tearDown/tearUp methods - // will actually panic the test (also happens with target=docker) - // Calling base here *first* so that it can skip NOW - does seem to workaround the problem - // If you have any idea how to properly do this, feel free to remove the following line and fix the underlying issue - testutil.NewBase(t) + helpers.Ensure("volume", "create", namedBusy) + helpers.Ensure("volume", "create", namedDangling) + helpers.Ensure("run", "--name", data.Identifier(), + "-v", namedBusy+":/whatever", + "-v", anonIDBusy+":/other", testutil.CommonImage) - subTearUp := func(tID string) { - base := testutil.NewBaseWithNamespace(t, tID) - res := base.Cmd("volume", "create").Run() - anonIDBusy := res.Stdout() - base.Cmd("volume", "create").Run() - base.Cmd("volume", "create", tID+"-busy").AssertOK() - base.Cmd("volume", "create", tID+"-free").AssertOK() - base.Cmd("run", "--name", tID, - "-v", tID+"-busy:/whatever", - "-v", anonIDBusy, testutil.CommonImage).AssertOK() + data.Set("anonIDBusy", anonIDBusy) + data.Set("anonIDDangling", anonIDDangling) + data.Set("namedBusy", namedBusy) + data.Set("namedDangling", namedDangling) } - subTearDown := func(tID string) { - base := testutil.NewBaseWithNamespace(t, tID) - base.Cmd("rm", "-f", tID).Run() - base.Cmd("namespace", "remove", "-f", tID).Run() + var cleanup = func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("rm", "-f", data.Get("anonIDBusy")) + helpers.Anyhow("rm", "-f", data.Get("anonIDDangling")) + helpers.Anyhow("rm", "-f", data.Get("namedBusy")) + helpers.Anyhow("rm", "-f", data.Get("namedDangling")) } - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ + // This set must be marked as private, since we cannot prune without interacting with other tests. + testGroup := &test.Group{ { - description: "prune anonymous only", - command: func(tID string) *testutil.Cmd { - base := testutil.NewBaseWithNamespace(t, tID) - return base.Cmd("volume", "prune", "-f") - }, - tearUp: subTearUp, - tearDown: subTearDown, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, + Description: "prune anonymous only", + Require: nerdtest.Private, + Command: test.RunCommand("volume", "prune", "-f"), + Setup: setup, + Cleanup: cleanup, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.DoesNotContain(data.Get("anonIDBusy")), + test.Contains(data.Get("anonIDDangling")), + test.DoesNotContain(data.Get("namedBusy")), + test.DoesNotContain(data.Get("namedDangling")), + func(stdout string, info string, t *testing.T) { + helpers.Ensure("volume", "inspect", data.Get("anonIDBusy")) + helpers.Fail("volume", "inspect", data.Get("anonIDDangling")) + helpers.Ensure("volume", "inspect", data.Get("namedBusy")) + helpers.Ensure("volume", "inspect", data.Get("namedDangling")) + }, + ), } }, - inspect: func(t *testing.T, stdout string, stderr string) { - tID := testutil.Identifier(t) - base := testutil.NewBaseWithNamespace(t, tID) - assert.Assert(base.T, !strings.Contains(stdout, tID+"-free")) - base.Cmd("volume", "inspect", tID+"-free").AssertOK() - assert.Assert(base.T, !strings.Contains(stdout, tID+"-busy")) - base.Cmd("volume", "inspect", tID+"-busy").AssertOK() - // TODO verify the anonymous volumes status - }, }, { - description: "prune all", - command: func(tID string) *testutil.Cmd { - base := testutil.NewBaseWithNamespace(t, tID) - return base.Cmd("volume", "prune", "-f", "--all") - }, - tearUp: subTearUp, - tearDown: subTearDown, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 0, + Description: "prune all", + Require: nerdtest.Private, + Command: test.RunCommand("volume", "prune", "-f", "--all"), + Setup: setup, + Cleanup: cleanup, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.All( + test.DoesNotContain(data.Get("anonIDBusy")), + test.Contains(data.Get("anonIDDangling")), + test.DoesNotContain(data.Get("namedBusy")), + test.Contains(data.Get("namedDangling")), + func(stdout string, info string, t *testing.T) { + helpers.Ensure("volume", "inspect", data.Get("anonIDBusy")) + helpers.Fail("volume", "inspect", data.Get("anonIDDangling")) + helpers.Ensure("volume", "inspect", data.Get("namedBusy")) + helpers.Fail("volume", "inspect", data.Get("namedDangling")) + }, + ), } }, - inspect: func(t *testing.T, stdout string, stderr string) { - tID := testutil.Identifier(t) - base := testutil.NewBaseWithNamespace(t, tID) - assert.Assert(t, !strings.Contains(stdout, tID+"-busy")) - base.Cmd("volume", "inspect", tID+"-busy").AssertOK() - assert.Assert(t, strings.Contains(stdout, tID+"-free")) - base.Cmd("volume", "inspect", tID+"-free").AssertFail() - // TODO verify the anonymous volumes status - }, }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - if testutil.GetTarget() != testutil.Docker { - tt.Parallel() - } - - subTID := testutil.Identifier(tt) - - if currentTest.tearDown != nil { - currentTest.tearDown(subTID) - tt.Cleanup(func() { - currentTest.tearDown(subTID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(subTID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(subTID) - res := cmd.Run() - cmd.Assert(currentTest.expected(subTID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + testGroup.Run(t) } diff --git a/cmd/nerdctl/volume/volume_remove_linux_test.go b/cmd/nerdctl/volume/volume_remove_linux_test.go index 04c67a131aa..ba775614766 100644 --- a/cmd/nerdctl/volume/volume_remove_linux_test.go +++ b/cmd/nerdctl/volume/volume_remove_linux_test.go @@ -17,16 +17,17 @@ package volume import ( + "errors" "fmt" - "strings" "testing" "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" "github.com/containerd/errdefs" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" ) // TestVolumeRemove does test a large variety of volume remove situations, albeit none of them being @@ -34,91 +35,51 @@ import ( // Behavior in such cases is largely unspecified, as there is no easy way to compare with Docker. // Anyhow, borked filesystem conditions is not something we should be expected to deal with in a smart way. func TestVolumeRemove(t *testing.T) { - t.Parallel() - - base := testutil.NewBase(t) - - inUse := errdefs.ErrFailedPrecondition.Error() - malformed := errdefs.ErrInvalidArgument.Error() - notFound := errdefs.ErrNotFound.Error() - requireArg := "requires at least 1 arg" - if base.Target == testutil.Docker { - malformed = "no such volume" - notFound = "no such volume" - inUse = "volume is in use" - } + nerdtest.Setup() - testCases := []struct { - description string - command func(tID string) *testutil.Cmd - tearUp func(tID string) - tearDown func(tID string) - expected func(tID string) icmd.Expected - inspect func(t *testing.T, stdout string, stderr string) - dockerIncompatible bool - }{ + testGroup := &test.Group{ { - description: "arg missing should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: requireArg, - } - }, + Description: "arg missing should fail", + Command: test.RunCommand("volume", "rm"), + Expected: test.Expects(1, []error{errors.New("requires at least 1 arg")}, nil), }, { - description: "invalid identifier should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "∞") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: malformed, - } - }, + Description: "invalid identifier should fail", + Command: test.RunCommand("volume", "rm", "∞"), + Expected: test.Expects(1, []error{errdefs.ErrInvalidArgument}, nil), }, { - description: "non existent volume should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "doesnotexist") - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: notFound, - } - }, + Description: "non existent volume should fail", + Command: test.RunCommand("volume", "rm", "doesnotexist"), + Expected: test.Expects(1, []error{errdefs.ErrNotFound}, nil), }, { - description: "busy volume should fail", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID) - }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID), "--name", tID, testutil.CommonImage).AssertOK() + Description: "busy volume should fail", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()), + "--name", data.Identifier(), testutil.CommonImage) }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID).Run() + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: inUse, - } + Expected: test.Expects(1, []error{errdefs.ErrFailedPrecondition}, nil), + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, }, { - description: "busy anonymous volume should fail", - command: func(tID string) *testutil.Cmd { + Description: "busy anonymous volume should fail", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()), "--name", data.Identifier(), testutil.CommonImage) // Inspect the container and find the anonymous volume id - inspect := base.InspectContainer(tID) + inspect := nerdtest.InspectContainer(helpers, data.Identifier()) var anonName string for _, v := range inspect.Mounts { if v.Destination == "/volume" { @@ -127,162 +88,142 @@ func TestVolumeRemove(t *testing.T) { } } assert.Assert(t, anonName != "", "Failed to find anonymous volume id") + data.Set("anonName", anonName) + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { // Try to remove that anon volume - return base.Cmd("volume", "rm", anonName) + return helpers.Command("volume", "rm", data.Get("anonName")) }, - tearUp: func(tID string) { - // base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID), "--name", tID, testutil.CommonImage).AssertOK() - }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - Err: inUse, - } + Expected: test.Expects(1, []error{errdefs.ErrFailedPrecondition}, nil), + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) }, }, { - description: "freed volume should succeed", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID) - }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID), "--name", tID, testutil.CommonImage).AssertOK() - base.Cmd("rm", "-f", tID).AssertOK() + Description: "freed volume should succeed", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()), "--name", data.Identifier(), testutil.CommonImage) + helpers.Ensure("rm", "-f", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID).Run() + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - Out: tID, + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) + }, }, { - description: "dangling volume should succeed", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID) + Description: "dangling volume should succeed", + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()) }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID).Run() + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - Out: tID, + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "\n"), } }, }, { - description: "part success multi-remove", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "invalid∞", "nonexistent", tID+"-busy", tID) + Description: "part success multi-remove", + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", "invalid∞", "nonexistent", data.Identifier()+"-busy", data.Identifier()) }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID).AssertOK() - base.Cmd("volume", "create", tID+"-busy").AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID+"-busy"), "--name", tID, testutil.CommonImage).AssertOK() + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()) + helpers.Ensure("volume", "create", data.Identifier()+"-busy") + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()+"-busy"), "--name", data.Identifier(), testutil.CommonImage) }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID+"-busy").Run() + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-busy") }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ ExitCode: 1, - Out: tID, + Errors: []error{ + errdefs.ErrNotFound, + errdefs.ErrFailedPrecondition, + errdefs.ErrInvalidArgument, + }, + Output: test.Equals(data.Identifier() + "\n"), } }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, inUse)) - assert.Assert(t, strings.Contains(stderr, malformed)) - }, }, { - description: "success multi-remove", - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", tID+"-1", tID+"-2") + Description: "success multi-remove", + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", data.Identifier()+"-1", data.Identifier()+"-2") }, - tearUp: func(tID string) { - base.Cmd("volume", "create", tID+"-1").AssertOK() - base.Cmd("volume", "create", tID+"-2").AssertOK() + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()+"-1") + helpers.Ensure("volume", "create", data.Identifier()+"-2") }, - tearDown: func(tID string) { - base.Cmd("volume", "rm", "-f", tID+"-1", tID+"-2").Run() + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-1", data.Identifier()+"-2") }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - Out: tID + "-1\n" + tID + "-2", + + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + Output: test.Equals(data.Identifier() + "-1\n" + data.Identifier() + "-2" + "\n"), } }, }, { - description: "failing multi-remove", - tearUp: func(tID string) { - base.Cmd("volume", "create", tID+"-busy").AssertOK() - base.Cmd("run", "-v", fmt.Sprintf("%s:/volume", tID+"-busy"), "--name", tID, testutil.CommonImage).AssertOK() - }, - tearDown: func(tID string) { - base.Cmd("rm", "-f", tID).Run() - base.Cmd("volume", "rm", "-f", tID+"-busy").Run() - }, - command: func(tID string) *testutil.Cmd { - return base.Cmd("volume", "rm", "invalid∞", "nonexistent", tID+"-busy") + Description: "failing multi-remove", + + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("volume", "create", data.Identifier()+"-busy") + helpers.Ensure("run", "-v", fmt.Sprintf("%s:/volume", data.Identifier()+"-busy"), "--name", data.Identifier(), testutil.CommonImage) }, - expected: func(tID string) icmd.Expected { - return icmd.Expected{ - ExitCode: 1, - } + + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rm", "-f", data.Identifier()) + helpers.Anyhow("volume", "rm", "-f", data.Identifier()+"-busy") }, - inspect: func(t *testing.T, stdout string, stderr string) { - assert.Assert(t, strings.Contains(stderr, notFound)) - assert.Assert(t, strings.Contains(stderr, inUse)) - assert.Assert(t, strings.Contains(stderr, malformed)) + + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("volume", "rm", "invalid∞", "nonexistent", data.Identifier()+"-busy") }, + + Expected: test.Expects(1, []error{ + errdefs.ErrNotFound, + errdefs.ErrFailedPrecondition, + errdefs.ErrInvalidArgument, + }, nil), }, } - for _, test := range testCases { - currentTest := test - t.Run(currentTest.description, func(tt *testing.T) { - if currentTest.dockerIncompatible { - testutil.DockerIncompatible(tt) - } - - tt.Parallel() - - tID := testutil.Identifier(tt) - - if currentTest.tearDown != nil { - currentTest.tearDown(tID) - tt.Cleanup(func() { - currentTest.tearDown(tID) - }) - } - if currentTest.tearUp != nil { - currentTest.tearUp(tID) - } - - // See https://github.com/containerd/nerdctl/issues/3130 - // We run first to capture the underlying icmd command and output - cmd := currentTest.command(tID) - res := cmd.Run() - cmd.Assert(currentTest.expected(tID)) - if currentTest.inspect != nil { - currentTest.inspect(tt, res.Stdout(), res.Stderr()) - } - }) - } + testGroup.Run(t) } diff --git a/docs/dev/testing.md b/docs/testing/README.md similarity index 94% rename from docs/dev/testing.md rename to docs/testing/README.md index 190733cb59d..c5bddc99285 100644 --- a/docs/dev/testing.md +++ b/docs/testing/README.md @@ -1,5 +1,10 @@ # Testing nerdctl +This document covers basic usage of nerdctl testing tasks, and generic recommendations +and principles about writing tests. + +For more comprehensive information about nerdctl test tools, see [tools.md](tools.md). + ## Lint ``` diff --git a/docs/testing/tools.md b/docs/testing/tools.md new file mode 100644 index 00000000000..655306dc48b --- /dev/null +++ b/docs/testing/tools.md @@ -0,0 +1,364 @@ +# Nerdctl testing tools + +## Preamble + +The integration test suite in nerdctl is meant to apply to both nerdctl and docker, +and further support additional test properties to target specific contexts (ipv6, kube). + +Basic _usage_ is covered in the [testing docs](testing.md). + +This here covers how to write tests, leveraging nerdctl `pkg/testutil/test` +which has been specifically developed to take care of repetitive tasks, +protect the developer against unintended side effects across tests, and generally +encourage clear testing structure with good debug-ability and a relatively simple API for +most cases. + +## Using `test.Case` + +Starting from scratch, the simplest, basic structure of a new test is: + +```go +package main + +import ( + "testing" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A first test", + // This is going to run `nerdctl info` (or `docker info`) + Command: test.RunCommand("info"), + // Verify the command exits with 0, and stdout contains the word `Kernel` + Expected: test.Expects(0, nil, test.Contains("Kernel")), + } + + // Run it + myTest.Run(t) +} +``` + +## Expectations + +There are a handful of helpers for "expectations". + +You already saw two (`test.Expects` and `test.Contains`): + +First, `test.Expects(exitCode int, errors []error, outputCompare Comparator)`, which is +convenient to quickly describe what you expect overall. + +`exitCode` is obvious. + +`errors` is a slice of go `error`, that allows you to compare what is seen on stderr +with existing errors (for example: `errdefs.ErrNotFound`), or more generally +any string you want to match. + +`outputCompare` can be either your own comparison function, or +one of the comparison helper. + +Secondly, `test.Contains`, is a `Comparator`. + +### Comparators + +Besides `test.Contains(string)`, there are a few more: +- `test.DoesNotContain(string)` +- `test.Equals(string)` +- `test.All(comparators ...Comparator)`, which allows you to bundle together a bunch of other comparators + +The following example shows how to implement your own custom `Comparator` +(this is actually the `Equals` comparator). + +```go +package whatever + +import ( + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func MyComparator(compare string) test.Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Assert(t, stdout == compare, info) + } +} +``` + +Note that you have access to an opaque `info` string. +It contains relevant debugging information in case your comparator is going to fail, +and you should make sure it is displayed. + +### Advanced expectations + +You may want to have expectations that contain a certain piece of data +that is being used in the command or at other stages of your test (Setup). + +For example, creating a container with a certain name, you might want to verify +that this name is then visible in the list of containers. + +To achieve that, you should write your own `Expecter`, leveraging test `Data`. + +Here is an example, where we are using `data.Get("sometestdata")`. + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/errdefs" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A subtest with custom data, manager, and comparator", + Data: test.WithData("sometestdata", "blah"), + Command: test.RunCommand("info"), + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + errdefs.ErrNotFound, + }, + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + }, + } + + myTest.Run(t) +} +``` + +## On `Data` + +`Data` is provided to first allow storing mutable key-value information that pertain to the test. + +While it can be provided through `WithData(key string, value string)` (or `WithConfig`, see below), +inside the testcase definition, it can also be dynamically manipulated inside `Setup`, or `Command`. + +Note that `Data` additionally exposes the following functions: +- `Identifier()` which returns the unique id associated with the _current_ test (or subtest) +- `TempDir()` which returns the private, temporary directory associated with the test + +... along with the `Get(key)` and `Set(key, value)` methods. + +Secondly, `Data` allows defining and manipulating "configuration" data. + +In the case of nerdctl here, the following configuration options are defined: +- `WithConfig(Docker, NotCompatible)` to flag a test as not compatible +- `WithConfig(Mode, Private)` will entirely isolate the test using a different +namespace, data root, nerdctl config, etc +- `WithConfig(NerdctlToml, "foo")` which allows specifying a custom config +- `WithConfig(DataRoot, "foo")` allowing to point to a custom data-root +- `WithConfig(HostsDir, "foo")` to point to a specific hosts directory +- `WithConfig(Namespace, "foo")` allows passing a specific namespace (otherwise defaults to `nerdctl-test`) + +## Commands + +For simple cases, `test.RunCommand(args ...string)` is the way to go. + +It will execute the binary to test (nerdctl or docker), with the provided arguments, +and will by default get cwd inside the temporary directory associated with the test. + +### Environment + +You can attach custom environment variables for your test in the `Env` property of your +test. + +These will be automatically added to the environment for your command, and also +your setup and cleanup routines (see below). + +If you would like to override the environment specifically for a command, but not for +others (eg: in `Setup` or `Cleanup`), you can do so with custom commands (see below). + +Note that environment as defined statically in the test will be inherited by subtests, +unless explicitly overridden. + +### Working directory + +By default, the working directory of the command will be set to the temporary directory +of the test. + +This behavior can be overridden using custom commands. + +### Custom commands + +Custom commands allow you to manipulate test `Data`, override important aspects +of the command to execute (`Env`, `WorkingDir`), or otherwise give you full control +on what the command does. + +You just need to implement an `Executor`: + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/errdefs" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A subtest with custom data, manager, and comparator", + Data: test.WithData("sometestdata", "blah"), + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("run", "--name", data.Identifier()+data.Get("sometestdata")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + errdefs.ErrNotFound, + }, + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + }, + } + + myTest.Run(t) +} +``` + +### On `helpers` + +Inside a custom `Executor`, `Manager`, or `Butler`, you have access to a collection of +`helpers` to simplify command execution: + +- `helpers.Ensure(args ...string)` will run a command and ensure it exits succesfully +- `helpers.Fail(args ...string)` will run a command and ensure it fails +- `helpers.Anyhow(args ...string)` will run a command but does not care if it succeeds or fails +- `helpers.Capture(args ...string)` will run a command, ensure it is successful, and return the output +- `helpers.Command(args ...string)` will return a command that can then be tested against expectations +- `helpers.CustomCommand(binary string, args ...string)` will do the same for any arbitrary command (not limited to nerdctl) + +## Setup and Cleanup + +Tests routinely require a set of actions to be performed _before_ you can run the +command you want to test. +A setup routine will get executed before your `Command`, and have access to and can +manipulate your test `Data`. + +Conversely, you very likely want to clean things up once your test is done. +While temporary directories are cleaned for you with no action needed on your part, +the app you are testing might have stateful data you may want to remove. +Note that a `Cleanup` routine will get executed twice - after your `Command` has run +its course evidently - but also, pre-emptively, before your `Setup`, so that possible leftovers from +previous runs are taken care of. + +```go +package main + +import ( + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/errdefs" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func TestMyThing(t *testing.T) { + nerdtest.Setup() + + // Declare your test + myTest := &test.Case{ + Description: "A subtest with custom data, manager, and comparator", + Data: test.WithData("sometestdata", "blah"), + Setup: func(data *test.Data, helpers test.Helpers){ + helpers.Ensure("volume", "create", "foo") + helpers.Ensure("volume", "create", "bar") + }, + Cleanup: func(data *test.Data, helpers test.Helpers){ + helpers.Anyhow("volume", "rm", "foo") + helpers.Anyhow("volume", "rm", "bar") + }, + Command: func(data test.Data, helpers test.Helpers) test.Command { + return helpers.Command("run", "--name", data.Identifier()+data.Get("sometestdata")) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{ + errors.New("foobla"), + errdefs.ErrNotFound, + }, + Output: func(stdout string, info string, t *testing.T) { + assert.Assert(t, stdout == data.Get("sometestdata"), info) + }, + } + }, + } + + myTest.Run(t) +} +``` + +## Subtests + +Subtests are just regular tests, attached to the `SubTests` slice of a test. + +Note that a subtest will inherit its parent `Data` and `Env`, in the state they are at +after the parent test has run its `Setup` and `Command` routines (but before `Cleanup`). +This does _not_ apply to `Identifier()` and `TempDir()`, which are unique to the sub-test. + +Also note that a test does not have to have a `Command`. +This is a convenient pattern if you just need a common `Setup` for a bunch of subtests. + +## Groups + +A `test.Group` is just a convenient way to represent a slice of tests. + +Note that unlike a `test.Case`, a group cannot define properties inherited by +subtests, nor `Setup` or `Cleanup` routines. + +- if you just have a bunch of subtests you want to run, put them in a `test.Group` +- if you want to have a global setup, or otherwise set a common property first for your subtests, use a `test.Case` with `SubTests` + +## Parallelism + +All tests (and subtests) are assumed to be parallelizable. + +You can force a specific `test.Case` to not be run in parallel though, +by setting its `NoParallel` property to `true`. + +Note that if you want better isolation, it is usually better to use +`WithConfig(nerdtest.Mode, nerdtest.Private)` instead. +This will keep the test parallel (for nerdctl), but isolate it in a different context. +For Docker (which does not support namespaces), it is equivalent to passing `NoParallel: true`. diff --git a/pkg/testutil/nerdtest/test.go b/pkg/testutil/nerdtest/test.go new file mode 100644 index 00000000000..a2f7a5bd3c2 --- /dev/null +++ b/pkg/testutil/nerdtest/test.go @@ -0,0 +1,253 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package nerdtest + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/v2/pkg/buildkitutil" + "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" + "github.com/containerd/nerdctl/v2/pkg/inspecttypes/native" + "github.com/containerd/nerdctl/v2/pkg/rootlessutil" + "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/test" +) + +func Setup() { + test.CustomCommand(nerdctlSetup) +} + +// Nerdctl specific config key and values +var NerdctlToml test.ConfigKey = "NerdctlToml" +var HostsDir test.ConfigKey = "HostsDir" +var DataRoot test.ConfigKey = "DataRoot" +var Namespace test.ConfigKey = "Namespace" + +var Mode test.ConfigKey = "Mode" +var ModePrivate test.ConfigValue = "Private" +var IPv6 test.ConfigKey = "IPv6Test" +var Only test.ConfigValue = "Only" + +var OnlyIPv6 = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + ret = testutil.GetEnableIPv6() + if !ret { + mess = "runner skips IPv6 compatible tests in the non-IPv6 environment" + } + data.WithConfig(IPv6, Only) + return ret, mess +}) + +var Private = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + data.WithConfig(Mode, ModePrivate) + return true, "" +}) + +var Docker = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + ret = testutil.GetTarget() == testutil.Docker + if ret { + mess = "current target is docker" + } else { + mess = "current target is not docker" + } + return ret, mess +}) + +var Rootless = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + ret = rootlessutil.IsRootless() + if ret { + mess = "environment is rootless" + } else { + mess = "environment is rootful" + } + return ret, mess +}) + +var Build = test.MakeRequirement(func(data test.Data) (ret bool, mess string) { + // FIXME: shouldn't we run buildkitd in a container? At least for testing, that would be so much easier than + // against the host install + ret = true + mess = "" + if testutil.GetTarget() == testutil.Nerdctl { + _, err := buildkitutil.GetBuildkitHost(testutil.Namespace) + if err != nil { + ret = false + mess = fmt.Sprintf("test requires buildkitd: %+v", err) + } + } + return ret, mess +}) + +type NerdCommand struct { + test.GenericCommand + // FIXME: annoying - forces custom Clone, etc + Target string +} + +// Run does override the generic command run, as we are testing both docker and nerdctl +func (nc *NerdCommand) Run(expect *test.Expected) { + // We are not in the business of testing docker error output, so, spay expect for errors testing, if any + if expect != nil && nc.Target != testutil.Nerdctl { + expect.Errors = nil + } + + nc.GenericCommand.Run(expect) +} + +// Clone is overridden as well, as we need to pass along the target +func (nc *NerdCommand) Clone() test.Command { + return &NerdCommand{ + GenericCommand: *((nc.GenericCommand.Clone()).(*test.GenericCommand)), + Target: nc.Target, + } +} + +// InspectContainer is a helper that can be used inside custom commands or Setup +func InspectContainer(helpers test.Helpers, name string) dockercompat.Container { + var dc []dockercompat.Container + cmd := helpers.Command("container", "inspect", name) + cmd.Run(&test.Expected{ + ExitCode: 0, + Output: func(stdout string, info string, t *testing.T) { + err := json.Unmarshal([]byte(stdout), &dc) + assert.NilError(t, err, "Unable to unmarshal output\n"+info) + assert.Equal(t, 1, len(dc), "Unexpectedly got multiple results\n"+info) + }, + }) + return dc[0] +} + +func InspectVolume(helpers test.Helpers, name string, args ...string) native.Volume { + var dc []native.Volume + cmdArgs := append([]string{"volume", "inspect"}, args...) + cmdArgs = append(cmdArgs, name) + + cmd := helpers.Command(cmdArgs...) + cmd.Run(&test.Expected{ + ExitCode: 0, + Output: func(stdout string, info string, t *testing.T) { + err := json.Unmarshal([]byte(stdout), &dc) + assert.NilError(t, err, "Unable to unmarshal output\n"+info) + assert.Equal(t, 1, len(dc), "Unexpectedly got multiple results\n"+info) + }, + }) + return dc[0] +} + +func InspectNetwork(helpers test.Helpers, name string, args ...string) dockercompat.Network { + var dc []dockercompat.Network + cmdArgs := append([]string{"network", "inspect"}, args...) + cmdArgs = append(cmdArgs, name) + + cmd := helpers.Command(cmdArgs...) + cmd.Run(&test.Expected{ + ExitCode: 0, + Output: func(stdout string, info string, t *testing.T) { + err := json.Unmarshal([]byte(stdout), &dc) + assert.NilError(t, err, "Unable to unmarshal output\n"+info) + assert.Equal(t, 1, len(dc), "Unexpectedly got multiple results\n"+info) + }, + }) + return dc[0] +} + +func nerdctlSetup(testCase *test.Case, t *testing.T) test.Command { + t.Helper() + + var testUtilBase *testutil.Base + dt := testCase.Data + var pvNamespace string + inherited := false + + if dt.ReadConfig(IPv6) != Only && testutil.GetEnableIPv6() { + t.Skip("runner skips non-IPv6 compatible tests in the IPv6 environment") + } + + if dt.ReadConfig(Mode) == ModePrivate { + // If private was inherited, we already got a configured namespace + if dt.ReadConfig(Namespace) != "" { + pvNamespace = string(dt.ReadConfig(Namespace)) + inherited = true + } else { + // Otherwise, we need to set everything up + pvNamespace = testCase.Data.Identifier() + dt.WithConfig(Namespace, test.ConfigValue(pvNamespace)) + testCase.Env["DOCKER_CONFIG"] = testCase.Data.TempDir() + testCase.Env["NERDCTL_TOML"] = filepath.Join(testCase.Data.TempDir(), "nerdctl.toml") + dt.WithConfig(HostsDir, test.ConfigValue(testCase.Data.TempDir())) + dt.WithConfig(DataRoot, test.ConfigValue(testCase.Data.TempDir())) + } + testUtilBase = testutil.NewBaseWithNamespace(t, pvNamespace) + if testUtilBase.Target == testutil.Docker { + // For docker, just disable parallel + testCase.NoParallel = true + } + } else if dt.ReadConfig(Namespace) != "" { + pvNamespace = string(dt.ReadConfig(Namespace)) + testUtilBase = testutil.NewBaseWithNamespace(t, pvNamespace) + } else { + testUtilBase = testutil.NewBase(t) + } + + // If we were passed custom content for NerdctlToml, save it + // Not happening if this is not nerdctl of course + if testUtilBase.Target == testutil.Nerdctl && dt.ReadConfig(NerdctlToml) != "" { + dest := filepath.Join(testCase.Data.TempDir(), "nerdctl.toml") + testCase.Env["NERDCTL_TOML"] = dest + err := os.WriteFile(dest, []byte(dt.ReadConfig(NerdctlToml)), 0400) + assert.NilError(t, err, "failed to write custom nerdctl toml file for test") + } + + // Build the base + baseCommand := &NerdCommand{} + baseCommand.WithBinary(testUtilBase.Binary) + baseCommand.WithArgs(testUtilBase.Args...) + baseCommand.WithEnv(testCase.Env) + baseCommand.WithT(t) + baseCommand.WithTempDir(testCase.Data.TempDir()) + baseCommand.Target = testUtilBase.Target + + if testUtilBase.Target == testutil.Nerdctl { + if dt.ReadConfig(HostsDir) != "" { + baseCommand.GenericCommand.WithArgs("--hosts-dir=" + string(dt.ReadConfig(HostsDir))) + } + + if dt.ReadConfig(DataRoot) != "" { + baseCommand.GenericCommand.WithArgs("--data-root=" + string(dt.ReadConfig(DataRoot))) + } + } + + // If we were in a custom namespace, not inherited - make sure we clean up the namespace + // FIXME: this is broken, and custom namespaces are not cleaned properly + if testUtilBase.Target == testutil.Nerdctl && pvNamespace != "" && !inherited { + cleanup := func() { + cl := baseCommand.Clone() + cl.WithArgs("namespace", "remove", pvNamespace) + cl.Run(nil) + } + cleanup() + t.Cleanup(cleanup) + } + + // Attach the base command + return baseCommand +} diff --git a/pkg/testutil/test/case.go b/pkg/testutil/test/case.go new file mode 100644 index 00000000000..eed35a929f6 --- /dev/null +++ b/pkg/testutil/test/case.go @@ -0,0 +1,175 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +// Group informally describes a slice of tests +type Group []*Case + +func (tg *Group) Run(t *testing.T) { + t.Helper() + // If the group contains only one test, no need to create a subtest + sub := len(*tg) > 1 + if sub { + t.Parallel() + } + // Run each subtest + for _, tc := range *tg { + tc.subIt = sub + tc.Run(t) + } +} + +// Case describes an entire test-case, including data, setup and cleanup routines, command and expectations +type Case struct { + // Description contains a human-readable short desc, used as a seed for the identifier and as a title for the test + Description string + // NoParallel disables parallel execution if set to true + NoParallel bool + // Env contains a map of environment variables to use for commands run in Setup, Command and Cleanup + // Note that the environment is inherited by subtests + Env map[string]string + // Data contains test specific data, accessible to all operations, also inherited by subtests + Data Data + + // Setup + Setup Butler + // Expected + Expected Manager + // Command + Command Executor + // Cleanup + Cleanup Butler + // Requirement + Require Requirement + + // SubTests + SubTests []*Case + + // Private + helpers Helpers + t *testing.T + parent *Case + baseCommand Command + + subIt bool +} + +// Run prepares and executes the test, and any possible subtests +func (test *Case) Run(t *testing.T) { + t.Helper() + // Run the test + testRun := func(tt *testing.T) { + tt.Helper() + test.seal(tt) + + if registeredInit == nil { + bc := &GenericCommand{} + bc.WithEnv(test.Env) + bc.WithT(tt) + bc.WithTempDir(test.Data.TempDir()) + test.baseCommand = bc + } else { + test.baseCommand = registeredInit(test, test.t) + } + + test.exec(tt) + } + + if test.subIt { + t.Run(test.Description, testRun) + } else { + testRun(t) + } +} + +// seal is a private method to prepare the test +func (test *Case) seal(t *testing.T) { + t.Helper() + assert.Assert(t, test.t == nil, "You cannot run a test multiple times") + assert.Assert(t, test.Description != "", "A test description cannot be empty") + assert.Assert(t, test.Command == nil || test.Expected != nil, + "Expectations for a test command cannot be nil. You may want to use Setup instead.") + + // Ensure we have env + if test.Env == nil { + test.Env = map[string]string{} + } + + // If we have a parent, get parent env and data + var parentData Data + if test.parent != nil { + parentData = test.parent.Data + for k, v := range test.parent.Env { + if _, ok := test.Env[k]; !ok { + test.Env[k] = v + } + } + } + + // Attach testing.T + test.t = t + // Inherit and attach Data + test.Data = configureData(t, test.Data, parentData) + + // Check the requirements + if test.Require != nil { + test.Require(test.Data, t) + } +} + +// exec is a private method that will take care of the test setup, command and cleanup execution +func (test *Case) exec(t *testing.T) { + t.Helper() + test.helpers = &helpers{ + test.baseCommand, + } + + // Set parallel unless asked not to + if !test.NoParallel { + t.Parallel() + } + + // Register cleanup if there is any, and run it to collect any leftovers from previous runs + if test.Cleanup != nil { + test.Cleanup(test.Data, test.helpers) + t.Cleanup(func() { + test.Cleanup(test.Data, test.helpers) + }) + } + + // Run setup + if test.Setup != nil { + test.Setup(test.Data, test.helpers) + } + + // Run the command if any, with expectations + if test.Command != nil { + test.Command(test.Data, test.helpers).Run(test.Expected(test.Data, test.helpers)) + } + + for _, subTest := range test.SubTests { + subTest.parent = test + subTest.subIt = true + subTest.Run(t) + } +} diff --git a/pkg/testutil/test/command.go b/pkg/testutil/test/command.go new file mode 100644 index 00000000000..6fbb1779d52 --- /dev/null +++ b/pkg/testutil/test/command.go @@ -0,0 +1,191 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + "gotest.tools/v3/assert" + "gotest.tools/v3/icmd" +) + +// GenericCommand is a concrete Command implementation +type GenericCommand struct { + WorkingDir string + Env map[string]string + + t *testing.T + tempDir string + helperBinary string + helperArgs []string + mainBinary string + mainArgs []string + result *icmd.Result + stdin io.Reader + async bool + timeout time.Duration +} + +func (gc *GenericCommand) WithBinary(binary string) Command { + gc.mainBinary = binary + return gc +} + +func (gc *GenericCommand) WithArgs(args ...string) Command { + gc.mainArgs = append(gc.mainArgs, args...) + return gc +} + +// WithEnv will overload the command env with values from the passed map +func (gc *GenericCommand) WithEnv(env map[string]string) Command { + if gc.Env == nil { + gc.Env = map[string]string{} + } + for k, v := range env { + gc.Env[k] = v + } + return gc +} + +func (gc *GenericCommand) WithWrapper(binary string, args ...string) Command { + gc.helperBinary = binary + gc.helperArgs = args + return gc +} + +// WithStdin sets the standard input of Cmd to the specified reader +func (gc *GenericCommand) WithStdin(r io.Reader) Command { + gc.stdin = r + return gc +} + +func (gc *GenericCommand) Background(timeout time.Duration) Command { + // Run it + gc.async = true + i := gc.boot() + gc.result = icmd.StartCmd(i) + gc.timeout = timeout + return gc +} + +// TODO: it should be possible to: +// - timeout execution +func (gc *GenericCommand) Run(expect *Expected) { + var result *icmd.Result + var env []string + if gc.async { + result = icmd.WaitOnCmd(gc.timeout, gc.result) + env = gc.result.Cmd.Env + } else { + icmdCmd := gc.boot() + env = icmdCmd.Env + // Run it + result = icmd.RunCmd(icmdCmd) + } + + // Check our expectations, if any + if expect != nil { + // Build the debug string - additionally attach the env (which icmd does not do) + debug := result.String() + "Env:\n" + strings.Join(env, "\n") + // ExitCode goes first + if expect.ExitCode == -1 { + assert.Assert(gc.t, result.ExitCode != 0, + "Expected exit code to be different than 0"+debug) + } else { + assert.Assert(gc.t, expect.ExitCode == result.ExitCode, + fmt.Sprintf("Expected exit code: %d", expect.ExitCode)+debug) + } + // Range through the expected errors and confirm they are seen on stderr + for _, expectErr := range expect.Errors { + assert.Assert(gc.t, strings.Contains(result.Stderr(), expectErr.Error()), + fmt.Sprintf("Expected error: %q to be found in stderr", expectErr.Error())+debug) + } + // Finally, check the output if we are asked to + if expect.Output != nil { + expect.Output(result.Stdout(), debug, gc.t) + } + } +} + +func (gc *GenericCommand) boot() icmd.Cmd { + // This is a helper function, not to appear in the debugging output + gc.t.Helper() + + binary := gc.mainBinary + args := gc.mainArgs + if gc.helperBinary != "" { + args = append([]string{binary}, args...) + args = append(gc.helperArgs, args...) + binary = gc.helperBinary + } + + // Create the command and set the env + // TODO: do we really need icmd? + icmdCmd := icmd.Command(binary, args...) + icmdCmd.Env = []string{} + for _, v := range os.Environ() { + // Ignore LS_COLORS from the env, just too much noise + if !strings.HasPrefix(v, "LS_COLORS") { + icmdCmd.Env = append(icmdCmd.Env, v) + } + } + + // Ensure the subprocess gets executed in a temporary directory unless explicitly instructed otherwise + icmdCmd.Dir = gc.WorkingDir + if icmdCmd.Dir == "" { + icmdCmd.Dir = gc.tempDir + } + + // Attach any extra env we have + for k, v := range gc.Env { + icmdCmd.Env = append(icmdCmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + + return icmdCmd +} + +func (gc *GenericCommand) Clone() Command { + // Copy the command and return a new one - with WorkingDir, binary, args, etc + cc := *gc + // Clone Env + cc.Env = make(map[string]string, len(gc.Env)) + for k, v := range gc.Env { + cc.Env[k] = v + } + return &cc +} + +func (gc *GenericCommand) Clear() Command { + gc.mainBinary = "" + gc.helperBinary = "" + gc.mainArgs = []string{} + gc.helperArgs = []string{} + return gc +} + +func (gc *GenericCommand) WithT(t *testing.T) { + gc.t = t +} + +func (gc *GenericCommand) WithTempDir(tempDir string) { + gc.tempDir = tempDir +} diff --git a/pkg/testutil/test/data.go b/pkg/testutil/test/data.go new file mode 100644 index 00000000000..99f2aa041ad --- /dev/null +++ b/pkg/testutil/test/data.go @@ -0,0 +1,147 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "crypto/sha256" + "fmt" + "strings" + "testing" +) + +// Contains the implementation of the Data interface + +type data struct { + config map[ConfigKey]ConfigValue + + system map[SystemKey]SystemValue + + labels map[string]string + testID string + tempDir string +} + +func (dt *data) WithConfig(key ConfigKey, value ConfigValue) Data { + if dt.config == nil { + dt.config = make(map[ConfigKey]ConfigValue) + } + dt.config[key] = value + return dt +} + +func (dt *data) ReadConfig(key ConfigKey) ConfigValue { + if dt.config == nil { + dt.config = make(map[ConfigKey]ConfigValue) + } + if val, ok := dt.config[key]; ok { + return val + } + return "" +} + +func (dt *data) Get(key string) string { + if dt.labels == nil { + dt.labels = map[string]string{} + } + return dt.labels[key] +} + +func (dt *data) Set(key string, value string) Data { + if dt.labels == nil { + dt.labels = map[string]string{} + } + dt.labels[key] = value + return dt +} + +func (dt *data) Identifier() string { + return dt.testID +} + +func (dt *data) TempDir() string { + return dt.tempDir +} + +func (dt *data) adopt(parent Data) { + for k, v := range parent.getLabels() { + // Only copy keys that are not set already + if _, ok := dt.labels[k]; !ok { + dt.Set(k, v) + } + } + for k, v := range parent.getConfig() { + // Only copy keys that are not set already + if _, ok := dt.config[k]; !ok { + dt.WithConfig(k, v) + } + } +} + +func (dt *data) Sink(key SystemKey, value SystemValue) { + if _, ok := dt.system[key]; !ok { + dt.system[key] = value + } else { + // XXX should we really panic? + panic(fmt.Sprintf("Unable to set system key %s multiple times", key)) + } +} + +func (dt *data) Surface(key SystemKey) SystemValue { + if v, ok := dt.system[key]; ok { + return v + } + // XXX should we really panic? + panic(fmt.Sprintf("Unable to retrieve system key %s", key)) +} + +func (dt *data) getLabels() map[string]string { + return dt.labels +} + +func (dt *data) getConfig() map[ConfigKey]ConfigValue { + return dt.config +} + +func defaultIdentifierHashing(name string) string { + s := strings.ReplaceAll(name, " ", "_") + s = strings.ReplaceAll(s, "/", "_") + s = strings.ReplaceAll(s, "-", "_") + s = strings.ReplaceAll(s, ",", "_") + s = strings.ToLower(s) + if len(s) > 76 { + s = fmt.Sprintf("%x", sha256.Sum256([]byte(s))) + } + + return s +} + +// TODO: allow to pass custom hashing methods? +func configureData(t *testing.T, seedData Data, parent Data) Data { + if seedData == nil { + seedData = &data{} + } + dat := &data{ + config: seedData.getConfig(), + labels: seedData.getLabels(), + tempDir: t.TempDir(), + testID: defaultIdentifierHashing(t.Name()), + } + if parent != nil { + dat.adopt(parent) + } + return dat +} diff --git a/pkg/testutil/test/expected.go b/pkg/testutil/test/expected.go new file mode 100644 index 00000000000..81d617acfdf --- /dev/null +++ b/pkg/testutil/test/expected.go @@ -0,0 +1,90 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "fmt" + "strings" + "testing" + + "gotest.tools/v3/assert" +) + +func RunCommand(args ...string) Executor { + return func(data Data, helpers Helpers) Command { + return helpers.Command(args...) + } +} + +// WithData returns a data object with a certain key value set +func WithData(key string, value string) Data { + dat := &data{} + dat.Set(key, value) + return dat +} + +// WithConfig returns a data object with a certain config property set +func WithConfig(key ConfigKey, value ConfigValue) Data { + dat := &data{} + dat.WithConfig(key, value) + return dat +} + +// Expects is provided as a simple helper covering "expectations" for simple use-cases where access to the test data is not necessary +func Expects(exitCode int, errors []error, output Comparator) Manager { + return func(_ Data, _ Helpers) *Expected { + return &Expected{ + ExitCode: exitCode, + Errors: errors, + Output: output, + } + } +} + +// All can be used as a parameter for expected.Output and allow passing a collection of conditions to match +func All(comparators ...Comparator) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + for _, comparator := range comparators { + comparator(stdout, info, t) + } + } +} + +// Contains can be used as a parameter for expected.Output and ensures a comparison string is found contained in the output +func Contains(compare string) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Assert(t, strings.Contains(stdout, compare), fmt.Sprintf("Expected output to contain: %q", compare)+info) + } +} + +// DoesNotContain is to be used for expected.Output to ensure a comparison string is NOT found in the output +func DoesNotContain(compare string) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Assert(t, !strings.Contains(stdout, compare), fmt.Sprintf("Expected output to not contain: %q", compare)+info) + } +} + +// Equals is to be used for expected.Output to ensure it is exactly the output +func Equals(compare string) Comparator { + return func(stdout string, info string, t *testing.T) { + t.Helper() + assert.Equal(t, compare, stdout, info) + } +} diff --git a/pkg/testutil/test/helpers.go b/pkg/testutil/test/helpers.go new file mode 100644 index 00000000000..64a734dd86c --- /dev/null +++ b/pkg/testutil/test/helpers.go @@ -0,0 +1,71 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import "testing" + +type Helpers interface { + Ensure(args ...string) + Anyhow(args ...string) + Fail(args ...string) + Capture(args ...string) string + + Command(args ...string) Command + CustomCommand(binary string, args ...string) Command +} + +type helpers struct { + cmd Command +} + +func (hel *helpers) Ensure(args ...string) { + hel.Command(args...).Run(&Expected{}) +} + +func (hel *helpers) Anyhow(args ...string) { + hel.Command(args...).Run(nil) +} + +func (hel *helpers) Fail(args ...string) { + hel.Command(args...).Run(&Expected{ + ExitCode: 1, + }) +} + +func (hel *helpers) Capture(args ...string) string { + var ret string + hel.Command(args...).Run(&Expected{ + Output: func(stdout string, info string, t *testing.T) { + ret = stdout + }, + }) + return ret +} + +func (hel *helpers) Command(args ...string) Command { + cc := hel.cmd.Clone() + cc.WithArgs(args...) + return cc +} + +func (hel *helpers) CustomCommand(binary string, args ...string) Command { + cc := hel.cmd.Clone() + cc.Clear() + cc.WithBinary(binary) + cc.WithArgs(args...) + return cc +} diff --git a/pkg/testutil/test/requirement.go b/pkg/testutil/test/requirement.go new file mode 100644 index 00000000000..1acad13af28 --- /dev/null +++ b/pkg/testutil/test/requirement.go @@ -0,0 +1,115 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "fmt" + "os/exec" + "runtime" + "testing" +) + +func MakeRequirement(fn func(data Data) (bool, string)) Requirement { + return func(data Data, t *testing.T) (bool, string) { + ret, mess := fn(data) + + if t != nil && !ret { + t.Helper() + t.Skipf("Test skipped as %s", mess) + } + + return ret, mess + } +} + +func Binary(name string) Requirement { + return MakeRequirement(func(data Data) (ret bool, mess string) { + mess = fmt.Sprintf("executable %q has been found in PATH", name) + ret = true + if _, err := exec.LookPath(name); err != nil { + ret = false + mess = fmt.Sprintf("executable %q doesn't exist in PATH", name) + } + + return ret, mess + }) +} + +func OS(os string) Requirement { + return MakeRequirement(func(data Data) (ret bool, mess string) { + mess = fmt.Sprintf("current operating is %q", runtime.GOOS) + ret = true + if runtime.GOOS != os { + ret = false + } + + return ret, mess + }) +} + +var Windows = MakeRequirement(func(data Data) (ret bool, mess string) { + ret = runtime.GOOS == "windows" + if ret { + mess = "operating system is Windows" + } else { + mess = "operating system is not Windows" + } + return ret, mess +}) + +var Linux = MakeRequirement(func(data Data) (ret bool, mess string) { + ret = runtime.GOOS == "linux" + if ret { + mess = "operating system is Linux" + } else { + mess = "operating system is not Linux" + } + return ret, mess +}) + +var Darwin = MakeRequirement(func(data Data) (ret bool, mess string) { + ret = runtime.GOOS == "darwin" + if ret { + mess = "operating system is Darwin" + } else { + mess = "operating system is not Darwin" + } + return ret, mess +}) + +func Not(requirement Requirement) Requirement { + return MakeRequirement(func(data Data) (ret bool, mess string) { + b, mess := requirement(data, nil) + return !b, mess + }) +} + +func Require(thing ...Requirement) Requirement { + return func(data Data, t *testing.T) (ret bool, mess string) { + for _, th := range thing { + b, m := th(data, nil) + if !b { + if t != nil { + t.Helper() + t.Skipf("Test skipped as %s", m) + } + return false, "" + } + } + return true, "" + } +} diff --git a/pkg/testutil/test/test.go b/pkg/testutil/test/test.go new file mode 100644 index 00000000000..2e6743be8e6 --- /dev/null +++ b/pkg/testutil/test/test.go @@ -0,0 +1,119 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package test + +import ( + "io" + "testing" + "time" +) + +// A Requirement is a function that can evaluate random requirement and possibly skip a test +// See test.MakeRequirement to make your own +type Requirement func(data Data, t *testing.T) (bool, string) + +// A Butler is the function signature meant to be attached to a Setup or Cleanup routine for a test.Case +type Butler func(data Data, helpers Helpers) + +// An Executor is the function signature meant to be attached to a test.Case Command +type Executor func(data Data, helpers Helpers) Command + +// A Manager is the function signature to be run to produce expectations to be fed to a command +type Manager func(data Data, helpers Helpers) *Expected + +// The Command interface represents a low-level command to execute, typically to be compared with an Expected +// A Command can be used as a Case Command obviously, but also as part of a Setup or Cleanup routine, +// and as the basis of any type of helper. +// A Command can be cloned, in which case, the subcommand inherits a copy of all of its Env and parameters. +// Typically, a Case has a base-command, from which all commands involved in the test are derived. +type Command interface { + // WithBinary specifies what binary to execute + WithBinary(binary string) Command + // WithArgs specifies the args to pass to the binary. Note that WithArgs is additive. + WithArgs(args ...string) Command + // WithEnv adds the passed map to the environment of the command to be executed + WithEnv(env map[string]string) Command + // WithWrapper allows wrapping a command with another command (for example: `time`, `unbuffer`) + WithWrapper(binary string, args ...string) Command + // WithStdin allows passing a reader to be used for stdin for the command + WithStdin(r io.Reader) Command + // Run does execute the command, and compare the output with the provided expectation. + // Passing nil for `Expected` will just run the command regardless of outcome. + // An empty `&Expected{}` is (of course) equivalent to &Expected{Exit: 0}, meaning the command is verified to be + // successful + Run(expect *Expected) + // Clone returns a copy of the command + Clone() Command + // Clear will clear binary and arguments, but retain the env, or any other custom properties + Clear() Command + // Allow starting a command in the background + Background(timeout time.Duration) Command +} + +type Comparator func(stdout string, info string, t *testing.T) + +// Expected expresses the expected output of a command +type Expected struct { + // ExitCode to expect + ExitCode int + // Errors contains any error that (once serialized) should be seen in stderr + Errors []error + // Output function to match against stdout + Output Comparator +} + +type ConfigKey string +type ConfigValue string + +type SystemKey string +type SystemValue string + +// Data is meant to hold information about a test: +// - first, any random key value data that the test implementer wants to carry / modify - this is test data +// - second, configuration specific to the binary being tested - typically defined by the specialized command being tested +// - third, immutable "system" info (unique identifier, tempdir, or other SystemKey/Value pairs) +type Data interface { + // Get returns the value of a certain key for custom data + Get(key string) string + // Set will save `value` for `key` + Set(key string, value string) Data + + // Identifier returns the test identifier that can be used to name resources + Identifier() string + // TempDir returns the test temporary directory + TempDir() string + // Sink allows to define ONCE a certain system property + Sink(key SystemKey, value SystemValue) + // Surface allows retrieving a certain system property + Surface(key SystemKey) SystemValue + + // WithConfig allows setting a declared ConfigKey to a ConfigValue + WithConfig(key ConfigKey, value ConfigValue) Data + ReadConfig(key ConfigKey) ConfigValue + + // Private methods + getLabels() map[string]string + getConfig() map[ConfigKey]ConfigValue +} + +var ( + registeredInit func(test *Case, t *testing.T) Command +) + +func CustomCommand(custom func(test *Case, t *testing.T) Command) { + registeredInit = custom +}