diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 92981426..675fd299 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -13,6 +13,19 @@ type RestConfig struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// TestFile contains attributes of a single test file. +type TestFile struct { + // The type meta object, should always be a GVK of kuttl.dev/v1beta1/TestFile. + metav1.TypeMeta `json:",inline"` + // Set labels or the test suite name. + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Which test runs should this file be used in. Empty selector matches all test runs. + TestRunSelector *metav1.LabelSelector `json:"testRunSelector,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + // TestSuite configures which tests should be loaded. type TestSuite struct { // The type meta object, should always be a GVK of kuttl.dev/v1beta1/TestSuite or kuttl.dev/v1beta1/TestSuite. diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index 68f8d6c1..4698a0c6 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -20,6 +20,7 @@ limitations under the License. package v1beta1 import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -147,6 +148,37 @@ func (in *TestCollector) DeepCopy() *TestCollector { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestFile) DeepCopyInto(out *TestFile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.TestRunSelector != nil { + in, out := &in.TestRunSelector, &out.TestRunSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestFile. +func (in *TestFile) DeepCopy() *TestFile { + if in == nil { + return nil + } + out := new(TestFile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TestFile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestStep) DeepCopyInto(out *TestStep) { *out = *in diff --git a/pkg/kuttlctl/cmd/test.go b/pkg/kuttlctl/cmd/test.go index a54698db..9b80db6d 100644 --- a/pkg/kuttlctl/cmd/test.go +++ b/pkg/kuttlctl/cmd/test.go @@ -61,6 +61,7 @@ func newTestCmd() *cobra.Command { //nolint:gocyclo reportName := "kuttl-report" namespace := "" suppress := []string{} + var runLabels labelSetValue options := harness.TestSuite{} @@ -229,6 +230,7 @@ For more detailed documentation, visit: https://kuttl.dev`, harness := test.Harness{ TestSuite: options, T: t, + RunLabels: runLabels.AsLabelSet(), } harness.Run() @@ -257,6 +259,7 @@ For more detailed documentation, visit: https://kuttl.dev`, testCmd.Flags().StringVar(&reportName, "report-name", "kuttl-report", "Name for the report. Report location determined by --artifacts-dir and report file type determined by --report.") testCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Namespace to use for tests. Provided namespaces must exist prior to running tests.") testCmd.Flags().StringSliceVar(&suppress, "suppress-log", []string{}, "Suppress logging for these kinds of logs (events).") + testCmd.Flags().Var(&runLabels, "test-run-labels", "Labels to use for this test run.") // This cannot be a global flag because pkg/test/utils.RunTests calls flag.Parse which barfs on unknown top-level flags. // Putting it here at least does not advertise it on a level where using it is impossible. test.SetFlags(testCmd.Flags()) diff --git a/pkg/kuttlctl/cmd/values.go b/pkg/kuttlctl/cmd/values.go new file mode 100644 index 00000000..244891de --- /dev/null +++ b/pkg/kuttlctl/cmd/values.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/labels" +) + +type labelSetValue labels.Set + +func (v *labelSetValue) String() string { + return labels.Set(*v).String() +} + +func (v *labelSetValue) Set(s string) error { + l, err := labels.ConvertSelectorToLabelsMap(s) + if err != nil { + return fmt.Errorf("cannot parse label set: %w", err) + } + *v = labelSetValue(l) + return nil +} + +func (v *labelSetValue) Type() string { + return "labelSet" +} + +func (v labelSetValue) AsLabelSet() labels.Set { + return labels.Set(v) +} diff --git a/pkg/test/case.go b/pkg/test/case.go index 97814b0d..b7093259 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -18,6 +18,7 @@ import ( eventsbeta1 "k8s.io/api/events/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,6 +41,7 @@ type Case struct { SkipDelete bool Timeout int PreferredNamespace string + RunLabels labels.Set Client func(forceNew bool) (client.Client, error) DiscoveryClient func() (discovery.DiscoveryInterface, error) @@ -458,13 +460,14 @@ func (t *Case) LoadTestSteps() error { for index, files := range testStepFiles { testStep := &Step{ - Timeout: t.Timeout, - Index: int(index), - SkipDelete: t.SkipDelete, - Dir: t.Dir, - Asserts: []client.Object{}, - Apply: []client.Object{}, - Errors: []client.Object{}, + Timeout: t.Timeout, + Index: int(index), + SkipDelete: t.SkipDelete, + Dir: t.Dir, + TestRunLabels: t.RunLabels, + Asserts: []client.Object{}, + Apply: []client.Object{}, + Errors: []client.Object{}, } for _, file := range files { diff --git a/pkg/test/case_test.go b/pkg/test/case_test.go index 0f479451..21a9a064 100644 --- a/pkg/test/case_test.go +++ b/pkg/test/case_test.go @@ -1,12 +1,14 @@ package test import ( + "fmt" "testing" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" @@ -18,10 +20,12 @@ import ( func TestLoadTestSteps(t *testing.T) { for _, tt := range []struct { path string + runLabels labels.Set testSteps []Step }{ { - "test_data/with-overrides/", + "test_data/with-overrides", + labels.Set{}, []Step{ { Name: "with-test-step-name-override", @@ -52,7 +56,8 @@ func TestLoadTestSteps(t *testing.T) { "qosClass": "BestEffort", }), }, - Errors: []client.Object{}, + Errors: []client.Object{}, + TestRunLabels: labels.Set{}, }, { Name: "test-assert", @@ -96,7 +101,8 @@ func TestLoadTestSteps(t *testing.T) { "qosClass": "BestEffort", }), }, - Errors: []client.Object{}, + Errors: []client.Object{}, + TestRunLabels: labels.Set{}, }, { Name: "pod", @@ -124,7 +130,8 @@ func TestLoadTestSteps(t *testing.T) { "qosClass": "BestEffort", }), }, - Errors: []client.Object{}, + Errors: []client.Object{}, + TestRunLabels: labels.Set{}, }, { Name: "name-overridden", @@ -164,12 +171,14 @@ func TestLoadTestSteps(t *testing.T) { "restartPolicy": "Never", }), }, - Errors: []client.Object{}, + Errors: []client.Object{}, + TestRunLabels: labels.Set{}, }, }, }, { "test_data/list-pods", + labels.Set{}, []Step{ { Name: "pod", @@ -217,6 +226,101 @@ func TestLoadTestSteps(t *testing.T) { }, }, }, + Errors: []client.Object{}, + TestRunLabels: labels.Set{}, + }, + }, + }, + { + "test_data/test-run-labels", + labels.Set{}, + []Step{ + { + Name: "", + Index: 1, + TestRunLabels: labels.Set{}, + Apply: []client.Object{}, + Asserts: []client.Object{}, + Errors: []client.Object{}, + }, + }, + }, + { + "test_data/test-run-labels", + labels.Set{"flavor": "a"}, + []Step{ + { + Name: "create-a", + Index: 1, + TestRunLabels: labels.Set{"flavor": "a"}, + Apply: []client.Object{ + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test", + }, + "data": map[string]interface{}{ + "flavor": "a", + }, + }, + }, + }, + Asserts: []client.Object{ + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test", + }, + "data": map[string]interface{}{ + "flavor": "a", + }, + }, + }, + }, + Errors: []client.Object{}, + }, + }, + }, + { + "test_data/test-run-labels", + labels.Set{"flavor": "b"}, + []Step{ + { + Name: "create-b", + Index: 1, + TestRunLabels: labels.Set{"flavor": "b"}, + Apply: []client.Object{ + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test", + }, + "data": map[string]interface{}{ + "flavor": "b", + }, + }, + }, + }, + Asserts: []client.Object{ + &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": "test", + }, + "data": map[string]interface{}{ + "flavor": "b", + }, + }, + }, + }, Errors: []client.Object{}, }, }, @@ -224,8 +328,8 @@ func TestLoadTestSteps(t *testing.T) { } { tt := tt - t.Run(tt.path, func(t *testing.T) { - test := &Case{Dir: tt.path, Logger: testutils.NewTestLogger(t, tt.path)} + t.Run(fmt.Sprintf("%s/%s", tt.path, tt.runLabels), func(t *testing.T) { + test := &Case{Dir: tt.path, Logger: testutils.NewTestLogger(t, tt.path), RunLabels: tt.runLabels} err := test.LoadTestSteps() assert.Nil(t, err) @@ -238,11 +342,11 @@ func TestLoadTestSteps(t *testing.T) { assert.Equal(t, len(tt.testSteps), len(testStepsVal)) for index := range tt.testSteps { tt.testSteps[index].Dir = tt.path - assert.Equal(t, tt.testSteps[index].Apply, testStepsVal[index].Apply) - assert.Equal(t, tt.testSteps[index].Asserts, testStepsVal[index].Asserts) - assert.Equal(t, tt.testSteps[index].Errors, testStepsVal[index].Errors) - assert.Equal(t, tt.testSteps[index].Step, testStepsVal[index].Step) - assert.Equal(t, tt.testSteps[index].Dir, testStepsVal[index].Dir) + assert.Equal(t, tt.testSteps[index].Apply, testStepsVal[index].Apply, "apply objects need to match") + assert.Equal(t, tt.testSteps[index].Asserts, testStepsVal[index].Asserts, "assert objects need to match") + assert.Equal(t, tt.testSteps[index].Errors, testStepsVal[index].Errors, "error objects need to match") + assert.Equal(t, tt.testSteps[index].Step, testStepsVal[index].Step, "step object needs to match") + assert.Equal(t, tt.testSteps[index].Dir, testStepsVal[index].Dir, "dir needs to match") assert.Equal(t, tt.testSteps[index], testStepsVal[index]) } }) diff --git a/pkg/test/harness.go b/pkg/test/harness.go index b221a5bf..379ae6a5 100644 --- a/pkg/test/harness.go +++ b/pkg/test/harness.go @@ -18,6 +18,7 @@ import ( volumetypes "github.com/docker/docker/api/types/volume" docker "github.com/docker/docker/client" "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" @@ -53,6 +54,7 @@ type Harness struct { stopping bool bgProcesses []*exec.Cmd report *report.Testsuites + RunLabels labels.Set } // LoadTests loads all of the tests in a given directory. @@ -85,6 +87,7 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) { Dir: filepath.Join(dir, file.Name()), SkipDelete: h.TestSuite.SkipDelete, Suppress: h.TestSuite.Suppress, + RunLabels: h.RunLabels, }) } diff --git a/pkg/test/step.go b/pkg/test/step.go index fa3fb555..ba73c9bc 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -11,7 +11,9 @@ import ( "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" @@ -36,7 +38,8 @@ type Step struct { Index int SkipDelete bool - Dir string + Dir string + TestRunLabels labels.Set Step *harness.TestStep Assert *harness.TestAssert @@ -495,9 +498,9 @@ func (s *Step) String() string { // if seen, mark a test immediately failed. // - All other YAML files are considered resources to create. func (s *Step) LoadYAML(file string) error { - objects, err := testutils.LoadYAMLFromFile(file) - if err != nil { - return fmt.Errorf("loading %s: %s", file, err) + skipFile, objects, err := s.loadOrSkipFile(file) + if skipFile || err != nil { + return err } if err = s.populateObjectsByFileName(filepath.Base(file), objects); err != nil { @@ -579,6 +582,38 @@ func (s *Step) LoadYAML(file string) error { return nil } +func (s *Step) loadOrSkipFile(file string) (bool, []client.Object, error) { + loadedObjects, err := testutils.LoadYAMLFromFile(file) + if err != nil { + return false, nil, fmt.Errorf("loading %s: %s", file, err) + } + + var objects []client.Object + shouldSkip := false + testFileObjEncountered := false + + for i, object := range loadedObjects { + if testFileObject, ok := object.(*harness.TestFile); ok { + if testFileObjEncountered { + return false, nil, fmt.Errorf("more than one TestFile object encountered in file %q", file) + } + testFileObjEncountered = true + selector, err := metav1.LabelSelectorAsSelector(testFileObject.TestRunSelector) + if err != nil { + return false, nil, fmt.Errorf("unrecognized test run selector in object %d of %q: %w", i, file, err) + } + if selector.Empty() || selector.Matches(s.TestRunLabels) { + continue + } + fmt.Printf("Skipping file %q, label selector does not match test run labels.\n", file) + shouldSkip = true + } else { + objects = append(objects, object) + } + } + return shouldSkip, objects, nil +} + // populateObjectsByFileName populates s.Asserts, s.Errors, and/or s.Apply for files containing // "assert", "errors", or no special string, respectively. func (s *Step) populateObjectsByFileName(fileName string, objects []client.Object) error { diff --git a/pkg/test/test_data/test-run-labels/01-assert-a.yaml b/pkg/test/test_data/test-run-labels/01-assert-a.yaml new file mode 100644 index 00000000..46faf4f2 --- /dev/null +++ b/pkg/test/test_data/test-run-labels/01-assert-a.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestFile +testRunSelector: + matchLabels: + flavor: a +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test +data: + flavor: a diff --git a/pkg/test/test_data/test-run-labels/01-assert-b.yaml b/pkg/test/test_data/test-run-labels/01-assert-b.yaml new file mode 100644 index 00000000..5d8e7be8 --- /dev/null +++ b/pkg/test/test_data/test-run-labels/01-assert-b.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestFile +testRunSelector: + matchLabels: + flavor: b +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test +data: + flavor: b diff --git a/pkg/test/test_data/test-run-labels/01-create-a.yaml b/pkg/test/test_data/test-run-labels/01-create-a.yaml new file mode 100644 index 00000000..559453e5 --- /dev/null +++ b/pkg/test/test_data/test-run-labels/01-create-a.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestFile +testRunSelector: + matchLabels: + flavor: a +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test +data: + flavor: a \ No newline at end of file diff --git a/pkg/test/test_data/test-run-labels/01-create-b.yaml b/pkg/test/test_data/test-run-labels/01-create-b.yaml new file mode 100644 index 00000000..ece5b5f5 --- /dev/null +++ b/pkg/test/test_data/test-run-labels/01-create-b.yaml @@ -0,0 +1,12 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestFile +testRunSelector: + matchLabels: + flavor: b +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test +data: + flavor: b \ No newline at end of file diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index ebcdcce7..b7706de7 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -400,6 +400,8 @@ func ConvertUnstructured(in client.Object) (client.Object, error) { return in, nil } switch { + case kind == "TestFile": + converted = &harness.TestFile{} case kind == "TestStep": converted = &harness.TestStep{} case kind == "TestAssert":