From 5eb66d514e01cef1d967fa94ca3d860296a98cd0 Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Tue, 24 Nov 2020 16:24:34 +0100 Subject: [PATCH 1/3] e2e: update explain tests with missing types --- test/extended/cli/explain.go | 123 ++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 44 deletions(-) diff --git a/test/extended/cli/explain.go b/test/extended/cli/explain.go index 6c1be24b51c0..41cf57abae85 100644 --- a/test/extended/cli/explain.go +++ b/test/extended/cli/explain.go @@ -34,6 +34,7 @@ var ( {Group: "image.openshift.io", Version: "v1", Resource: "imagestreamimports"}, {Group: "image.openshift.io", Version: "v1", Resource: "imagestreams"}, + {Group: "image.openshift.io", Version: "v1", Resource: "imagetags"}, {Group: "project.openshift.io", Version: "v1", Resource: "projects"}, @@ -47,18 +48,39 @@ var ( } crdTypes = []schema.GroupVersionResource{ - // FIXME - // {Group: "operators.coreos.com", Version: "v1", Resource: "catalogsourceconfigs"}, - // {Group: "operators.coreos.com", Version: "v1", Resource: "catalogsources"}, - // {Group: "operators.coreos.com", Version: "v1", Resource: "clusterserviceversions"}, - // {Group: "operators.coreos.com", Version: "v1", Resource: "installplans"}, - // {Group: "operators.coreos.com", Version: "v1", Resource: "operatorgroups"}, - // {Group: "operators.coreos.com", Version: "v1", Resource: "operatorsources"}, - // {Group: "operators.coreos.com", Version: "v1", Resource: "subscriptions"}, - - // FIXME - // {Group: "autoscaling.openshift.io", Version: "v1", Resource: "clusterautoscalers"}, - // {Group: "autoscaling.openshift.io", Version: "v1", Resource: "machineautoscalers"}, + + // coreos.com groups: + + {Group: "monitoring.coreos.com", Version: "v1alpha1", Resource: "alertmanagerconfigs"}, + + {Group: "monitoring.coreos.com", Version: "v1", Resource: "alertmanagers"}, + {Group: "monitoring.coreos.com", Version: "v1", Resource: "probes"}, + {Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheuses"}, + {Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"}, + {Group: "monitoring.coreos.com", Version: "v1", Resource: "podmonitors"}, + {Group: "monitoring.coreos.com", Version: "v1", Resource: "servicemonitors"}, + {Group: "monitoring.coreos.com", Version: "v1", Resource: "thanosrulers"}, + + {Group: "operators.coreos.com", Version: "v1", Resource: "operators"}, + {Group: "operators.coreos.com", Version: "v1", Resource: "operatorgroups"}, + + {Group: "operators.coreos.com", Version: "v1alpha1", Resource: "catalogsources"}, + {Group: "operators.coreos.com", Version: "v1alpha1", Resource: "clusterserviceversions"}, + {Group: "operators.coreos.com", Version: "v1alpha1", Resource: "installplans"}, + {Group: "operators.coreos.com", Version: "v1alpha1", Resource: "subscriptions"}, + {Group: "operators.coreos.com", Version: "v1alpha2", Resource: "operatorgroups"}, + + {Group: "packages.operators.coreos.com", Version: "v1", Resource: "packagemanifests"}, + + // metal3.io groups: + + {Group: "metal3.io", Version: "v1alpha1", Resource: "baremetalhosts"}, + {Group: "metal3.io", Version: "v1alpha1", Resource: "provisionings"}, + + // openshift.io groups: + + {Group: "autoscaling.openshift.io", Version: "v1beta1", Resource: "machineautoscalers"}, + {Group: "autoscaling.openshift.io", Version: "v1", Resource: "clusterautoscalers"}, {Group: "authorization.openshift.io", Version: "v1", Resource: "selfsubjectrulesreviews"}, {Group: "authorization.openshift.io", Version: "v1", Resource: "subjectrulesreviews"}, @@ -81,24 +103,18 @@ var ( {Group: "config.openshift.io", Version: "v1", Resource: "proxies"}, {Group: "config.openshift.io", Version: "v1", Resource: "schedulers"}, - // FIXME - // {Group: "cloudcredential.openshift.io", Version: "v1", Resource: "credentialsrequests"}, + {Group: "cloudcredential.openshift.io", Version: "v1", Resource: "credentialsrequests"}, + + {Group: "helm.openshift.io", Version: "v1beta1", Resource: "helmchartrepositories"}, - // FIXME - // {Group: "healthchecking.openshift.io", Version: "v1", Resource: "machinehealthchecks"}, - // {Group: "imageregistry.operator.openshift", Version: "v1", Resource: "configs"}, + {Group: "imageregistry.operator.openshift.io", Version: "v1", Resource: "configs"}, + {Group: "imageregistry.operator.openshift.io", Version: "v1", Resource: "imagepruners"}, - // FIXME - // {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machinedisruptionbudgets"}, - // {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machinehealthchecks"}, - // {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machines"}, - // {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machinesets"}, + {Group: "ingress.operator.openshift.io", Version: "v1", Resource: "dnsrecords"}, - // FIXME - // {Group: "monitoring.coreos.com", Version: "v1", Resource: "alertmanagers"}, - // {Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheuses"}, - // {Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"}, - // {Group: "monitoring.coreos.com", Version: "v1", Resource: "servicemonitors"}, + {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machinehealthchecks"}, + {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machines"}, + {Group: "machine.openshift.io", Version: "v1beta1", Resource: "machinesets"}, {Group: "machineconfiguration.openshift.io", Version: "v1", Resource: "containerruntimeconfigs"}, {Group: "machineconfiguration.openshift.io", Version: "v1", Resource: "controllerconfigs"}, @@ -108,39 +124,38 @@ var ( {Group: "operator.openshift.io", Version: "v1alpha1", Resource: "imagecontentsourcepolicies"}, - {Group: "operator.openshift.io", Version: "v1", Resource: "consoles"}, - {Group: "operator.openshift.io", Version: "v1", Resource: "openshiftapiservers"}, - - // FIXME - // {Group: "operator.openshift.io", Version: "v1", Resource: "credentialsrequestses"}, {Group: "operator.openshift.io", Version: "v1", Resource: "authentications"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "cloudcredentials"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "clustercsidrivers"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "configs"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "consoles"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "csisnapshotcontrollers"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "dnses"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "etcds"}, {Group: "operator.openshift.io", Version: "v1", Resource: "ingresscontrollers"}, {Group: "operator.openshift.io", Version: "v1", Resource: "kubeapiservers"}, {Group: "operator.openshift.io", Version: "v1", Resource: "kubecontrollermanagers"}, {Group: "operator.openshift.io", Version: "v1", Resource: "kubeschedulers"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "kubestorageversionmigrators"}, {Group: "operator.openshift.io", Version: "v1", Resource: "networks"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "openshiftapiservers"}, {Group: "operator.openshift.io", Version: "v1", Resource: "openshiftcontrollermanagers"}, {Group: "operator.openshift.io", Version: "v1", Resource: "servicecas"}, + {Group: "operator.openshift.io", Version: "v1", Resource: "storages"}, {Group: "quota.openshift.io", Version: "v1", Resource: "appliedclusterresourcequotas"}, - {Group: "quota.openshift.io", Version: "v1", Resource: "clusterresourcequotas"}, - // FIXME - // {Group: "imageregistry.operator.openshift.io", Version: "v1", Resource: "configs"}, - - // FIXME - // {Group: "ingress.operator.openshift.io", Version: "v1", Resource: "dnsrecords"}, - {Group: "samples.operator.openshift.io", Version: "v1", Resource: "configs"}, + {Group: "tuned.openshift.io", Version: "v1", Resource: "profiles"}, {Group: "tuned.openshift.io", Version: "v1", Resource: "tuneds"}, } specialTypes = []explainExceptions{ { gv: schema.GroupVersion{Group: "apps.openshift.io", Version: "v1"}, - field: "dc.status.replicas", + field: "deploymentconfigs.status.replicas", pattern: `FIELD\: +replicas`, }, { @@ -223,6 +238,11 @@ var ( field: "imagestreamtags.tag", pattern: `FIELDS\:.*`, }, + { + gv: schema.GroupVersion{Group: "security.internal.openshift.io", Version: "v1"}, + field: "rangeallocations.range", + pattern: `FIELD\: +range`, + }, { gv: schema.GroupVersion{Group: "oauth.openshift.io", Version: "v1"}, field: "oauthaccesstokens.refreshToken", @@ -253,6 +273,11 @@ var ( field: "rangeallocations.range", pattern: `FIELD\: +range`, }, + { + gv: schema.GroupVersion{Group: "template.openshift.io", Version: "v1"}, + field: "brokertemplateinstances.spec", + pattern: `DESCRIPTION\:.*`, + }, { gv: schema.GroupVersion{Group: "template.openshift.io", Version: "v1"}, field: "processedtemplates.objects", @@ -313,27 +338,37 @@ var ( field: "consoleyamlsamples.spec", pattern: `DESCRIPTION\:.*`, }, + { + gv: schema.GroupVersion{Group: "console.openshift.io", Version: "v1"}, + field: "consolequickstarts.spec", + pattern: `DESCRIPTION\:.*`, + }, } specialNetworkingTypes = []explainExceptions{ + { + gv: schema.GroupVersion{Group: "network.operator.openshift.io", Version: "v1"}, + field: "operatorpkis.spec", + pattern: `DESCRIPTION\:.*`, + }, { gv: schema.GroupVersion{Group: "network.openshift.io", Version: "v1"}, - field: "clusternetworks", + field: "clusternetworks.clusterNetworks", pattern: `DESCRIPTION\:.*`, }, { gv: schema.GroupVersion{Group: "network.openshift.io", Version: "v1"}, - field: "hostsubnets", + field: "hostsubnets.subnet", pattern: `DESCRIPTION\:.*`, }, { gv: schema.GroupVersion{Group: "network.openshift.io", Version: "v1"}, - field: "netnamespaces", + field: "netnamespaces.netname", pattern: `DESCRIPTION\:.*`, }, { gv: schema.GroupVersion{Group: "network.openshift.io", Version: "v1"}, - field: "egressnetworkpolicies", + field: "egressnetworkpolicies.spec", pattern: `DESCRIPTION\:.*`, }, } From d2fb93ee78dc400bc1800bc03695f206a24fb474 Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Tue, 24 Nov 2020 16:25:53 +0100 Subject: [PATCH 2/3] e2e: add new test which looks for not covered types --- test/extended/cli/explain.go | 49 +++++++++++++++++++ .../generated/zz_generated.annotations.go | 2 + 2 files changed, 51 insertions(+) diff --git a/test/extended/cli/explain.go b/test/extended/cli/explain.go index 41cf57abae85..5594b6f5e9c7 100644 --- a/test/extended/cli/explain.go +++ b/test/extended/cli/explain.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "regexp" + "strings" g "github.com/onsi/ginkgo" o "github.com/onsi/gomega" @@ -13,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + kclientset "k8s.io/client-go/kubernetes" e2e "k8s.io/kubernetes/test/e2e/framework" "github.com/openshift/origin/test/extended/networking" @@ -379,6 +381,53 @@ var _ = g.Describe("[sig-cli] oc explain", func() { oc := exutil.NewCLI("oc-explain") + g.It("list uncovered GroupVersionResources", func() { + resourceMap := make(map[schema.GroupVersionResource]bool) + kubeClient := kclientset.NewForConfigOrDie(oc.AdminConfig()) + _, resourceList, err := kubeClient.Discovery().ServerGroupsAndResources() + if err != nil { + e2e.Failf("Failed reading groups and resources %v", err) + } + for _, rl := range resourceList { + for _, r := range rl.APIResources { + gv, err := schema.ParseGroupVersion(rl.GroupVersion) + if err != nil { + e2e.Failf("Couldn't parse GroupVersion for %s: %v", gv, err) + } + resourceMap[gv.WithResource(r.Name)] = true + } + } + + for _, bt := range builtinTypes { + delete(resourceMap, bt) + } + for _, ct := range crdTypes { + delete(resourceMap, ct) + } + for _, st := range specialTypes { + resource := strings.Split(st.field, ".") + delete(resourceMap, st.gv.WithResource(resource[0])) + } + for _, snt := range specialNetworkingTypes { + resource := strings.Split(snt.field, ".") + delete(resourceMap, snt.gv.WithResource(resource[0])) + } + + e2e.Logf("These GroupVersionResources are missing proper explain test:") + for k := range resourceMap { + // ignore all k8s built-ins and sub-resources + if k.Group == "" || strings.Contains(k.Group, "k8s.io") || + k.Group == "apps" || k.Group == "autoscaling" || + k.Group == "batch" || k.Group == "extensions" || + k.Group == "policy" || + strings.Contains(k.Group, "cncf.io") || + strings.Contains(k.Resource, "/") { + continue + } + e2e.Logf(" - %s", k) + } + }) + g.It("should contain spec+status for builtinTypes", func() { for _, bt := range builtinTypes { e2e.Logf("Checking %s...", bt) diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index e487b45ab449..9de058ffe8a9 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1491,6 +1491,8 @@ var annotations = map[string]string{ "[Top Level] [sig-cli] oc debug ensure it works with image streams": "ensure it works with image streams [Suite:openshift/conformance/parallel]", + "[Top Level] [sig-cli] oc explain list uncovered GroupVersionResources": "list uncovered GroupVersionResources [Suite:openshift/conformance/parallel]", + "[Top Level] [sig-cli] oc explain networking types when using openshift-sdn should contain proper fields description for special networking types": "should contain proper fields description for special networking types [Suite:openshift/conformance/parallel]", "[Top Level] [sig-cli] oc explain should contain proper fields description for special types": "should contain proper fields description for special types [Suite:openshift/conformance/parallel]", From 3e7a12aa02c8f87db7904d88f5462b65f22ac399 Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Tue, 24 Nov 2020 16:27:05 +0100 Subject: [PATCH 3/3] e2e: scrape CRD yaml when we fail regexp match --- test/extended/cli/explain.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/extended/cli/explain.go b/test/extended/cli/explain.go index 5594b6f5e9c7..580f81943705 100644 --- a/test/extended/cli/explain.go +++ b/test/extended/cli/explain.go @@ -446,7 +446,9 @@ var _ = g.Describe("[sig-cli] oc explain", func() { g.It("should contain proper fields description for special types", func() { for _, st := range specialTypes { e2e.Logf("Checking %s, Field=%s...", st.gv, st.field) - o.Expect(verifyExplain(oc, nil, schema.GroupVersionResource{}, + resource := strings.Split(st.field, ".") + gvr := st.gv.WithResource(resource[0]) + o.Expect(verifyExplain(oc, nil, gvr, st.pattern, st.field, fmt.Sprintf("--api-version=%s", st.gv))).NotTo(o.HaveOccurred()) } }) @@ -459,9 +461,12 @@ var _ = g.Describe("[sig-cli] oc explain networking types", func() { networking.InOpenShiftSDNContext(func() { g.It("should contain proper fields description for special networking types", func() { + crdClient := apiextensionsclientset.NewForConfigOrDie(oc.AdminConfig()) for _, st := range specialNetworkingTypes { e2e.Logf("Checking %s, Field=%s...", st.gv, st.field) - o.Expect(verifyExplain(oc, nil, schema.GroupVersionResource{}, + resource := strings.Split(st.field, ".") + gvr := st.gv.WithResource(resource[0]) + o.Expect(verifyExplain(oc, crdClient, gvr, st.pattern, st.field, fmt.Sprintf("--api-version=%s", st.gv))).NotTo(o.HaveOccurred()) } })