[wip] test: experiment with size labels for resource-aware parallelism

pkg/test/ginkgo/cmd_runsuite.go

@@ -437,53 +437,55 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		return strings.Contains(t.name, "[sig-storage]")
 	})
 
-	networkK8sTests, kubeTests := splitTests(kubeTests, func(t *testCase) bool {
-		return strings.Contains(t.name, "[sig-network]")
-	})
-
 	networkTests, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool {
 		return strings.Contains(t.name, "[sig-network]")
 	})
 
-	buildsTests, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool {
-		return strings.Contains(t.name, "[sig-builds]")
-	})
-
 	// separate from cliTests
 	mustGatherTests, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool {
 		return strings.Contains(t.name, "[sig-cli] oc adm must-gather")
 	})
 
-	logrus.Infof("Found %d openshift tests", len(openshiftTests))
+	// Split openshift tests by t-shirt size
+	openshiftTestsL, openshiftTests := splitTests(openshiftTests, func(t *testCase) bool {
+		return strings.Contains(t.name, "Size:L")
+	})
+
+	openshiftTestsS, openshiftTestsM := splitTests(openshiftTests, func(t *testCase) bool {
+		return strings.Contains(t.name, "Size:S")
+	})
+	// openshiftTestsM now contains Size:M and tests without size labels
+
+	logrus.Infof("Found %d openshift Size:S tests", len(openshiftTestsS))
+	logrus.Infof("Found %d openshift Size:M tests (includes unlabeled)", len(openshiftTestsM))
+	logrus.Infof("Found %d openshift Size:L tests", len(openshiftTestsL))
 	logrus.Infof("Found %d kubernetes tests", len(kubeTests))
 	logrus.Infof("Found %d storage tests", len(storageTests))
-	logrus.Infof("Found %d network k8s tests", len(networkK8sTests))
 	logrus.Infof("Found %d network tests", len(networkTests))
-	logrus.Infof("Found %d builds tests", len(buildsTests))
 	logrus.Infof("Found %d must-gather tests", len(mustGatherTests))
 
 	// If user specifies a count, duplicate the kube and openshift tests that many times.
 	expectedTestCount := len(early) + len(late)
 	if count != -1 {
 		originalKube := kubeTests
-		originalOpenshift := openshiftTests
+		originalOpenshiftS := openshiftTestsS
+		originalOpenshiftM := openshiftTestsM
+		originalOpenshiftL := openshiftTestsL
 		originalStorage := storageTests
-		originalNetworkK8s := networkK8sTests
 		originalNetwork := networkTests
-		originalBuilds := buildsTests
 		originalMustGather := mustGatherTests
 
 		for i := 1; i < count; i++ {
 			kubeTests = append(kubeTests, copyTests(originalKube)...)
-			openshiftTests = append(openshiftTests, copyTests(originalOpenshift)...)
+			openshiftTestsS = append(openshiftTestsS, copyTests(originalOpenshiftS)...)
+			openshiftTestsM = append(openshiftTestsM, copyTests(originalOpenshiftM)...)
+			openshiftTestsL = append(openshiftTestsL, copyTests(originalOpenshiftL)...)
 			storageTests = append(storageTests, copyTests(originalStorage)...)
-			networkK8sTests = append(networkK8sTests, copyTests(originalNetworkK8s)...)
 			networkTests = append(networkTests, copyTests(originalNetwork)...)
-			buildsTests = append(buildsTests, copyTests(originalBuilds)...)
 			mustGatherTests = append(mustGatherTests, copyTests(originalMustGather)...)
 		}
 	}
-	expectedTestCount += len(openshiftTests) + len(kubeTests) + len(storageTests) + len(networkK8sTests) + len(networkTests) + len(buildsTests) + len(mustGatherTests)
+	expectedTestCount += len(openshiftTestsS) + len(openshiftTestsM) + len(openshiftTestsL) + len(kubeTests) + len(storageTests) + len(networkTests) + len(mustGatherTests)
 
 	abortFn := neverAbort
 	testCtx := ctx
@@ -501,7 +503,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 	// TODO: will move to the monitor
 	pc.SetEvents([]string{upgradeEvent})
 
-	// Run kube, storage, openshift, and must-gather tests. If user specified a count of -1,
+	// Run kube, storage, openshift (by size), and must-gather tests. If user specified a count of -1,
 	// we loop indefinitely.
 	for i := 0; (i < 1 || count == -1) && testCtx.Err() == nil; i++ {
 
@@ -514,21 +516,22 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		q.Execute(testCtx, storageTestsCopy, max(1, parallelism/2), testOutputConfig, abortFn) // storage tests only run at half the parallelism, so we can avoid cloud provider quota problems.
 		tests = append(tests, storageTestsCopy...)
 
-		networkK8sTestsCopy := copyTests(networkK8sTests)
-		q.Execute(testCtx, networkK8sTestsCopy, max(1, parallelism/2), testOutputConfig, abortFn) // run network tests separately.
-		tests = append(tests, networkK8sTestsCopy...)
-
 		networkTestsCopy := copyTests(networkTests)
 		q.Execute(testCtx, networkTestsCopy, max(1, parallelism/2), testOutputConfig, abortFn) // run network tests separately.
 		tests = append(tests, networkTestsCopy...)
 
-		buildsTestsCopy := copyTests(buildsTests)
-		q.Execute(testCtx, buildsTestsCopy, max(1, parallelism/2), testOutputConfig, abortFn) // builds tests only run at half the parallelism, so we can avoid high cpu problems.
-		tests = append(tests, buildsTestsCopy...)
+		// Run openshift tests by size with appropriate parallelism
+		openshiftTestsSCopy := copyTests(openshiftTestsS)
+		q.Execute(testCtx, openshiftTestsSCopy, max(1, parallelism*2), testOutputConfig, abortFn) // Size:S tests run at 2x parallelism
+		tests = append(tests, openshiftTestsSCopy...)
+
+		openshiftTestsMCopy := copyTests(openshiftTestsM)
+		q.Execute(testCtx, openshiftTestsMCopy, parallelism, testOutputConfig, abortFn) // Size:M tests run at 1x parallelism
+		tests = append(tests, openshiftTestsMCopy...)
 
-		openshiftTestsCopy := copyTests(openshiftTests)
-		q.Execute(testCtx, openshiftTestsCopy, parallelism, testOutputConfig, abortFn)
-		tests = append(tests, openshiftTestsCopy...)
+		openshiftTestsLCopy := copyTests(openshiftTestsL)
+		q.Execute(testCtx, openshiftTestsLCopy, max(1, parallelism/2), testOutputConfig, abortFn) // Size:L tests run at 1/2 parallelism
+		tests = append(tests, openshiftTestsLCopy...)
 
 		// run the must-gather tests after parallel tests to reduce resource contention
 		mustGatherTestsCopy := copyTests(mustGatherTests)

test/e2e/upgrade/upgrade.go

@@ -137,15 +137,15 @@ var _ = g.Describe("[sig-arch][Feature:ClusterUpgrade]", func() {
 	f := framework.NewDefaultFramework("cluster-upgrade")
 	f.SkipNamespaceCreation = true
 
-	g.It("Cluster should be upgradeable before beginning upgrade [Early][Suite:upgrade]", func() {
+	g.It("Cluster should be upgradeable before beginning upgrade [Early][Suite:upgrade]", g.Label("Size:S"), func() {
 		config, err := framework.LoadConfig()
 		framework.ExpectNoError(err)
 		client := configv1client.NewForConfigOrDie(config)
 		err = checkUpgradeability(client)
 		framework.ExpectNoError(err)
 	})
 
-	g.It("All nodes should be in ready state [Early][Suite:upgrade]", func() {
+	g.It("All nodes should be in ready state [Early][Suite:upgrade]", g.Label("Size:S"), func() {
 		config, err := framework.LoadConfig()
 		framework.ExpectNoError(err)
 		client := kubernetes.NewForConfigOrDie(config)
@@ -169,7 +169,7 @@ var _ = g.Describe("[sig-arch][Feature:ClusterUpgrade]", func() {
 		}
 	})
 
-	g.It("Cluster should remain functional during upgrade [Disruptive]", func() {
+	g.It("Cluster should remain functional during upgrade [Disruptive]", g.Label("Size:L"), func() {
 		config, err := framework.LoadConfig()
 		framework.ExpectNoError(err)
 		client := configv1client.NewForConfigOrDie(config)
@@ -197,7 +197,7 @@ var _ = g.Describe("[sig-arch][Feature:ClusterUpgrade]", func() {
 		)
 	})
 
-	g.It("Cluster should be upgradeable after finishing upgrade [Late][Suite:upgrade]", func() {
+	g.It("Cluster should be upgradeable after finishing upgrade [Late][Suite:upgrade]", g.Label("Size:S"), func() {
 		config, err := framework.LoadConfig()
 		framework.ExpectNoError(err)
 		client := configv1client.NewForConfigOrDie(config)

test/extended/adminack/adminack.go

@@ -17,7 +17,7 @@ var _ = g.Describe("[sig-cluster-lifecycle]", func() {
 	oc := exutil.NewCLI("cli-deployment")
 
 	g.Describe("TestAdminAck", func() {
-		g.It("should succeed [apigroup:config.openshift.io]", func() {
+		g.It("should succeed [apigroup:config.openshift.io]", g.Label("Size:M"), func() {
 			config, err := framework.LoadConfig()
 			o.Expect(err).NotTo(o.HaveOccurred())
 			ctx := context.Background()

test/extended/apiserver/access.go

@@ -33,7 +33,7 @@ var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] kube-api
 		"api-int":         "api-int",
 		"api-ext":         "api-ext",
 	} {
-		g.It(fmt.Sprintf("%s endpoint", description), func() {
+		g.It(fmt.Sprintf("%s endpoint", description), g.Label("Size:M"), func() {
 			// skip on microshift
 			isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
 			o.Expect(err).NotTo(o.HaveOccurred())

test/extended/apiserver/api_requests.go

@@ -36,7 +36,7 @@ var _ = g.Describe("[sig-arch][Late]", func() {
 
 	oc := exutil.NewCLIWithoutNamespace("api-requests")
 
-	g.It("clients should not use APIs that are removed in upcoming releases [apigroup:apiserver.openshift.io]", func() {
+	g.It("clients should not use APIs that are removed in upcoming releases [apigroup:apiserver.openshift.io]", g.Label("Size:S"), func() {
 		ctx := context.Background()
 		adminConfig := oc.AdminConfig()
 

test/extended/apiserver/apply.go

@@ -66,7 +66,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:ServerSideApply] Server-Side App
 			continue
 		}
 
-		g.It(fmt.Sprintf("should work for %s [apigroup:%s]", gvr, gvr.Group), func() {
+		g.It(fmt.Sprintf("should work for %s [apigroup:%s]", gvr, gvr.Group), g.Label("Size:M"), func() {
 			// create the testing namespace
 			testNamespace := oc.SetupProject()
 

test/extended/apiserver/config_admission_test.go

@@ -21,7 +21,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() {
 
 	oc := exutil.NewCLI("apiserver")
 
-	g.It("validates APIServer in config.openshift.io/v1", func() {
+	g.It("validates APIServer in config.openshift.io/v1", g.Label("Size:S"), func() {
 		client, err := dynamic.NewForConfig(oc.AdminConfig())
 		if err != nil {
 			g.Fail(fmt.Sprintf("Unexpected error: %v", err))

test/extended/apiserver/graceful_termination.go

@@ -26,7 +26,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Late]", func() {
 	// This test checks whether the apiserver reports any events that may indicate a problem at any time,
 	// not just when the suite is running. We already have invariant tests that fail if these are violated
 	// during suite execution, but we want to know if there are fingerprints of these failures outside of tests.
-	g.It("kubelet terminates kube-apiserver gracefully", func() {
+	g.It("kubelet terminates kube-apiserver gracefully", g.Label("Size:S"), func() {
 		client, err := kubernetes.NewForConfig(oc.AdminConfig())
 		if err != nil {
 			g.Fail(fmt.Sprintf("Unexpected error: %v", err))
@@ -57,7 +57,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Late]", func() {
 	// This test extends the previous test by checking the content of the termination files for kube-apiservers.
 	// It should catch cases where the event is not persisted in the database. It should also catch
 	// cases where the KAS is immediately restarted or shut down after an ungraceful termination.
-	g.It("kubelet terminates kube-apiserver gracefully extended", func() {
+	g.It("kubelet terminates kube-apiserver gracefully extended", g.Label("Size:M"), func() {
 		var finalMessageBuilder strings.Builder
 		terminationRegexp := regexp.MustCompile(`Previous pod .* did not terminate gracefully`)
 
@@ -93,7 +93,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Late]", func() {
 	// This test checks whether the apiserver reports any events that may indicate a problem at any time,
 	// not just when the suite is running. We already have invariant tests that fail if these are violated
 	// during suite execution, but we want to know if there are fingerprints of these failures outside of tests.
-	g.It("kube-apiserver terminates within graceful termination period", func() {
+	g.It("kube-apiserver terminates within graceful termination period", g.Label("Size:S"), func() {
 		client, err := kubernetes.NewForConfig(oc.AdminConfig())
 		if err != nil {
 			g.Fail(fmt.Sprintf("Unexpected error: %v", err))
@@ -121,7 +121,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Late]", func() {
 		}
 	})
 
-	g.It("API LBs follow /readyz of kube-apiserver and stop sending requests", func() {
+	g.It("API LBs follow /readyz of kube-apiserver and stop sending requests", g.Label("Size:S"), func() {
 		t := g.GinkgoT()
 
 		client, err := kubernetes.NewForConfig(oc.AdminConfig())
@@ -147,7 +147,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer][Late]", func() {
 		}
 	})
 
-	g.It("API LBs follow /readyz of kube-apiserver and don't send request early", func() {
+	g.It("API LBs follow /readyz of kube-apiserver and don't send request early", g.Label("Size:S"), func() {
 		t := g.GinkgoT()
 
 		client, err := kubernetes.NewForConfig(oc.AdminConfig())

test/extended/apiserver/health_endpoints.go

@@ -23,7 +23,7 @@ var _ = ginkgo.Describe("[sig-api-machinery] API health endpoints", func() {
 
 	oc := exutil.NewCLIWithoutNamespace("api-health-endpoints").AsAdmin()
 
-	ginkgo.It("should contain the required checks for the openshift-apiserver APIs", func() {
+	ginkgo.It("should contain the required checks for the openshift-apiserver APIs", ginkgo.Label("Size:S"), func() {
 		ctx := context.Background()
 		supported, msg := isSupportedPlatform(ctx, "openshift-apiserver", oc)
 		if !supported {
@@ -86,7 +86,7 @@ var _ = ginkgo.Describe("[sig-api-machinery] API health endpoints", func() {
 		return
 	})
 
-	ginkgo.It("should contain the required checks for the oauth-apiserver APIs", func() {
+	ginkgo.It("should contain the required checks for the oauth-apiserver APIs", ginkgo.Label("Size:S"), func() {
 		ctx := context.Background()
 		supported, msg := isSupportedPlatform(ctx, "oauth-apiserver", oc)
 		if !supported {

test/extended/apiserver/kubeconfigs.go

@@ -36,13 +36,13 @@ var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local ku
 	oc := exutil.NewCLIWithPodSecurityLevel("apiserver", admissionapi.LevelPrivileged)
 
 	for kubeconfig := range kcLocations {
-		g.It(fmt.Sprintf("%q should be present on all masters and work", kubeconfig), func() {
+		g.It(fmt.Sprintf("%q should be present on all masters and work", kubeconfig), g.Label("Size:M"), func() {
 			testKubeConfig(oc, kubeconfig, testNode)
 		})
 	}
 
 	for kubeconfig := range kubeApiserverLocations {
-		g.It(fmt.Sprintf("%q should be present in all kube-apiserver containers", kubeconfig), func() {
+		g.It(fmt.Sprintf("%q should be present in all kube-apiserver containers", kubeconfig), g.Label("Size:M"), func() {
 			// skip on microshift
 			isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
 			o.Expect(err).NotTo(o.HaveOccurred())

test/extended/apiserver/openapiv3.go

@@ -27,7 +27,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() {
 
 	oc := exutil.NewCLIWithoutNamespace("apiserver-openapi")
 
-	g.It("should serve openapi v3 discovery", func() {
+	g.It("should serve openapi v3 discovery", g.Label("Size:S"), func() {
 		transport, err := rest.TransportFor(oc.AdminConfig())
 		o.Expect(err).NotTo(o.HaveOccurred())
 
@@ -54,7 +54,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() {
 		}
 	})
 
-	g.It("should serve openapi v3", func() {
+	g.It("should serve openapi v3", g.Label("Size:M"), func() {
 		transport, err := rest.TransportFor(oc.AdminConfig())
 		o.Expect(err).NotTo(o.HaveOccurred())
 

test/extended/apiserver/patch.go

@@ -38,7 +38,7 @@ var _ = g.Describe("[sig-api-machinery] JSON Patch [apigroup:operator.openshift.
 		}
 	})
 
-	g.It("should delete an entry from an array with a test precondition provided", func() {
+	g.It("should delete an entry from an array with a test precondition provided", g.Label("Size:S"), func() {
 		g.By("Creating KubeAPIServerOperator CR for the test")
 		resourceClient := createResourceClient(oc.AdminConfig(), gvr)
 		kasOperator := createWellKnownKubeAPIServerOperatorResource(ctx, resourceClient)
@@ -51,7 +51,7 @@ var _ = g.Describe("[sig-api-machinery] JSON Patch [apigroup:operator.openshift.
 			{NodeName: "master-1"},
 		}))
 	})
-	g.It("should delete multiple entries from an array when multiple test precondition provided", func() {
+	g.It("should delete multiple entries from an array when multiple test precondition provided", g.Label("Size:S"), func() {
 		g.By("Creating KubeAPIServerOperator CR for the test")
 		resourceClient := createResourceClient(oc.AdminConfig(), gvr)
 		kasOperator := createWellKnownKubeAPIServerOperatorResource(ctx, resourceClient)
@@ -64,7 +64,7 @@ var _ = g.Describe("[sig-api-machinery] JSON Patch [apigroup:operator.openshift.
 		o.Expect(err).NotTo(o.HaveOccurred())
 		o.Expect(kasOperator.Status.NodeStatuses).To(o.HaveLen(0))
 	})
-	g.It("should error when the test precondition provided doesn't match", func() {
+	g.It("should error when the test precondition provided doesn't match", g.Label("Size:S"), func() {
 		g.By("Creating KubeAPIServerOperator CR for the test")
 		resourceClient := createResourceClient(oc.AdminConfig(), gvr)
 		kasOperator := createWellKnownKubeAPIServerOperatorResource(ctx, resourceClient)
@@ -75,7 +75,7 @@ var _ = g.Describe("[sig-api-machinery] JSON Patch [apigroup:operator.openshift.
 		o.Expect(k8serrors.IsInvalid(err)).To(o.BeTrue(), fmt.Sprintf("unexpected error received = %v", err))
 	})
 
-	g.It("should delete an entry from an array with multiple field owners", func() {
+	g.It("should delete an entry from an array with multiple field owners", g.Label("Size:S"), func() {
 		g.By("Creating KubeAPIServerOperator CR for the test")
 		resourceClient := createResourceClient(oc.AdminConfig(), gvr)
 		kasOperator := createWellKnownKubeAPIServerOperatorResource(ctx, resourceClient)

test/extended/apiserver/resiliency.go

@@ -34,7 +34,7 @@ var _ = ginkgo.Describe("[Conformance][sig-sno][Serial] Cluster", func() {
 
 	oc := exutil.NewCLIWithoutNamespace("cluster-resiliency")
 
-	ginkgo.It("should allow a fast rollout of kube-apiserver with no pods restarts during API disruption [apigroup:config.openshift.io][apigroup:operator.openshift.io]", func() {
+	ginkgo.It("should allow a fast rollout of kube-apiserver with no pods restarts during API disruption [apigroup:config.openshift.io][apigroup:operator.openshift.io]", ginkgo.Label("Size:L"), func() {
 		ctx := context.Background()
 
 		controlPlaneTopology, _ := single_node.GetTopologies(f)

test/extended/apiserver/rollout.go

@@ -39,7 +39,7 @@ var _ = ginkgo.Describe("[Conformance][Suite:openshift/kube-apiserver/rollout][J
 
 	oc := exutil.NewCLIWithoutNamespace("rollout-resiliency")
 
-	ginkgo.It("should roll out new revisions without disruption [apigroup:config.openshift.io][apigroup:operator.openshift.io]", func() {
+	ginkgo.It("should roll out new revisions without disruption [apigroup:config.openshift.io][apigroup:operator.openshift.io]", ginkgo.Label("Size:L"), func() {
 		ctx := context.Background()
 
 		// separate context so we exit our loop, but it is still possible to use the main context for client calls

test/extended/apiserver/root_403.go

@@ -22,7 +22,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() {
 
 	oc := exutil.NewCLIWithoutNamespace("apiserver")
 
-	g.It("anonymous browsers should get a 403 from /", func() {
+	g.It("anonymous browsers should get a 403 from /", g.Label("Size:S"), func() {
 		transport, err := anonymousHttpTransport(oc.AdminConfig())
 		o.Expect(err).NotTo(o.HaveOccurred())
 
@@ -34,7 +34,7 @@ var _ = g.Describe("[sig-api-machinery][Feature:APIServer]", func() {
 		o.Expect(resp.StatusCode).Should(o.Equal(http.StatusForbidden))
 	})
 
-	g.It("authenticated browser should get a 200 from /", func() {
+	g.It("authenticated browser should get a 200 from /", g.Label("Size:S"), func() {
 		transport, err := rest.TransportFor(oc.AdminConfig())
 		o.Expect(err).NotTo(o.HaveOccurred())
 

test/extended/apiserver/security_context.go

@@ -25,7 +25,7 @@ var _ = g.Describe("[sig-auth][Feature:ControlPlaneSecurity]", func() {
 	// Related issues:
 	// OCP-32383: Control plane security context verification
 	//bug 1793694: Init container security context
-	g.It("should have privileged securityContext for control plane init and main containers", func() {
+	g.It("should have privileged securityContext for control plane init and main containers", g.Label("Size:S"), func() {
 		// Skip on MicroShift clusters
 		isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
 		o.Expect(err).NotTo(o.HaveOccurred())