UPSTREAM: 134071: Wait for quota to report used before creating pvc

This should avoid races between pvc creation and quota creation

Author: gnufied

test/e2e/storage/csimock/base.go

@@ -28,10 +28,13 @@ import (
 
 	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega/gcustom"
+	gomegatypes "github.com/onsi/gomega/types"
 	"google.golang.org/grpc/codes"
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/fields"
@@ -518,9 +521,46 @@ func (m *mockDriverSetup) createResourceQuota(ctx context.Context, quota *v1.Res
 	quota, err = f.ClientSet.CoreV1().ResourceQuotas(f.Namespace.Name).Create(ctx, quota, metav1.CreateOptions{})
 	framework.ExpectNoError(err, "Failed to create resourceQuota")
 	m.quotas = append(m.quotas, quota)
+	usedResources := v1.ResourceList{}
+	usedResources[pvcSizeQuotaKey] = resource.MustParse("0")
+	usedResources[pvcCountQuotaKey] = resource.MustParse("0")
+	err = m.waitForResourceQuota(ctx, f.Namespace.Name, quota.Name, usedResources)
+	framework.ExpectNoError(err, "Failed to wait for resourcequota creation")
 	return quota
 }
 
+func (m *mockDriverSetup) waitForResourceQuota(ctx context.Context, ns, quotaName string, used v1.ResourceList) error {
+	var lastResourceQuota *v1.ResourceQuota
+	f := m.f
+	err := framework.Gomega().Eventually(ctx, framework.GetObject(f.ClientSet.CoreV1().ResourceQuotas(ns).Get, quotaName, metav1.GetOptions{})).Should(haveUsedResources(used, &lastResourceQuota))
+	if lastResourceQuota != nil && err == nil {
+		framework.Logf("Got expected ResourceQuota:\n%s", format.Object(lastResourceQuota, 1))
+	}
+	return err
+}
+
+func haveUsedResources(used v1.ResourceList, lastResourceQuota **v1.ResourceQuota) gomegatypes.GomegaMatcher {
+	// The template emits the actual ResourceQuota object as YAML.
+	// In particular the ManagedFields are interesting because both
+	// kube-apiserver and kube-controller-manager set the status.
+	return gcustom.MakeMatcher(func(resourceQuota *v1.ResourceQuota) (bool, error) {
+		if lastResourceQuota != nil {
+			*lastResourceQuota = resourceQuota
+		}
+		// used may not yet be calculated
+		if resourceQuota.Status.Used == nil {
+			return false, nil
+		}
+		// verify that the quota shows the expected used resource values
+		for k, v := range used {
+			if actualValue, found := resourceQuota.Status.Used[k]; !found || (actualValue.Cmp(v) != 0) {
+				return false, nil
+			}
+		}
+		return true, nil
+	}).WithTemplate("Expected:\n{{.FormattedActual}}\n{{.To}} have the following .status.used entries:\n{{range $key, $value := .Data}}    {{$key}}: \"{{$value.ToUnstructured}}\"\n{{end}}").WithTemplateData(used /* Formatting of the map is done inside the template. */)
+}
+
 func waitForCSIDriver(cs clientset.Interface, driverName string) error {
 	timeout := 4 * time.Minute