[YUNIKORN-1687] add e2e for user & group quota enforcement

Closes: #634

Signed-off-by: Manikandan R <[email protected]>

go.mod

@@ -22,7 +22,7 @@ go 1.20
 
 require (
 	github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20201215015655-2e8b733f5ad0
-	github.com/apache/yunikorn-core v0.0.0-20230809131940-0ecf24d2aad2
+	github.com/apache/yunikorn-core v0.0.0-20230831174655-2feb6ecc1384
 	github.com/apache/yunikorn-scheduler-interface v0.0.0-20230831133811-e7622cf54e95
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0

go.sum

@@ -108,8 +108,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
-github.com/apache/yunikorn-core v0.0.0-20230809131940-0ecf24d2aad2 h1:8slKtc1QS7NFVcNk5nqrl5/1ZhBrSsdsc9x9i8/gGqs=
-github.com/apache/yunikorn-core v0.0.0-20230809131940-0ecf24d2aad2/go.mod h1:yI17LJoTnaNipTgk+uuJTdgBzfwLNrsRVktNYqYjAwI=
+github.com/apache/yunikorn-core v0.0.0-20230831174655-2feb6ecc1384 h1:yczvF3glSsCVz+vANH3md5vHHk1U8JoxSJdGEAw1IU0=
+github.com/apache/yunikorn-core v0.0.0-20230831174655-2feb6ecc1384/go.mod h1:g58rxTrJWK80i21F1YC85lXEH1YlHzefPqv/dXmMHxo=
 github.com/apache/yunikorn-scheduler-interface v0.0.0-20230831133811-e7622cf54e95 h1:ssLPAWkGdrBJwfPEPiv0np99IgynCmmR402TM0FxgPg=
 github.com/apache/yunikorn-scheduler-interface v0.0.0-20230831133811-e7622cf54e95/go.mod h1:/n67iTTOytyVor6wETVjleqzsp/NxCUmPslHcTvJ+Nw=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=

test/e2e/framework/configmanager/constants.go

@@ -45,6 +45,8 @@ const (
 	AppPath          = "ws/v1/partition/%s/queue/%s/application/%s"
 	ClustersPath     = "ws/v1/clusters"
 	NodesPath        = "ws/v1/partition/%s/nodes"
+	UserUsagePath    = "ws/v1/partition/%s/usage/user/%s"
+	GroupUsagePath   = "ws/v1/partition/%s/usage/group/%s"
 	HealthCheckPath  = "ws/v1/scheduler/healthcheck"
 	ValidateConfPath = "ws/v1/validate-conf"
 

test/e2e/framework/helpers/yunikorn/rest_api_utils.go

@@ -398,3 +398,23 @@ func (c *RClient) GetPartitions(partition string) (*dao.PartitionQueueDAOInfo, e
 	_, err = c.do(req, &partitions)
 	return partitions, err
 }
+
+func (c *RClient) GetUserUsage(partition string, userName string) (*dao.UserResourceUsageDAOInfo, error) {
+	req, err := c.newRequest("GET", fmt.Sprintf(configmanager.UserUsagePath, partition, userName), nil)
+	if err != nil {
+		return nil, err
+	}
+	var userUsage *dao.UserResourceUsageDAOInfo
+	_, err = c.do(req, &userUsage)
+	return userUsage, err
+}
+
+func (c *RClient) GetGroupUsage(partition string, groupName string) (*dao.GroupResourceUsageDAOInfo, error) {
+	req, err := c.newRequest("GET", fmt.Sprintf(configmanager.GroupUsagePath, partition, groupName), nil)
+	if err != nil {
+		return nil, err
+	}
+	var groupUsage *dao.GroupResourceUsageDAOInfo
+	_, err = c.do(req, &groupUsage)
+	return groupUsage, err
+}

test/e2e/framework/helpers/yunikorn/wrappers.go

@@ -70,6 +70,10 @@ func UpdateConfigMapWrapper(oldConfigMap *v1.ConfigMap, schedPolicy string, anno
 }
 
 func UpdateCustomConfigMapWrapper(oldConfigMap *v1.ConfigMap, schedPolicy string, annotation string, mutator func(sc *configs.SchedulerConfig) error) {
+	UpdateCustomConfigMapWrapperWithMap(oldConfigMap, schedPolicy, annotation, nil, mutator)
+}
+
+func UpdateCustomConfigMapWrapperWithMap(oldConfigMap *v1.ConfigMap, schedPolicy string, annotation string, customMap map[string]string, mutator func(sc *configs.SchedulerConfig) error) {
 	Ω(k.SetClient()).To(BeNil())
 	By("Port-forward the scheduler pod")
 	fwdErr := k.PortForwardYkSchedulerPod()
@@ -104,6 +108,10 @@ func UpdateCustomConfigMapWrapper(oldConfigMap *v1.ConfigMap, schedPolicy string
 	configStr, yamlErr := common.ToYAML(sc)
 	Ω(yamlErr).NotTo(HaveOccurred())
 	c.Data[configmanager.DefaultPolicyGroup] = configStr
+
+	for k, v := range customMap {
+		c.Data[k] = v
+	}
 	var d, err3 = k.UpdateConfigMap(c, configmanager.YuniKornTestConfig.YkNamespace)
 	Ω(err3).NotTo(HaveOccurred())
 	Ω(d).NotTo(BeNil())
@@ -125,6 +133,7 @@ func RestoreConfigMapWrapper(oldConfigMap *v1.ConfigMap, annotation string) {
 	Ω(err).NotTo(HaveOccurred())
 	ts, tsErr := common.SetQueueTimestamp(oldSC, "default", "root")
 	Ω(tsErr).NotTo(HaveOccurred())
+	c.Data = oldConfigMap.Data
 	c.Data[configmanager.DefaultPolicyGroup], err = common.ToYAML(oldSC)
 	Ω(err).NotTo(HaveOccurred())
 

test/e2e/user_group_limit/user_group_limit_suite_test.go

@@ -0,0 +1,50 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package user_group_limit_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2/reporters"
+	"github.com/onsi/gomega"
+
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager"
+)
+
+func init() {
+	configmanager.YuniKornTestConfig.ParseFlags()
+}
+
+func TestUserGroupLimit(t *testing.T) {
+	ginkgo.ReportAfterSuite("TestUserGroupLimit", func(report ginkgo.Report) {
+		err := reporters.GenerateJUnitReportWithConfig(
+			report,
+			filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-user_group_limit_junit.xml"),
+			reporters.JunitReportConfig{OmitSpecLabels: true},
+		)
+		Ω(err).NotTo(HaveOccurred())
+	})
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "TestUserGroupLimit", ginkgo.Label("TestUserGroupLimit"))
+}
+
+var Ω = gomega.Ω
+var HaveOccurred = gomega.HaveOccurred