[YUNIKORN-2931] Create foreign pod e2e tests

Makefile

@@ -727,4 +727,4 @@ kind-e2e: $(KIND_BIN)
 e2e_test: tools
 	@echo "running e2e tests"
 	cd ./test/e2e && \
-	ginkgo -r $(E2E_TEST) -v -keep-going -- -yk-namespace "yunikorn" -kube-config $(KUBECONFIG)
+	ginkgo -r foreign_pod -v -keep-going -- -yk-namespace "yunikorn" -kube-config $(KUBECONFIG)

test/e2e/foreign_pod/foreign_pod_suite_test.go

@@ -0,0 +1,89 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package foreign_pod
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2/reporters"
+	"github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn"
+)
+
+func init() {
+	configmanager.YuniKornTestConfig.ParseFlags()
+}
+
+func TestForeignPodHandling(t *testing.T) {
+	ginkgo.ReportAfterSuite("TestForeignPodHandling", func(report ginkgo.Report) {
+		err := common.CreateJUnitReportDir()
+		Ω(err).NotTo(gomega.HaveOccurred())
+		err = reporters.GenerateJUnitReportWithConfig(
+			report,
+			filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-foreign_pod_junit.xml"),
+			reporters.JunitReportConfig{OmitSpecLabels: true},
+		)
+		Ω(err).NotTo(HaveOccurred())
+	})
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "TestForeignPodHandling", ginkgo.Label("TestForeignPodHandling"))
+}
+
+var suiteName string
+var oldConfigMap = new(v1.ConfigMap)
+var kClient = k8s.KubeCtl{} //nolint
+
+var _ = BeforeSuite(func() {
+	_, filename, _, _ := runtime.Caller(0)
+	suiteName = common.GetSuiteName(filename)
+	yunikorn.EnsureYuniKornConfigsPresent()
+	yunikorn.UpdateConfigMapWrapper(oldConfigMap, "fifo")
+})
+
+var _ = AfterSuite(func() {
+	yunikorn.RestoreConfigMapWrapper(oldConfigMap)
+})
+
+// Declarations for Ginkgo DSL
+var Describe = ginkgo.Describe
+
+var It = ginkgo.It
+var PIt = ginkgo.PIt
+var By = ginkgo.By
+var BeforeSuite = ginkgo.BeforeSuite
+var AfterSuite = ginkgo.AfterSuite
+var BeforeEach = ginkgo.BeforeEach
+var AfterEach = ginkgo.AfterEach
+var DescribeTable = ginkgo.Describe
+var Entry = ginkgo.Entry
+
+// Declarations for Gomega Matchers
+var Equal = gomega.Equal
+var BeNumerically = gomega.BeNumerically
+var Ω = gomega.Expect
+var BeNil = gomega.BeNil
+var HaveOccurred = gomega.HaveOccurred

test/e2e/foreign_pod/foreign_pod_test.go

@@ -0,0 +1,117 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package foreign_pod
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/apache/yunikorn-core/pkg/webservice/dao"
+	tests "github.com/apache/yunikorn-k8shim/test/e2e"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn"
+)
+
+var (
+	restClient yunikorn.RClient
+	ns         string
+)
+
+var _ = Describe("", func() {
+	BeforeEach(func() {
+		kClient = k8s.KubeCtl{}
+		Ω(kClient.SetClient()).To(BeNil())
+
+		ns = "ns-" + common.RandSeq(10)
+		By(fmt.Sprintf("Creating namespace: %s", ns))
+		namespace, err := kClient.CreateNamespace(ns, nil)
+		Ω(err).NotTo(HaveOccurred())
+		Ω(namespace.Status.Phase).To(Equal(v1.NamespaceActive))
+	})
+
+	It("Verify foreign pod tracking", func() {
+		sleepPod1Config := k8s.SleepPodConfig{Name: "foreign-pod-1", NS: ns, Mem: 10, Time: 60}
+		sleepPod2Config := k8s.SleepPodConfig{Name: "foreign-pod-2", NS: ns, Mem: 15, Time: 60}
+
+		// create sleep pod1, scheduled by the default scheduler
+		sleepObj1, err := k8s.InitSleepPod(sleepPod1Config)
+		sleepObj1.Spec.SchedulerName = ""
+		Ω(err).NotTo(gomega.HaveOccurred())
+		By("Creating foreign sleep pod #1")
+		_, err = kClient.CreatePod(sleepObj1, ns)
+		Ω(err).NotTo(gomega.HaveOccurred())
+
+		// create sleep pod2, scheduled by the default scheduler
+		sleepObj2, err := k8s.InitSleepPod(sleepPod2Config)
+		sleepObj2.Spec.SchedulerName = ""
+		Ω(err).NotTo(gomega.HaveOccurred())
+		By("Creating foreign sleep pod #2")
+		_, err = kClient.CreatePod(sleepObj2, ns)
+		Ω(err).NotTo(gomega.HaveOccurred())
+
+		// wait until pod1 & pod2 are in Running state
+		By("Waiting for sleep pod #1 to be scheduled")
+		err = kClient.WaitForPodRunning(ns, sleepPod1Config.Name, 30*time.Second)
+		Ω(err).NotTo(gomega.HaveOccurred())
+		By("Waiting for sleep pod #2 to be scheduled")
+		err = kClient.WaitForPodRunning(ns, sleepPod2Config.Name, 30*time.Second)
+		Ω(err).NotTo(gomega.HaveOccurred())
+
+		// retrieve foreign pod info
+		By("Retrieving foreign allocations")
+		nodes, err := restClient.GetNodes("default")
+		Ω(err).NotTo(gomega.HaveOccurred())
+		foreignAllocs := make([]*dao.ForeignAllocationDAOInfo, 0, 2)
+		for _, n := range *nodes {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "Checking node %s", n.NodeID)
+			if len(n.ForeignAllocations) > 0 {
+				for _, falloc := range n.ForeignAllocations {
+					fmt.Fprintf(ginkgo.GinkgoWriter, "Found allocation %s", falloc.AllocationKey)
+					foreignAllocs = append(foreignAllocs, falloc)
+				}
+			}
+		}
+		Ω(len(foreignAllocs)).To(Equal(2))
+
+		if foreignAllocs[0].AllocationKey == "foreign-pod-1" {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "%v\n", foreignAllocs[0].ResourcePerAlloc)
+			fmt.Fprintf(ginkgo.GinkgoWriter, "%v\n", foreignAllocs[1].ResourcePerAlloc)
+		} else {
+			fmt.Fprintf(ginkgo.GinkgoWriter, "%v\n", foreignAllocs[0].ResourcePerAlloc)
+			fmt.Fprintf(ginkgo.GinkgoWriter, "%v\n", foreignAllocs[1].ResourcePerAlloc)
+		}
+	})
+
+	ginkgo.AfterEach(func() {
+		tests.DumpClusterInfoIfSpecFailed(suiteName, []string{ns})
+		ginkgo.By("Tear down namespace: " + ns)
+		err := kClient.TearDownNamespace(ns)
+		gomega.Ω(err).NotTo(gomega.HaveOccurred())
+		// call the healthCheck api to check scheduler health
+		ginkgo.By("Check YuniKorn's health")
+		checks, err2 := yunikorn.GetFailedHealthChecks()
+		gomega.Ω(err2).ShouldNot(gomega.HaveOccurred())
+		gomega.Ω(checks).Should(gomega.Equal(""), checks)
+	})
+})