Running functional pipeline

Author: bjee19

tests/suite/graceful_recovery_test.go

@@ -107,9 +107,11 @@ var _ = Describe("Graceful Recovery test", Ordered, Label("functional", "gracefu
 		runRestartNodeTest(teaURL, coffeeURL, files, &ns, true)
 	})
 
-	//It("recovers when node is restarted abruptly", func() {
-	//	runRestartNodeTest(teaURL, coffeeURL, files, &ns, false)
-	//})
+	It("recovers when node is restarted abruptly", func() {
+		// FIXME(bjee19) remove Skip() when https://github.com/nginxinc/nginx-gateway-fabric/issues/1108 is completed.
+		Skip("Test currently fails due to this issue: https://github.com/nginxinc/nginx-gateway-fabric/issues/1108")
+		runRestartNodeTest(teaURL, coffeeURL, files, &ns, false)
+	})
 })
 
 func runRestartNodeTest(teaURL, coffeeURL string, files []string, ns *core.Namespace, drain bool) {
@@ -146,27 +148,28 @@ func runRestartNodeTest(teaURL, coffeeURL string, files []string, ns *core.Names
 		}
 	}
 
-	output, err := exec.Command(
+	containerOutput, err := exec.Command(
 		"docker",
 		"container",
 		"ls",
 	).CombinedOutput()
 	if err != nil {
 		Expect(err).ToNot(HaveOccurred())
 	}
-
-	fmt.Println(string(output))
+	fmt.Println(string(containerOutput))
 
 	var containerName string
-	for _, line := range strings.Split(string(output), "\n") {
+	for _, line := range strings.Split(string(containerOutput), "\n") {
 		for _, word := range strings.Split(line, " ") {
+			// This is a potential weak spot in the code where we rely on the container which NGF
+			// is running on to contain "control-plane" in the name and for no other container to have that either.
+			// This is currently working in our test framework may break in the future.
 			if strings.Contains(word, "control-plane") {
 				containerName = strings.TrimSpace(word)
 				break
 			}
 		}
 	}
-
 	Expect(containerName).ToNot(Equal(""))
 
 	// really jank - get the string that contains "control-plane"
@@ -182,9 +185,10 @@ func runRestartNodeTest(teaURL, coffeeURL string, files []string, ns *core.Names
 		Expect(err).ToNot(HaveOccurred())
 	}
 
+	// need to wait for docker container to restart and be running before polling for ready NGF Pods or else we will error
 	Eventually(
 		func() bool {
-			output, err = exec.Command(
+			output, err := exec.Command(
 				"docker",
 				"container",
 				"inspect",
@@ -212,42 +216,10 @@ func runRestartNodeTest(teaURL, coffeeURL string, files []string, ns *core.Names
 		ports := []string{fmt.Sprintf("%d:80", ngfHTTPForwardedPort), fmt.Sprintf("%d:443", ngfHTTPSForwardedPort)}
 		portForwardStopCh = make(chan struct{})
 		err = framework.PortForward(ctlr.GetConfigOrDie(), ngfNamespace, ngfPodName, ports, portForwardStopCh)
-		address = "127.0.0.1"
-		portFwdPort = ngfHTTPForwardedPort
-		portFwdHTTPSPort = ngfHTTPSForwardedPort
 		Expect(err).ToNot(HaveOccurred())
 	}
 
-	Eventually(
-		func() error {
-			return checkForWorkingTraffic(teaURL, coffeeURL)
-		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
-		WithPolling(500 * time.Millisecond).
-		Should(Succeed())
-
-	Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
-
-	Eventually(
-		func() error {
-			return checkForFailingTraffic(teaURL, coffeeURL)
-		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
-		WithPolling(500 * time.Millisecond).
-		Should(Succeed())
-
-	Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
-	Expect(resourceManager.WaitForAppsToBeReadyWithPodCount(ns.Name, 2)).To(Succeed())
-
-	Eventually(
-		func() error {
-			return checkForWorkingTraffic(teaURL, coffeeURL)
-		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
-		WithPolling(500 * time.Millisecond).
-		Should(Succeed())
-
-	checkContainerLogsForErrors(ngfPodName)
+	checkNGFFunctionality(teaURL, coffeeURL, ngfPodName, files, ns)
 }
 
 func runRecoveryTest(teaURL, coffeeURL, ngfPodName, containerName string, files []string, ns *core.Namespace) {
@@ -276,37 +248,7 @@ func runRecoveryTest(teaURL, coffeeURL, ngfPodName, containerName string, files
 			WithPolling(500 * time.Millisecond).
 			Should(Succeed())
 	}
-
-	Eventually(
-		func() error {
-			return checkForWorkingTraffic(teaURL, coffeeURL)
-		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
-		WithPolling(500 * time.Millisecond).
-		Should(Succeed())
-
-	Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
-
-	Eventually(
-		func() error {
-			return checkForFailingTraffic(teaURL, coffeeURL)
-		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
-		WithPolling(500 * time.Millisecond).
-		Should(Succeed())
-
-	Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
-	Expect(resourceManager.WaitForAppsToBeReadyWithPodCount(ns.Name, 2)).To(Succeed())
-
-	Eventually(
-		func() error {
-			return checkForWorkingTraffic(teaURL, coffeeURL)
-		}).
-		WithTimeout(timeoutConfig.RequestTimeout).
-		WithPolling(500 * time.Millisecond).
-		Should(Succeed())
-
-	checkContainerLogsForErrors(ngfPodName)
+	checkNGFFunctionality(teaURL, coffeeURL, ngfPodName, files, ns)
 }
 
 func restartContainer(ngfPodName, containerName string) {
@@ -404,6 +346,39 @@ func expectRequestToFail(appURL, address string) error {
 	return nil
 }
 
+func checkNGFFunctionality(teaURL, coffeeURL, ngfPodName string, files []string, ns *core.Namespace) {
+	Eventually(
+		func() error {
+			return checkForWorkingTraffic(teaURL, coffeeURL)
+		}).
+		WithTimeout(timeoutConfig.RequestTimeout).
+		WithPolling(500 * time.Millisecond).
+		Should(Succeed())
+
+	Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
+
+	Eventually(
+		func() error {
+			return checkForFailingTraffic(teaURL, coffeeURL)
+		}).
+		WithTimeout(timeoutConfig.RequestTimeout).
+		WithPolling(500 * time.Millisecond).
+		Should(Succeed())
+
+	Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
+	Expect(resourceManager.WaitForAppsToBeReadyWithPodCount(ns.Name, 2)).To(Succeed())
+
+	Eventually(
+		func() error {
+			return checkForWorkingTraffic(teaURL, coffeeURL)
+		}).
+		WithTimeout(timeoutConfig.RequestTimeout).
+		WithPolling(500 * time.Millisecond).
+		Should(Succeed())
+
+	checkContainerLogsForErrors(ngfPodName)
+}
+
 // checkContainerLogsForErrors checks both nginx and ngf container's logs for any possible errors.
 // Since this function retrieves all the logs from both containers and the NGF pod may be shared between tests,
 // the logs retrieved may contain log messages from previous tests, thus any errors in the logs from previous tests