Bug 1986003: Retry kubeconfig checks, when kube-apiserver is temporarily unavailable

test/extended/apiserver/kubeconfigs.go

@@ -3,12 +3,15 @@ package apiserver
 import (
 	"context"
 	"fmt"
+	"regexp"
 
 	g "github.com/onsi/ginkgo"
 	o "github.com/onsi/gomega"
-	exutil "github.com/openshift/origin/test/extended/util"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
+
+	exutil "github.com/openshift/origin/test/extended/util"
 )
 
 var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local kubeconfig", func() {
@@ -29,13 +32,29 @@ var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local ku
 			framework.Logf("Discovered %d master nodes.", len(masterNodes.Items))
 			o.Expect(masterNodes.Items).NotTo(o.HaveLen(0))
 			for _, master := range masterNodes.Items {
-				g.By("Testing master node " + master.Name)
-				kubeconfigPath := "/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/" + kubeconfig
-				framework.Logf("Verifying kubeconfig %q on master %s", master.Name)
-				out, err := oc.AsAdmin().Run("debug").Args("node/"+master.Name, "--", "chroot", "/host", "/bin/bash", "-euxo", "pipefail", "-c", fmt.Sprintf(`oc --kubeconfig "%s" get namespace kube-system`, kubeconfigPath)).Output()
+				retry, err := testNode(oc, kubeconfig, master.Name)
+				for retries := 2; retries > 0; retries-- {
+					if !retry {
+						break
+					}
+					g.By("There was a retryable error for " + fmt.Sprintf("%s/%s", master.Name, kubeconfig))
+					retry, err = testNode(oc, kubeconfig, master.Name)
+				}
 				o.Expect(err).NotTo(o.HaveOccurred())
-				framework.Logf(out)
 			}
 		})
 	}
 })
+
+func testNode(oc *exutil.CLI, kubeconfig, masterName string) (bool, error) {
+	g.By("Testing master node " + masterName)
+	kubeconfigPath := "/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/" + kubeconfig
+	framework.Logf("Verifying kubeconfig %q on master %q", kubeconfig, masterName)
+	out, err := oc.AsAdmin().Run("debug").Args("node/"+masterName, "--", "chroot", "/host", "/bin/bash", "-euxo", "pipefail", "-c",
+		fmt.Sprintf(`oc --kubeconfig "%s" get namespace kube-system`, kubeconfigPath)).Output()
+	framework.Logf(out)
+	// retry error when kube-apiserver was temporarily unavailable, this matches oc error coming from:
+	// https://github.com/kubernetes/kubernetes/blob/cbb5ea8210596ada1efce7e7a271ca4217ae598e/staging/src/k8s.io/kubectl/pkg/cmd/util/helpers.go#L237-L243
+	matched, _ := regexp.MatchString("The connection to the server .+ was refused - did you specify the right host or port", out)
+	return !matched, err
+}