diff --git a/test/extended/apiserver/kubeconfigs.go b/test/extended/apiserver/kubeconfigs.go index af01a58e8db6..a88f2261b47c 100644 --- a/test/extended/apiserver/kubeconfigs.go +++ b/test/extended/apiserver/kubeconfigs.go @@ -3,12 +3,15 @@ package apiserver import ( "context" "fmt" + "regexp" g "github.com/onsi/ginkgo" o "github.com/onsi/gomega" - exutil "github.com/openshift/origin/test/extended/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" + + exutil "github.com/openshift/origin/test/extended/util" ) var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local kubeconfig", func() { @@ -29,13 +32,29 @@ var _ = g.Describe("[Conformance][sig-api-machinery][Feature:APIServer] local ku framework.Logf("Discovered %d master nodes.", len(masterNodes.Items)) o.Expect(masterNodes.Items).NotTo(o.HaveLen(0)) for _, master := range masterNodes.Items { - g.By("Testing master node " + master.Name) - kubeconfigPath := "/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/" + kubeconfig - framework.Logf("Verifying kubeconfig %q on master %s", master.Name) - out, err := oc.AsAdmin().Run("debug").Args("node/"+master.Name, "--", "chroot", "/host", "/bin/bash", "-euxo", "pipefail", "-c", fmt.Sprintf(`oc --kubeconfig "%s" get namespace kube-system`, kubeconfigPath)).Output() + retry, err := testNode(oc, kubeconfig, master.Name) + for retries := 2; retries > 0; retries-- { + if !retry { + break + } + g.By("There was a retryable error for " + fmt.Sprintf("%s/%s", master.Name, kubeconfig)) + retry, err = testNode(oc, kubeconfig, master.Name) + } o.Expect(err).NotTo(o.HaveOccurred()) - framework.Logf(out) } }) } }) + +func testNode(oc *exutil.CLI, kubeconfig, masterName string) (bool, error) { + g.By("Testing master node " + masterName) + kubeconfigPath := "/etc/kubernetes/static-pod-resources/kube-apiserver-certs/secrets/node-kubeconfigs/" + kubeconfig + framework.Logf("Verifying kubeconfig %q on master %q", kubeconfig, masterName) + out, err := oc.AsAdmin().Run("debug").Args("node/"+masterName, "--", "chroot", "/host", "/bin/bash", "-euxo", "pipefail", "-c", + fmt.Sprintf(`oc --kubeconfig "%s" get namespace kube-system`, kubeconfigPath)).Output() + framework.Logf(out) + // retry error when kube-apiserver was temporarily unavailable, this matches oc error coming from: + // https://github.com/kubernetes/kubernetes/blob/cbb5ea8210596ada1efce7e7a271ca4217ae598e/staging/src/k8s.io/kubectl/pkg/cmd/util/helpers.go#L237-L243 + matched, _ := regexp.MatchString("The connection to the server .+ was refused - did you specify the right host or port", out) + return !matched, err +}