Skip to content

Enable tests against existing workload cluster #1350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
k8s-ci-robot merged 1 commit into from
May 4, 2021
Merged

Enable tests against existing workload cluster #1350

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/development.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,7 @@ You can optionally set the following variables:
| `SKIP_CREATE_MGMT_CLUSTER` | Skip management cluster creation. If skipping managment cluster creation you must specify `KUBECONFIG` and `SKIP_CLEANUP` | `false` |
| `LOCAL_ONLY` | Use Kind local registry and run the subset of tests which don't require a remotely pushed controller image. | `true` |
| `REGISTRY` | Registry to push the controller image. | `capzci.azurecr.io/ci-e2e` |
| `CLUSTER_NAME` | Name of an exisiting workload cluster. Will run specs against existing workload cluster. Use in conjunction with `SKIP_CREATE_MGMT_CLUSTER`, `GINKGO_FOCUS` and `KUBECONFIG`. Must specify only one e2e spec to run against with `GINKGO_FOCUS` such as `export GINKO_FOCUS=Creating.a.VMSS.cluster.with.a.single.control.plane.node`. | |
| `KUBECONFIG` | Used with `SKIP_CREATE_MGMT_CLUSTER` set to true. Location of kubeconfig for the management cluster you would like to use. Use `kind get kubeconfig --name capz-e2e > kubeconfig.capz-e2e` to get the capz e2e kind cluster config | '~/.kube/config' |

You can also customize the configuration of the CAPZ cluster created by the E2E tests (except for `CLUSTER_NAME`, `AZURE_RESOURCE_GROUP`, `AZURE_VNET_NAME`, `CONTROL_PLANE_MACHINE_COUNT`, and `WORKER_MACHINE_COUNT`, since they are generated by individual test cases). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details.
Expand Down
8 changes: 5 additions & 3 deletions test/e2e/azure_lb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"fmt"
"net"

"sigs.k8s.io/cluster-api/util"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
k8snet "k8s.io/utils/net"
Expand Down Expand Up @@ -69,10 +71,10 @@ func AzureLBSpec(ctx context.Context, inputGetter func() AzureLBSpecInput) {
Expect(clientset).NotTo(BeNil())

By("creating an HTTP deployment")
deploymentName := "web"
deploymentName := "web" + util.RandomString(6)
// if case of input.SkipCleanup we need a unique name for windows
if input.Windows {
deploymentName = "web-windows"
deploymentName = "web-windows" + util.RandomString(6)
}

webDeployment := deploymentBuilder.CreateDeployment("httpd", deploymentName, corev1.NamespaceDefault)
Expand Down Expand Up @@ -173,7 +175,7 @@ func AzureLBSpec(ctx context.Context, inputGetter func() AzureLBSpecInput) {

elbIP := extractServiceIp(svc)
Log("starting to create curl-to-elb job")
elbJob := job.CreateCurlJob("curl-to-elb-job", elbIP)
elbJob := job.CreateCurlJob("curl-to-elb-job"+util.RandomString(6), elbIP)
_, err = jobsClient.Create(ctx, elbJob, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
elbJobInput := WaitForJobCompleteInput{
Expand Down
14 changes: 11 additions & 3 deletions test/e2e/azure_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,15 @@ import (
"path/filepath"
"time"

"sigs.k8s.io/cluster-api/util"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/cluster-api/util"
)

var _ = Describe("Workload cluster creation", func() {
Expand All @@ -58,10 +59,17 @@ var _ = Describe("Workload cluster creation", func() {
Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion))
Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath))

clusterName = os.Getenv("CLUSTER_NAME")
if clusterName == "" {
clusterName = fmt.Sprintf("capz-e2e-%s", util.RandomString(6))
}
fmt.Fprintf(GinkgoWriter, "INFO: Cluster name is %s\n", clusterName)

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
var err error
namespace, cancelWatches, err = setupSpecNamespace(ctx, clusterName, bootstrapClusterProxy, artifactFolder)
Expect(err).NotTo(HaveOccurred())

clusterName = fmt.Sprintf("capz-e2e-%s", util.RandomString(6))
Expect(os.Setenv(AzureResourceGroup, clusterName)).NotTo(HaveOccurred())
Expect(os.Setenv(AzureVNetName, fmt.Sprintf("%s-vnet", clusterName))).NotTo(HaveOccurred())
result = new(clusterctl.ApplyClusterTemplateAndWaitResult)
Expand Down
38 changes: 32 additions & 6 deletions test/e2e/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,17 @@ import (
"path"
"path/filepath"

apierrors "k8s.io/apimachinery/pkg/api/errors"

e2e_namespace "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/namespace"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/util/kubeconfig"
crclient "sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -61,16 +64,39 @@ func Byf(format string, a ...interface{}) {
By(fmt.Sprintf(format, a...))
}

func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) {
Byf("Creating a namespace for hosting the %q test spec", specName)
func setupSpecNamespace(ctx context.Context, namespaceName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc, error) {
Byf("Creating namespace %q for hosting the cluster", namespaceName)
logPath := filepath.Join(artifactFolder, "clusters", clusterProxy.GetName())
namespace, err := e2e_namespace.Get(ctx, clusterProxy.GetClientSet(), namespaceName)
if err != nil && !apierrors.IsNotFound(err) {
return nil, nil, err
}

// namespace exists wire it up
if err == nil {
Byf("Creating event watcher for existing namespace %q", namespace.Name)
watchesCtx, cancelWatches := context.WithCancel(ctx)
go func() {
defer GinkgoRecover()
framework.WatchNamespaceEvents(watchesCtx, framework.WatchNamespaceEventsInput{
ClientSet: clusterProxy.GetClientSet(),
Name: namespace.Name,
LogFolder: logPath,
})
}()

return namespace, cancelWatches, nil
}
Comment thread
jsturtevant marked this conversation as resolved.

// create and wire up namespace
namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
Creator: clusterProxy.GetClient(),
ClientSet: clusterProxy.GetClientSet(),
Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()),
Name: namespaceName,
LogFolder: logPath,
})

return namespace, cancelWatches
return namespace, cancelWatches, nil
}

func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) {
Expand Down
11 changes: 9 additions & 2 deletions test/e2e/conformance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,17 @@ var _ = Describe("Conformance Tests", func() {
Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion))
Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath))

clusterName = os.Getenv("CLUSTER_NAME")
if clusterName == "" {
clusterName = fmt.Sprintf("capz-conf-%s", util.RandomString(6))
}
fmt.Fprintf(GinkgoWriter, "INFO: Cluster name is %s\n", clusterName)

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
var err error
namespace, cancelWatches, err = setupSpecNamespace(ctx, clusterName, bootstrapClusterProxy, artifactFolder)
Expect(err).NotTo(HaveOccurred())

clusterName = fmt.Sprintf("capz-conf-%s", util.RandomString(6))
Expect(os.Setenv(AzureResourceGroup, clusterName)).NotTo(HaveOccurred())
Expect(os.Setenv(AzureVNetName, fmt.Sprintf("%s-vnet", clusterName))).NotTo(HaveOccurred())
})
Expand Down