diff --git a/cmd/clusterctl/client/cluster/components.go b/cmd/clusterctl/client/cluster/components.go index baa5de10e2d5..445dde3ad078 100644 --- a/cmd/clusterctl/client/cluster/components.go +++ b/cmd/clusterctl/client/cluster/components.go @@ -219,7 +219,7 @@ func (p *providerComponents) Delete(options DeleteOptions) error { func (p *providerComponents) DeleteWebhookNamespace() error { log := logf.Log - log.V(5).Info("Deleting %s namespace", repository.WebhookNamespaceName) + log.V(5).Info("Deleting", "namespace", repository.WebhookNamespaceName) c, err := p.proxy.NewClient() if err != nil { diff --git a/cmd/clusterctl/hack/create-local-repository.py b/cmd/clusterctl/hack/create-local-repository.py index 1e2610ef3d8f..eabc66e0bee6 100755 --- a/cmd/clusterctl/hack/create-local-repository.py +++ b/cmd/clusterctl/hack/create-local-repository.py @@ -53,24 +53,24 @@ providers = { 'cluster-api': { 'componentsFile': 'core-components.yaml', - 'nextVersion': 'v0.4.0', + 'nextVersion': 'v0.4.99', 'type': 'CoreProvider', }, 'bootstrap-kubeadm': { 'componentsFile': 'bootstrap-components.yaml', - 'nextVersion': 'v0.4.0', + 'nextVersion': 'v0.4.99', 'type': 'BootstrapProvider', 'configFolder': 'bootstrap/kubeadm/config/default', }, 'control-plane-kubeadm': { 'componentsFile': 'control-plane-components.yaml', - 'nextVersion': 'v0.4.0', + 'nextVersion': 'v0.4.99', 'type': 'ControlPlaneProvider', 'configFolder': 'controlplane/kubeadm/config/default', }, 'infrastructure-docker': { 'componentsFile': 'infrastructure-components.yaml', - 'nextVersion': 'v0.4.0', + 'nextVersion': 'v0.4.99', 'type': 'InfrastructureProvider', 'configFolder': 'test/infrastructure/docker/config/default', }, diff --git a/test/e2e/clusterctl_upgrade.go b/test/e2e/clusterctl_upgrade.go new file mode 100644 index 000000000000..db10d08e250f --- /dev/null +++ b/test/e2e/clusterctl_upgrade.go @@ -0,0 +1,302 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + clusterv1old "sigs.k8s.io/cluster-api/api/v1alpha3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" + "sigs.k8s.io/cluster-api/test/e2e/internal/log" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const initWithBinaryVariableName = "INIT_WITH_BINARY" + +// ClusterctlUpgradeSpecInput is the input for ClusterctlUpgradeSpec. +type ClusterctlUpgradeSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// ClusterctlUpgradeSpec implements a test that verifies clusterctl upgrade of a management cluster. +// +// NOTE: this test is designed to test v1alpha3 --> v1alpha4 upgrades. +func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpgradeSpecInput) { + var ( + specName = "clusterctl-upgrade" + input ClusterctlUpgradeSpecInput + + testNamespace *corev1.Namespace + testCancelWatches context.CancelFunc + + managementClusterName string + managementClusterNamespace *corev1.Namespace + managementClusterCancelWatches context.CancelFunc + managementClusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + managementClusterProxy framework.ClusterProxy + + workLoadClusterName string + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.E2EConfig.Variables).To(HaveKey(initWithBinaryVariableName), "Invalid argument. %s variable must be defined when calling %s spec", initWithBinaryVariableName, specName) + Expect(input.E2EConfig.Variables[initWithBinaryVariableName]).ToNot(BeEmpty(), "Invalid argument. %s variable can't be empty when calling %s spec", initWithBinaryVariableName, specName) + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + managementClusterNamespace, managementClusterCancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + managementClusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) + + It("Should create a management cluster and then upgrade all the providers", func() { + By("Creating a workload cluster to be used as a new management cluster") + // NOTE: given that the bootstrap cluster could be shared by several tests, it is not practical to use it for testing clusterctl upgrades. + // So we are creating a workload cluster that will be used as a new management cluster where to install older version of providers + managementClusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: managementClusterNamespace.Name, + ClusterName: managementClusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, managementClusterResources) + + By("Turning the workload cluster into a management cluster with older versions of providers") + + // If the cluster is a DockerCluster, we should load controller images into the nodes. + // Nb. this can be achieved also by changing the DockerMachine spec, but for the time being we are using + // this approach because this allows to have a single source of truth for images, the e2e config + // Nb. the images for official version of the providers will be pulled from internet, but the latest images must be + // built locally and loaded into kind + cluster := managementClusterResources.Cluster + if cluster.Spec.InfrastructureRef.Kind == "DockerCluster" { + Expect(bootstrap.LoadImagesToKindCluster(ctx, bootstrap.LoadImagesToKindClusterInput{ + Name: cluster.Name, + Images: input.E2EConfig.Images, + })).To(Succeed()) + } + + // Get a ClusterProxy so we can interact with the workload cluster + managementClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name) + + // Download the v1alpha3 clusterctl version to be used for setting up the management cluster to be upgraded + clusterctlBinaryURL := input.E2EConfig.GetVariable(initWithBinaryVariableName) + clusterctlBinaryURL = strings.ReplaceAll(clusterctlBinaryURL, "{OS}", runtime.GOOS) + clusterctlBinaryURL = strings.ReplaceAll(clusterctlBinaryURL, "{ARCH}", runtime.GOARCH) + + log.Logf("downloading clusterctl binary from %s", clusterctlBinaryURL) + clusterctlBinaryPath := downloadToTmpFile(clusterctlBinaryURL) + defer os.Remove(clusterctlBinaryPath) // clean up + + err := os.Chmod(clusterctlBinaryPath, 0744) + Expect(err).ToNot(HaveOccurred(), "failed to chmod temporary file") + + By("Initializing the workload cluster with older versions of providers") + clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{ + ClusterctlBinaryPath: clusterctlBinaryPath, // use older version of clusterctl to init the management cluster + ClusterProxy: managementClusterProxy, + ClusterctlConfigPath: input.ClusterctlConfigPath, + CoreProvider: input.E2EConfig.GetProvidersWithOldestVersion(config.ClusterAPIProviderName)[0], + BootstrapProviders: input.E2EConfig.GetProvidersWithOldestVersion(config.KubeadmBootstrapProviderName), + ControlPlaneProviders: input.E2EConfig.GetProvidersWithOldestVersion(config.KubeadmControlPlaneProviderName), + InfrastructureProviders: input.E2EConfig.GetProvidersWithOldestVersion(input.E2EConfig.InfrastructureProviders()...), + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name), + }, input.E2EConfig.GetIntervals(specName, "wait-controllers")...) + + By("THE MANAGEMENT CLUSTER WITH THE OLDER VERSION OF PROVIDERS IS UP&RUNNING!") + + Byf("Creating a namespace for hosting the %s test workload cluster", specName) + testNamespace, testCancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: managementClusterProxy.GetClient(), + ClientSet: managementClusterProxy.GetClientSet(), + Name: specName, + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", "bootstrap"), + }) + + By("Creating a test workload cluster") + + // NOTE: This workload cluster is used to check the old management cluster works fine. + // In this case ApplyClusterTemplateAndWait can't be used because this helper is linked to the last version of the API; + // so we are getting a template using the downloaded version of clusterctl, applying it, and wait for machines to be provisioned. + + workLoadClusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + kubernetesVersion := input.E2EConfig.GetVariable(KubernetesVersion) + controlPlaneMachineCount := pointer.Int64Ptr(1) + workerMachineCount := pointer.Int64Ptr(1) + + log.Logf("Creating the workload cluster with name %q using the %q template (Kubernetes %s, %d control-plane machines, %d worker machines)", + workLoadClusterName, "(default)", kubernetesVersion, controlPlaneMachineCount, workerMachineCount) + + log.Logf("Getting the cluster template yaml") + workloadClusterTemplate := clusterctl.ConfigClusterWithBinary(ctx, clusterctlBinaryPath, clusterctl.ConfigClusterInput{ + // pass reference to the management cluster hosting this test + KubeconfigPath: managementClusterProxy.GetKubeconfigPath(), + // pass the clusterctl config file that points to the local provider repository created for this test, + ClusterctlConfigPath: input.ClusterctlConfigPath, + // select template + Flavor: clusterctl.DefaultFlavor, + // define template variables + Namespace: testNamespace.Name, + ClusterName: workLoadClusterName, + KubernetesVersion: kubernetesVersion, + ControlPlaneMachineCount: controlPlaneMachineCount, + WorkerMachineCount: workerMachineCount, + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + // setup clusterctl logs folder + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", managementClusterProxy.GetName()), + }) + Expect(workloadClusterTemplate).ToNot(BeNil(), "Failed to get the cluster template") + + log.Logf("Applying the cluster template yaml to the cluster") + Expect(managementClusterProxy.Apply(ctx, workloadClusterTemplate)).To(Succeed()) + + By("Waiting for the machines to exists") + Eventually(func() (int64, error) { + var n int64 = 0 + machineList := &clusterv1old.MachineList{} + if err := managementClusterProxy.GetClient().List(ctx, machineList, client.InNamespace(testNamespace.Name), client.MatchingLabels{clusterv1.ClusterLabelName: workLoadClusterName}); err == nil { + for _, machine := range machineList.Items { + if machine.Status.NodeRef != nil { + n++ + } + } + } + return n, nil + }, input.E2EConfig.GetIntervals(specName, "wait-worker-nodes")...).Should(Equal(*controlPlaneMachineCount + *workerMachineCount)) + + By("THE MANAGEMENT CLUSTER WITH OLDER VERSION OF PROVIDERS WORKS!") + + By("Upgrading providers to the latest version available") + clusterctl.UpgradeManagementClusterAndWait(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{ + ClusterctlConfigPath: input.ClusterctlConfigPath, + ClusterProxy: managementClusterProxy, + ManagementGroup: fmt.Sprintf("capi-system/%s", config.ClusterAPIProviderName), + Contract: clusterv1.GroupVersion.Version, + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", cluster.Name), + }, input.E2EConfig.GetIntervals(specName, "wait-controllers")...) + + By("THE MANAGEMENT CLUSTER WAS SUCCESSFULLY UPGRADED!") + + // After upgrading we are sure the version is the latest version of the API, + // so it is possible to use the standard helpers + + testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ + Lister: managementClusterProxy.GetClient(), + ClusterName: workLoadClusterName, + Namespace: testNamespace.Name, + }) + + framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{ + ClusterProxy: managementClusterProxy, + Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}}, + MachineDeployment: testMachineDeployments[0], + Replicas: 2, + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("THE UPGRADED MANAGEMENT CLUSTER WORKS!") + + By("PASSED!") + }) + + AfterEach(func() { + if testNamespace != nil { + // Dump all the logs from the workload cluster before deleting them. + managementClusterProxy.CollectWorkloadClusterLogs(ctx, testNamespace.Name, managementClusterName, filepath.Join(input.ArtifactFolder, "clusters", managementClusterName, "machines")) + + framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ + Lister: managementClusterProxy.GetClient(), + Namespace: managementClusterNamespace.Name, + LogPath: filepath.Join(input.ArtifactFolder, "clusters", managementClusterResources.Cluster.Name, "resources"), + }) + + if !input.SkipCleanup { + Byf("Deleting cluster %s and %s", testNamespace.Name, managementClusterName) + framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ + Client: managementClusterProxy.GetClient(), + Namespace: testNamespace.Name, + }, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster")...) + + Byf("Deleting namespace used for hosting the %q test", specName) + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: managementClusterProxy.GetClient(), + Name: testNamespace.Name, + }) + } + testCancelWatches() + } + + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, managementClusterNamespace, managementClusterCancelWatches, managementClusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} + +func downloadToTmpFile(url string) string { + tmpFile, err := ioutil.TempFile("", "clusterctl") + Expect(err).ToNot(HaveOccurred(), "failed to get temporary file") + defer tmpFile.Close() + + // Get the data + resp, err := http.Get(url) + Expect(err).ToNot(HaveOccurred(), "failed to get clusterctl") + defer resp.Body.Close() + + // Write the body to file + _, err = io.Copy(tmpFile, resp.Body) + Expect(err).ToNot(HaveOccurred(), "failed to write temporary file") + + return tmpFile.Name() +} diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go new file mode 100644 index 000000000000..6ef92109a8a9 --- /dev/null +++ b/test/e2e/clusterctl_upgrade_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("When testing clusterctl upgrades", func() { + + ClusterctlUpgradeSpec(context.TODO(), func() ClusterctlUpgradeSpecInput { + return ClusterctlUpgradeSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index 178e622cd7d8..b99d5f486913 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -29,8 +29,13 @@ providers: - name: cluster-api type: CoreProvider versions: - - name: v0.4.0 - # Use manifest from source files + - name: v0.3.16 # latest published release + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.16/core-components.yaml" + type: "url" + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + - name: v0.4.99 # next; use manifest from source files value: ../../../config/default replacements: - old: --metrics-bind-addr=127.0.0.1:8080 @@ -41,8 +46,13 @@ providers: - name: kubeadm type: BootstrapProvider versions: - - name: v0.4.0 - # Use manifest from source files + - name: v0.3.16 # latest published release + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.16/bootstrap-components.yaml" + type: "url" + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + - name: v0.4.99 # next; use manifest from source files value: ../../../bootstrap/kubeadm/config/default replacements: - old: --metrics-bind-addr=127.0.0.1:8080 @@ -53,8 +63,13 @@ providers: - name: kubeadm type: ControlPlaneProvider versions: - - name: v0.4.0 - # Use manifest from source files + - name: v0.3.16 # latest published release + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.16/control-plane-components.yaml" + type: "url" + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + - name: v0.4.99 # next; use manifest from source files value: ../../../controlplane/kubeadm/config/default replacements: - old: --metrics-bind-addr=127.0.0.1:8080 @@ -65,8 +80,16 @@ providers: - name: docker type: InfrastructureProvider versions: - - name: v0.4.0 - # Use manifest from source files + - name: v0.3.16 # latest published release + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.16/infrastructure-components-development.yaml" + type: "url" + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + # Add cluster templates + - sourcePath: "../data/infrastructure-docker/v1alpha3/cluster-template.yaml" + - name: v0.4.99 # next; use manifest from source files value: ../../../test/infrastructure/docker/config/default replacements: - old: --metrics-bind-addr=127.0.0.1:8080 @@ -100,6 +123,8 @@ variables: EXP_MACHINE_POOL: "true" KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" NODE_DRAIN_TIMEOUT: "60s" + # NOTE: INIT_WITH_BINARY is used only by the clusterctl upgrade test to initialize the management cluster to be upgraded + INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.16/clusterctl-{OS}-{ARCH}" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/framework/clusterctl/client.go b/test/framework/clusterctl/client.go index 24ef314405da..cca204cd9eb8 100644 --- a/test/framework/clusterctl/client.go +++ b/test/framework/clusterctl/client.go @@ -19,7 +19,10 @@ package clusterctl import ( "context" "fmt" + "io/ioutil" "os" + "os/exec" + "path/filepath" "strings" . "github.com/onsi/ginkgo" @@ -58,9 +61,9 @@ type InitInput struct { func Init(ctx context.Context, input InitInput) { log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s", input.CoreProvider, - strings.Join(input.BootstrapProviders, ", "), - strings.Join(input.ControlPlaneProviders, ", "), - strings.Join(input.InfrastructureProviders, ", "), + strings.Join(input.BootstrapProviders, ","), + strings.Join(input.ControlPlaneProviders, ","), + strings.Join(input.InfrastructureProviders, ","), ) initOpt := clusterctlclient.InitOptions{ @@ -82,6 +85,61 @@ func Init(ctx context.Context, input InitInput) { Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init") } +// InitWithBinary uses clusterctl binary to run init with the list of providers defined in the local repository. +func InitWithBinary(_ context.Context, binary string, input InitInput) { + log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s", + input.CoreProvider, + strings.Join(input.BootstrapProviders, ","), + strings.Join(input.ControlPlaneProviders, ","), + strings.Join(input.InfrastructureProviders, ","), + ) + + cmd := exec.Command(binary, "init", + "--core", input.CoreProvider, + "--bootstrap", strings.Join(input.BootstrapProviders, ","), + "--control-plane", strings.Join(input.ControlPlaneProviders, ","), + "--infrastructure", strings.Join(input.InfrastructureProviders, ","), + "--config", input.ClusterctlConfigPath, + "--kubeconfig", input.KubeconfigPath, + ) + + out, err := cmd.CombinedOutput() + _ = ioutil.WriteFile(filepath.Join(input.LogFolder, "clusterctl-init.log"), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts + Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl init") +} + +// UpgradeInput is the input for Upgrade. +type UpgradeInput struct { + LogFolder string + ClusterctlConfigPath string + KubeconfigPath string + ManagementGroup string + Contract string +} + +// Upgrade calls clusterctl upgrade apply with the list of providers defined in the local repository. +func Upgrade(ctx context.Context, input UpgradeInput) { + log.Logf("clusterctl upgrade apply --management-group %s --contract %s", + input.ManagementGroup, + input.Contract, + ) + + upgradeOpt := clusterctlclient.ApplyUpgradeOptions{ + Kubeconfig: clusterctlclient.Kubeconfig{ + Path: input.KubeconfigPath, + Context: "", + }, + ManagementGroup: input.ManagementGroup, + Contract: input.Contract, + } + + clusterctlClient, log := getClusterctlClientWithLogger(input.ClusterctlConfigPath, "clusterctl-upgrade.log", input.LogFolder) + defer log.Close() + + err := clusterctlClient.ApplyUpgrade(upgradeOpt) + Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade") +} + // ConfigClusterInput is the input for ConfigCluster. type ConfigClusterInput struct { LogFolder string @@ -137,6 +195,36 @@ func ConfigCluster(ctx context.Context, input ConfigClusterInput) []byte { return yaml } +// ConfigClusterWithBinary uses clusterctl binary to run config cluster. +func ConfigClusterWithBinary(_ context.Context, clusterctlBinaryPath string, input ConfigClusterInput) []byte { + log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s", + input.ClusterName, + valueOrDefault(input.InfrastructureProvider), + input.KubernetesVersion, + *input.ControlPlaneMachineCount, + *input.WorkerMachineCount, + valueOrDefault(input.Flavor), + ) + + cmd := exec.Command(clusterctlBinaryPath, "config", "cluster", + input.ClusterName, + "--infrastructure", input.InfrastructureProvider, + "--kubernetes-version", input.KubernetesVersion, + "--control-plane-machine-count", fmt.Sprint(*input.ControlPlaneMachineCount), + "--worker-machine-count", fmt.Sprint(*input.WorkerMachineCount), + "--flavor", input.Flavor, + "--target-namespace", input.Namespace, + "--config", input.ClusterctlConfigPath, + "--kubeconfig", input.KubeconfigPath, + ) + + out, err := cmd.Output() + _ = ioutil.WriteFile(filepath.Join(input.LogFolder, fmt.Sprintf("%s-cluster-template.yaml", input.ClusterName)), out, 0644) //nolint:gosec // this is a log file to be shared via prow artifacts + Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl config cluster") + + return out +} + // MoveInput is the input for ClusterctlMove. type MoveInput struct { LogFolder string diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index 47928806ad34..c6a9e31c821a 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -35,9 +35,13 @@ import ( type InitManagementClusterAndWatchControllerLogsInput struct { ClusterProxy framework.ClusterProxy ClusterctlConfigPath string + CoreProvider string + BootstrapProviders []string + ControlPlaneProviders []string InfrastructureProviders []string LogFolder string DisableMetricsCollection bool + ClusterctlBinaryPath string } // InitManagementClusterAndWatchControllerLogs initializes a management using clusterctl and setup watches for controller logs. @@ -50,24 +54,40 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init Expect(input.InfrastructureProviders).ToNot(BeEmpty(), "Invalid argument. input.InfrastructureProviders can't be empty when calling InitManagementClusterAndWatchControllerLogs") Expect(os.MkdirAll(input.LogFolder, 0755)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for InitManagementClusterAndWatchControllerLogs") + if input.CoreProvider == "" { + input.CoreProvider = config.ClusterAPIProviderName + } + if len(input.BootstrapProviders) == 0 { + input.BootstrapProviders = []string{config.KubeadmBootstrapProviderName} + } + if len(input.ControlPlaneProviders) == 0 { + input.ControlPlaneProviders = []string{config.KubeadmControlPlaneProviderName} + } + client := input.ClusterProxy.GetClient() controllersDeployments := framework.GetControllerDeployments(ctx, framework.GetControllerDeploymentsInput{ Lister: client, }) if len(controllersDeployments) == 0 { - Init(ctx, InitInput{ + initInput := InitInput{ // pass reference to the management cluster hosting this test KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(), // pass the clusterctl config file that points to the local provider repository created for this test ClusterctlConfigPath: input.ClusterctlConfigPath, // setup the desired list of providers for a single-tenant management cluster - CoreProvider: config.ClusterAPIProviderName, - BootstrapProviders: []string{config.KubeadmBootstrapProviderName}, - ControlPlaneProviders: []string{config.KubeadmControlPlaneProviderName}, + CoreProvider: input.CoreProvider, + BootstrapProviders: input.BootstrapProviders, + ControlPlaneProviders: input.ControlPlaneProviders, InfrastructureProviders: input.InfrastructureProviders, // setup clusterctl logs folder LogFolder: input.LogFolder, - }) + } + + if input.ClusterctlBinaryPath != "" { + InitWithBinary(ctx, input.ClusterctlBinaryPath, initInput) + } else { + Init(ctx, initInput) + } } log.Logf("Waiting for provider controllers to be running") @@ -101,6 +121,63 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init } } +// UpgradeManagementClusterAndWaitInput is the input type for UpgradeManagementClusterAndWait. +type UpgradeManagementClusterAndWaitInput struct { + ClusterProxy framework.ClusterProxy + ClusterctlConfigPath string + ManagementGroup string + Contract string + LogFolder string +} + +// UpgradeManagementClusterAndWait upgrades provider a management cluster using clusterctl, and waits for the cluster to be ready. +func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagementClusterAndWaitInput, intervals ...interface{}) { + Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeManagementClusterAndWait") + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeManagementClusterAndWait") + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling UpgradeManagementClusterAndWait") + Expect(input.ManagementGroup).ToNot(BeEmpty(), "Invalid argument. input.ManagementGroup can't be empty when calling UpgradeManagementClusterAndWait") + Expect(input.Contract).ToNot(BeEmpty(), "Invalid argument. input.Contract can't be empty when calling UpgradeManagementClusterAndWait") + Expect(os.MkdirAll(input.LogFolder, 0755)).To(Succeed(), "Invalid argument. input.LogFolder can't be created for UpgradeManagementClusterAndWait") + + Upgrade(ctx, UpgradeInput{ + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(), + ManagementGroup: input.ManagementGroup, + Contract: input.Contract, + LogFolder: input.LogFolder, + }) + + client := input.ClusterProxy.GetClient() + + log.Logf("Waiting for provider controllers to be running") + controllersDeployments := framework.GetControllerDeployments(ctx, framework.GetControllerDeploymentsInput{ + Lister: client, + ExcludeNamespaces: []string{"capi-webhook-system"}, // this namespace has been dropped in v1alpha4; this ensures we are not waiting for deployments being deleted as part of the upgrade process + }) + Expect(controllersDeployments).ToNot(BeEmpty(), "The list of controller deployments should not be empty") + for _, deployment := range controllersDeployments { + framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: client, + Deployment: deployment, + }, intervals...) + + // Start streaming logs from all controller providers + framework.WatchDeploymentLogs(ctx, framework.WatchDeploymentLogsInput{ + GetLister: client, + ClientSet: input.ClusterProxy.GetClientSet(), + Deployment: deployment, + LogPath: filepath.Join(input.LogFolder, "controllers"), + }) + + framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{ + GetLister: client, + ClientSet: input.ClusterProxy.GetClientSet(), + Deployment: deployment, + MetricsPath: filepath.Join(input.LogFolder, "controllers"), + }) + } +} + // ApplyClusterTemplateAndWaitInput is the input type for ApplyClusterTemplateAndWait. type ApplyClusterTemplateAndWaitInput struct { ClusterProxy framework.ClusterProxy @@ -130,7 +207,7 @@ func ApplyClusterTemplateAndWait(ctx context.Context, input ApplyClusterTemplate Expect(result).ToNot(BeNil(), "Invalid argument. result can't be nil when calling ApplyClusterTemplateAndWait") log.Logf("Creating the workload cluster with name %q using the %q template (Kubernetes %s, %d control-plane machines, %d worker machines)", - input.ConfigCluster.ClusterName, valueOrDefault(input.ConfigCluster.Flavor), input.ConfigCluster.KubernetesVersion, *input.ConfigCluster.ControlPlaneMachineCount, *input.ConfigCluster.WorkerMachineCount) + input.ConfigCluster.ClusterName, valueOrDefault(input.ConfigCluster.Flavor), input.ConfigCluster.KubernetesVersion, input.ConfigCluster.ControlPlaneMachineCount, input.ConfigCluster.WorkerMachineCount) log.Logf("Getting the cluster template yaml") workloadClusterTemplate := ConfigCluster(ctx, ConfigClusterInput{ diff --git a/test/framework/clusterctl/e2e_config.go b/test/framework/clusterctl/e2e_config.go index 2c031a9d8cda..161fdc7f25dd 100644 --- a/test/framework/clusterctl/e2e_config.go +++ b/test/framework/clusterctl/e2e_config.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strconv" "strings" "time" @@ -533,6 +534,15 @@ func (c *E2EConfig) GetIntervals(spec, key string) []interface{} { return intervalsInterfaces } +func (c *E2EConfig) HasVariable(varName string) bool { + if _, ok := os.LookupEnv(varName); ok { + return true + } + + _, ok := c.Variables[varName] + return ok +} + // GetVariable returns a variable from environment variables or from the e2e config file. func (c *E2EConfig) GetVariable(varName string) string { if value, ok := os.LookupEnv(varName); ok { @@ -567,3 +577,34 @@ func (c *E2EConfig) GetInt32PtrVariable(varName string) *int32 { Expect(err).NotTo(HaveOccurred()) return pointer.Int32Ptr(int32(wCount)) } + +// GetProviderVersions returns the sorted list of versions defined for a provider. +func (c *E2EConfig) GetProviderVersions(provider string) []string { + versions := []string{} + for _, p := range c.Providers { + if p.Name == provider { + for _, v := range p.Versions { + versions = append(versions, v.Name) + } + } + } + + sort.Slice(versions, func(i, j int) bool { + // NOTE: Ignoring errors because the validity of the format is ensured by Validation. + vI, _ := version.ParseSemantic(versions[i]) + vJ, _ := version.ParseSemantic(versions[j]) + return vI.LessThan(vJ) + }) + return versions +} + +func (c *E2EConfig) GetProvidersWithOldestVersion(providers ...string) []string { + ret := make([]string, 0, len(providers)) + for _, p := range providers { + versions := c.GetProviderVersions(p) + if len(versions) > 0 { + ret = append(ret, fmt.Sprintf("%s:%s", p, versions[0])) + } + } + return ret +} diff --git a/test/framework/controller_helpers.go b/test/framework/controller_helpers.go index 5c35d6f899c8..2ab726eae147 100644 --- a/test/framework/controller_helpers.go +++ b/test/framework/controller_helpers.go @@ -26,7 +26,8 @@ import ( // GetControllerDeploymentsInput is the input for GetControllerDeployments. type GetControllerDeploymentsInput struct { - Lister Lister + Lister Lister + ExcludeNamespaces []string } // GetControllerDeployments returns all the deployment for the cluster API controllers existing in a management cluster. @@ -34,9 +35,24 @@ func GetControllerDeployments(ctx context.Context, input GetControllerDeployment deploymentList := &appsv1.DeploymentList{} Expect(input.Lister.List(ctx, deploymentList, capiProviderOptions()...)).To(Succeed(), "Failed to list deployments for the cluster API controllers") - deployments := make([]*appsv1.Deployment, len(deploymentList.Items)) + deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items)) for i := range deploymentList.Items { - deployments[i] = &deploymentList.Items[i] + d := &deploymentList.Items[i] + if !skipDeployment(d, input.ExcludeNamespaces) { + deployments = append(deployments, d) + } } return deployments } + +func skipDeployment(d *appsv1.Deployment, excludeNamespaces []string) bool { + if !d.DeletionTimestamp.IsZero() { + return true + } + for _, n := range excludeNamespaces { + if d.Namespace == n { + return true + } + } + return false +} diff --git a/test/framework/convenience.go b/test/framework/convenience.go index 3ede32e2aae2..a46b62cca8a0 100644 --- a/test/framework/convenience.go +++ b/test/framework/convenience.go @@ -25,6 +25,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1beta "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/runtime" + clusterv1old "sigs.k8s.io/cluster-api/api/v1alpha3" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4" @@ -46,13 +47,16 @@ func TryAddDefaultSchemes(scheme *runtime.Scheme) { // Add the apps schemes. _ = appsv1.AddToScheme(scheme) - // Add the core CAPI scheme. + // Add the core CAPI v1alpha4 scheme. _ = clusterv1.AddToScheme(scheme) - // Add the experiments CAPI scheme. + // Add the CAPI v1alpha4 experiments scheme. _ = expv1.AddToScheme(scheme) _ = addonsv1.AddToScheme(scheme) + // Add the core CAPI v1alpha3 scheme. + _ = clusterv1old.AddToScheme(scheme) + // Add the kubeadm bootstrapper scheme. _ = bootstrapv1.AddToScheme(scheme) diff --git a/test/framework/interfaces.go b/test/framework/interfaces.go index f6eca9a1efff..95e2dd33b1a2 100644 --- a/test/framework/interfaces.go +++ b/test/framework/interfaces.go @@ -49,11 +49,3 @@ type GetLister interface { Getter Lister } - -// ComponentGenerator is used to install components, generally any YAML bundle. -type ComponentGenerator interface { - // GetName returns the name of the component. - GetName() string - // Manifests return the YAML bundle. - Manifests(context.Context) ([]byte, error) -} diff --git a/test/infrastructure/docker/config/webhook/service.yaml b/test/infrastructure/docker/config/webhook/service.yaml index 31e0f8295919..67b7891bf8a4 100644 --- a/test/infrastructure/docker/config/webhook/service.yaml +++ b/test/infrastructure/docker/config/webhook/service.yaml @@ -1,4 +1,3 @@ - apiVersion: v1 kind: Service metadata: @@ -7,6 +6,5 @@ metadata: spec: ports: - port: 443 - targetPort: 9443 - selector: - control-plane: controller-manager + targetPort: webhook-server +