Skip to content

Commit

Permalink
Add ClusterResourceSet e2e test
Browse files Browse the repository at this point in the history
  • Loading branch information
Sedef committed May 29, 2020
1 parent ae45ada commit 1564d8a
Show file tree
Hide file tree
Showing 7 changed files with 390 additions and 17 deletions.
47 changes: 35 additions & 12 deletions exp/clusterresourceset/controllers/clusterresourceset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ func GetConfigMapFromNamespacedName(ctx context.Context, c client.Client, secret
// applyClusterResourceSet applies resources in a ClusterResourceSet to a Cluster. Once applied, a record will be added to the
// cluster's ClusterResourceSetBinding.
// In ApplyOnce mode, resources are applied only once to a particular cluster. ClusterResourceSetBinding is used to check if a resource is applied before.
// It applies resources best effort and continue on scenarios like: unsupported resource types, failure during creation, missing resources.
// TODO: If a resource already exists in the cluster but not applied by ClusterResourceSet, the resource will be updated ?
func (r *ClusterResourceSetReconciler) applyClusterResourceSet(ctx context.Context, cluster *clusterv1.Cluster, clusterResourceSet *clusterresourcesetv1.ClusterResourceSet) error {
logger := r.Log.WithValues("ClusterResourceSet", clusterResourceSet.Name, "namespace", clusterResourceSet.Namespace)
Expand Down Expand Up @@ -202,6 +203,8 @@ func (r *ClusterResourceSetReconciler) applyClusterResourceSet(ctx context.Conte
return errors.New("failed to find ClusterResourceSet " + clusterResourceSet.Name + " in ClusterResourceBinding")
}

errList := []error{}

// Iterate all resources and apply them to the cluster and update the resource status in the ClusterResourceSetBinding object.
for _, resource := range clusterResourceSet.Spec.Resources {
// If resource is already applied successfully and clusterResourceSet mod is "ApplyOnce", continue. (No need to check hash changes here)
Expand All @@ -219,33 +222,52 @@ func (r *ClusterResourceSetReconciler) applyClusterResourceSet(ctx context.Conte
// Retrieve data in the resource as an array because there can be many key-value pairs in resource and all will be applied to the cluster.
switch resource.Kind {
case string(clusterresourcesetv1.ConfigMapClusterResourceSetResourceKind):
// Set status in ClusterResourceSetBinding in case of early continue due to a failure. Do this for only supported kinds.
clusterResourceSetInMap.Resources[GenerateResourceKindName(resource)] = clusterresourcesetv1.ClusterResourceSetResourceStatus{
Hash: "",
Successful: false,
LastAppliedTime: &metav1.Time{Time: time.Now().UTC()},
}

resourceConfigMap, err := GetConfigMapFromNamespacedName(context.Background(), r.Client, typedName)
if err != nil {
return errors.Wrapf(err,
"failed to fetch ClusterResourceSet configmap %q in namespace %q", resource.Name, clusterResourceSet.Namespace)
logger.Error(err, "Failed to find ConfigMap resource", "ConfigMap name", typedName)
errList = append(errList, err)
continue
}

for _, dataStr := range resourceConfigMap.Data {
data := []byte(dataStr)
dataList = append(dataList, data)
}
case string(clusterresourcesetv1.SecretClusterResourceSetResourceKind):
// Set status in ClusterResourceSetBinding in case of early continue due to a failure. Do this for only supported kinds.
clusterResourceSetInMap.Resources[GenerateResourceKindName(resource)] = clusterresourcesetv1.ClusterResourceSetResourceStatus{
Hash: "",
Successful: false,
LastAppliedTime: &metav1.Time{Time: time.Now().UTC()},
}

resourceSecret, err := secret.GetAnySecretFromNamespacedName(context.Background(), r.Client, typedName)
if err != nil {
return errors.Wrapf(err,
"failed to fetch ClusterResourceSet secret %q in namespace %q", resource.Name, clusterResourceSet.Namespace)
logger.Error(err, "Failed to find Secret resource", "Secret name", typedName)
errList = append(errList, err)
continue
}

if resourceSecret.Type != clusterresourcesetv1.ClusterResourceSetSecretType {
return errors.Wrapf(err,
"Unsupported ClusterResourceSet secret resource: %q/%q type: %q", clusterResourceSet.Namespace, resource.Name, resourceSecret.Type)
logger.Error(err, "Unsupported ClusterResourceSet Secret type", "Resource type", resourceSecret.Type)
errList = append(errList, err)
continue
}
for _, dataStr := range resourceSecret.Data {
dataList = append(dataList, dataStr)
}

default:
return errors.New("resource kind is not supported")
logger.Error(err, "Unsupported ClusterResourceSet resource kind", "Resource Kind", resource.Kind)
errList = append(errList, err)
continue
}

// Apply all values in the key-value pair of the resource to the cluster.
Expand All @@ -256,15 +278,16 @@ func (r *ClusterResourceSetReconciler) applyClusterResourceSet(ctx context.Conte

if err := Apply(ctx, c, data); err != nil {
isSuccessful = false
logger.Error(err,
"failed to apply ClusterResourceSet resource %q/%q in namespace %q", resource.Kind, resource.Name, clusterResourceSet.Namespace)
logger.Error(err, "failed to apply ClusterResourceSet resource %q/%q in namespace %q", "Resource kind", resource.Kind, "Resource name", resource.Name)
errList = append(errList, err)
}
}

resourceInMap = *GenerateResourceStatus(isSuccessful, ComputeHash(dataList))
// or clusterResourceSetBinding.ClusterResourceSetMap[clusterResourceSet.Name].Resources[GenerateResourceKindName(resource)] = *GenerateResourceStatus(isSuccessful, ComputeHash(dataList))
clusterResourceSetInMap.Resources[GenerateResourceKindName(resource)] = resourceInMap

}
if len(errList) > 0 {
return kerrors.NewAggregate(errList)
}
return nil
}
Expand Down Expand Up @@ -307,7 +330,7 @@ func (r *ClusterResourceSetReconciler) clusterToClusterResourceSet(o handler.Map
}

resourceList := &clusterresourcesetv1.ClusterResourceSetList{}
if err := r.Client.List(context.Background(), resourceList, client.MatchingLabels(cluster.GetLabels())); err != nil {
if err := r.Client.List(context.Background(), resourceList, client.InNamespace(cluster.Namespace), client.MatchingLabels(cluster.GetLabels())); err != nil {
r.Log.Error(err, "failed to list ClusterResourceSet")
return nil
}
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/config/docker-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ providers:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
new: "--enable-leader-election=false\n - --feature-gates=ClusterResourceSet=true"

- name: kubeadm
type: BootstrapProvider
Expand Down
37 changes: 36 additions & 1 deletion test/e2e/data/infrastructure-docker/cluster-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,39 @@ spec:
unhealthyConditions:
- type: E2ENodeUnhealthy
status: "True"
timeout: 30s
timeout: 30s
---
apiVersion: clusterresourceset.cluster.x-k8s.io/v1alpha3
kind: ClusterResourceSet
metadata:
name: "${CLUSTER_NAME}-crs-0"
spec:
clusterSelector:
matchLabels:
label: e2e
resources:
- name: test-secret1
kind: Secret
- name: test-configmap1
kind: ConfigMap
- name: test-notexist
kind: Secret
---
apiVersion: v1
kind: Secret
metadata:
name: test-secret1
type: clusterResourceSet
stringData:
addon.yaml: |-
{
"kind": "ConfigMap",
"apiVersion": "v1",
"metadata": {
"name": "test-e2e-cm",
"namespace": "default"
},
"data": {
"service_name": "none"
}
}
112 changes: 112 additions & 0 deletions test/e2e/experimental_features.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"
"fmt"
"os"
"path/filepath"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
clusterresourcesetv1 "sigs.k8s.io/cluster-api/exp/clusterresourceset/api/v1alpha3"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/cluster-api/util"
)

// ExperimentalFeaturesSpecInput is the input for ExperimentalFeaturesSpec.
type ExperimentalFeaturesSpecInput struct {
E2EConfig *clusterctl.E2EConfig
ClusterctlConfigPath string
BootstrapClusterProxy framework.ClusterProxy
ArtifactFolder string
SkipCleanup bool
}

// ExperimentalFeaturesSpec implements a spec that tests experimental featured that are gated with feature gates.
// Once an experimental feature graduates, its test can be a separate spec.
func ExperimentalFeaturesSpec(ctx context.Context, inputGetter func() ExperimentalFeaturesSpecInput) {
var (
specName = "experimental-features"
input ExperimentalFeaturesSpecInput
namespace *corev1.Namespace
cancelWatches context.CancelFunc
cluster *clusterv1.Cluster
)

BeforeEach(func() {
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
input = inputGetter()
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(input.E2EConfig.Variables).To(HaveKey(CNIPath))

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
})

It("Should successfully apply resources in ClusterResourceSet object to all matching clusters", func() {

By("Creating a workload cluster")

cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
ClusterProxy: input.BootstrapClusterProxy,
ConfigCluster: clusterctl.ConfigClusterInput{
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
ClusterctlConfigPath: input.ClusterctlConfigPath,
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
},
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})

// Add the experimental feature ClusterResourceSet to the scheme.
_ = clusterresourcesetv1.AddToScheme(input.BootstrapClusterProxy.GetScheme())

By("Patching cluster with a matching label with ClusterResourceSet and wait for creation of resources")
framework.DiscoverClusterResourceSetAndWaitForSuccess(ctx, framework.DiscoverClusterResourceSetAndWaitForSuccessInput{
ClusterProxy: input.BootstrapClusterProxy,
Cluster: cluster,
}, input.E2EConfig.GetIntervals(specName, "wait-cluster")...)

By("PASSED!")
})

AfterEach(func() {
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
})
}
39 changes: 39 additions & 0 deletions test/e2e/experimental_features_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// +build e2e

/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"

. "github.com/onsi/ginkgo"
)

var _ = Describe("When testing the Cluster API experimental features", func() {

ExperimentalFeaturesSpec(context.TODO(), func() ExperimentalFeaturesSpecInput {
return ExperimentalFeaturesSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
}
})

})
28 changes: 25 additions & 3 deletions test/framework/cluster_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,14 @@ import (

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
"sigs.k8s.io/cluster-api/test/framework/internal/log"
"sigs.k8s.io/cluster-api/test/framework/options"
"sigs.k8s.io/cluster-api/util/patch"
"sigs.k8s.io/controller-runtime/pkg/client"

"k8s.io/apimachinery/pkg/runtime"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
)

// CreateClusterInput is the input for CreateCluster.
Expand Down Expand Up @@ -92,6 +93,27 @@ func GetClusterByName(ctx context.Context, input GetClusterByNameInput) *cluster
return cluster
}

// PatchClusterLabelInput is the input for PatchClusterLabel.
type PatchClusterLabelInput struct {
ClusterProxy ClusterProxy
Cluster *clusterv1.Cluster
Labels map[string]string
}

// PatchClusterLabel patches labels to a cluster.
func PatchClusterLabel(ctx context.Context, input PatchClusterLabelInput) {
Expect(ctx).NotTo(BeNil(), "ctx is required for PatchClusterLabel")
Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling PatchClusterLabel")
Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling PatchClusterLabel")
Expect(input.Labels).ToNot(BeEmpty(), "Invalid argument. input.Labels can't be empty when calling PatchClusterLabel")

log.Logf("Patching the label to the cluster")
patchHelper, err := patch.NewHelper(input.Cluster, input.ClusterProxy.GetClient())
Expect(err).ToNot(HaveOccurred())
input.Cluster.SetLabels(input.Labels)
Expect(patchHelper.Patch(ctx, input.Cluster)).To(Succeed())
}

// WaitForClusterToProvisionInput is the input for WaitForClusterToProvision.
type WaitForClusterToProvisionInput struct {
Getter Getter
Expand Down
Loading

0 comments on commit 1564d8a

Please sign in to comment.