refactor: separate CAPA resources from cluster

Gopkg.toml

@@ -65,3 +65,7 @@ version="v1.4.7"
   [[prune.project]]
   name = "sigs.k8s.io/cluster-api"
   unused-packages = false
+
+[[constraint]]
+  name = "github.com/aws/aws-sdk-go"
+  version = "1.17.5"

cmd/clusterawsadm/cmd/alpha/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//cmd/clusterawsadm/cmd/alpha/bootstrap:go_default_library",
+        "//cmd/clusterawsadm/cmd/alpha/migrate:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
     ],
 )

cmd/clusterawsadm/cmd/alpha/alpha.go

@@ -19,6 +19,7 @@ package alpha
 import (
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/alpha/bootstrap"
+	"sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/alpha/migrate"
 )
 
 // AlphaCmd is the top-level alpha set of commands
@@ -32,5 +33,6 @@ func AlphaCmd() *cobra.Command { // nolint
 		},
 	}
 	newCmd.AddCommand(bootstrap.RootCmd())
+	newCmd.AddCommand(migrate.MigrateCmd())
 	return newCmd
 }

cmd/clusterawsadm/cmd/alpha/migrate/BUILD

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["migrate.go"],
+    importpath = "sigs.k8s.io/cluster-api-provider-aws/cmd/clusterawsadm/cmd/alpha/migrate",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/cloud/aws/tags:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
+        "//vendor/github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
+    ],
+)

cmd/clusterawsadm/cmd/alpha/migrate/migrate.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrate
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	awstags "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
+	"github.com/spf13/cobra"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/tags"
+)
+
+var (
+	// See: https://docs.aws.amazon.com/sdk-for-go/api/service/resourcegroupstaggingapi/#TagResourcesInput
+	maxARNs           = 20
+	supportedVersions = []string{"0.3.0"}
+
+	clusterName string
+)
+
+// MigrateCmd is the command for migrating AWS resources to be compatible
+// with specific CAPA versions
+func MigrateCmd() *cobra.Command { // nolint
+	newCmd := &cobra.Command{
+		Use:   "migrate [target version]",
+		Short: "migrate between CAPA versions",
+		Long: fmt.Sprintf(`Migrate AWS resources between incompatible versions of Cluster API Provider AWS.
+Supported versions: %v`, supportedVersions),
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) != 1 {
+				fmt.Printf("Error: requires target version as an argument. Supported versions: %v\n\n", supportedVersions)
+				if err := cmd.Help(); err != nil {
+					return err
+				}
+				os.Exit(200)
+			}
+			if !isValidVersion(args[0]) {
+				fmt.Printf("Error: unsupported migration target. Supported versions: %v\n\n", supportedVersions)
+				cmd.Help()
+				os.Exit(201)
+			}
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			sess, err := session.NewSessionWithOptions(session.Options{
+				SharedConfigState: session.SharedConfigEnable,
+			})
+			if err != nil {
+				fmt.Printf("Error: %v", err)
+				return nil
+			}
+
+			tagsSvc := awstags.New(sess)
+
+			resources, err := getResourcesByCluster(tagsSvc, clusterName)
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("Found %v resources owned by cluster %q.\n", len(resources), clusterName)
+			fmt.Printf("Applying new tags to cluster %q.\n", clusterName)
+
+			err = applyNewTags(tagsSvc, resources, clusterName)
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("Removing legacy tags from cluster %q\n", clusterName)
+
+			return removeOldTags(tagsSvc, resources, clusterName)
+		},
+	}
+
+	newCmd.Flags().StringVarP(&clusterName, "clusterName", "n", "", "name of existing Cluster object")
+	newCmd.MarkFlagRequired("clusterName")
+
+	return newCmd
+}
+
+func getResourcesByCluster(svc *awstags.ResourceGroupsTaggingAPI, name string) ([]*string, error) {
+	input := &awstags.GetResourcesInput{
+		TagFilters: []*awstags.TagFilter{
+			{
+				Key:    aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", name)),
+				Values: []*string{aws.String("owned")},
+			},
+			{
+				Key:    aws.String("sigs.k8s.io/cluster-api-provider-aws/managed"),
+				Values: []*string{aws.String("true")},
+			},
+		},
+	}
+
+	out, err := svc.GetResources(input)
+	if err != nil {
+		return nil, err
+	}
+
+	arns := make([]*string, 0, len(out.ResourceTagMappingList))
+	for _, resource := range out.ResourceTagMappingList {
+		arns = append(arns, resource.ResourceARN)
+	}
+
+	return arns, nil
+}
+
+func applyNewTags(svc *awstags.ResourceGroupsTaggingAPI, arns []*string, name string) error {
+
+	for i := 0; i <= (len(arns) / maxARNs); i++ {
+		end := (i + 1) * maxARNs
+		if end > len(arns) {
+			end = len(arns)
+		}
+
+		input := &awstags.TagResourcesInput{
+			ResourceARNList: arns[i*maxARNs : end],
+			Tags: map[string]*string{
+				tags.ClusterKey(name): aws.String("owned"),
+			},
+		}
+
+		_, err := svc.TagResources(input)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func removeOldTags(svc *awstags.ResourceGroupsTaggingAPI, arns []*string, name string) error {
+	for i := 0; i <= (len(arns) / maxARNs); i++ {
+		end := (i + 1) * maxARNs
+		if end > len(arns) {
+			end = len(arns)
+		}
+
+		managedInput := &awstags.UntagResourcesInput{
+			ResourceARNList: arns[i*maxARNs : end],
+			TagKeys: []*string{
+				aws.String("sigs.k8s.io/cluster-api-provider-aws/managed"),
+			},
+		}
+
+		_, err := svc.UntagResources(managedInput)
+		if err != nil {
+			return err
+		}
+	}
+
+	var filteredARNs []*string
+
+	for _, v := range arns {
+		// instances should have both ownership tags, so filter those out
+		// TODO(rudoi): is there a better way to filter?
+		if !strings.Contains(aws.StringValue(v), "instance") {
+			filteredARNs = append(filteredARNs, v)
+		}
+	}
+
+	for i := 0; i <= (len(filteredARNs) / maxARNs); i++ {
+		end := (i + 1) * maxARNs
+		if i == (len(filteredARNs) / maxARNs) {
+			end = len(filteredARNs)
+		}
+
+		ownedInput := &awstags.UntagResourcesInput{
+			ResourceARNList: filteredARNs[i*maxARNs : end],
+			TagKeys: []*string{
+				aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", name)),
+			},
+		}
+
+		_, err := svc.UntagResources(ownedInput)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func isValidVersion(s string) bool {
+	for _, v := range supportedVersions {
+		if s == v {
+			return true
+		}
+	}
+	return false
+}

docs/proposal/aws-resource-handling.md

@@ -38,7 +38,11 @@ Where possible use [client tokens](https://docs.aws.amazon.com/AWSEC2/latest/API
 
 ## Tagging of resources
 
-Resources that are managed by the controllers/actuators should be tagged with: `kubernetes.io/cluster/<name or id>=owned` and `sigs.k8s.io/cluster-api-provider-aws=true`. The latter tag being used to differentiate from resources managed by other tools/components that make use of the common tag.
+Resources handled by these components fall into one of three categories:
+
+1. Fully-managed resources whose lifecycle is tied to the cluster. These resources should be tagged with `sigs.k8s.io/cluster-api-provider-aws/cluster/<name or id>=owned`, and the actuator is expected to keep these resources as closely in sync with the spec as possible.
+2. Resources whose management is shared with the in-cluster aws cloud provider, such as a security group for load balancer ingress rules. These resources should be tagged with `sigs.k8s.io/cluster-api-provider-aws/cluster/<name or id>=owned` and `kubernetes.io/cluster/<name or id>=owned`, with the latter being the tag defined by the cloud provider. These resources are create/delete only: that is to say their ongoing management is "handed off" to the cloud provider.
+3. Unmanaged resources that are provided by config (such as a common VPC). The provider will avoid changing these resources as much as is possible.
 
 TODO: Define additional tags that can be used to provide additional metadata about the resource configuration/usage by the actuator. This is would allow us to rebuild status without relying on polluting the object config.
 

docs/upgrade-to-0.3.0.md

@@ -0,0 +1,13 @@
+# Upgrading to 0.3.0
+
+In 0.3.0, the tagging scheme changed for identifying AWS resources. In order not to lose track, there is a partial migration tool included in 0.3.0's `clusterawsadm`.
+
+The migration path is as follows:
+
+1. `kubectl scale statefulset -n aws-provider-system aws-provider-controller-manager --replicas=0`
+2. `clusterawsadm migrate -n CLUSTER_NAME 0.3.0`
+3. Update the image for the aws-provider-controller-manager
+4. `kubectl scale statefulset -n aws-provider-system aws-provider-controller-manager --replicas=1`
+5. Wait ~2 minutes for the security group changes to all settle
+   - All of the nodes and control plane machines should have exactly one security group tagged with `kubernetes.io/cluster/<CLUSTER_NAME>=owned`: the new `CLUSTER_NAME-lb` group.
+6. Find the names of your controller-manager pods, and run `kubectl exec -n kube-system -it CONTROLLER_MANAGER_POD_NAME -- sh -c 'kill 1' as a workaround for kubernetes/kubernetes#77019`

pkg/apis/awsprovider/v1alpha1/awsclusterproviderconfig_types.go

@@ -111,9 +111,9 @@ func (v *VPCSpec) String() string {
 	return fmt.Sprintf("id=%s", v.ID)
 }
 
-// IsProvided returns true if the VPC is unmanaged.
-func (v *VPCSpec) IsProvided() bool {
-	return v.ID != "" && !v.Tags.HasManaged()
+// IsUnmanaged returns true if the VPC is unmanaged.
+func (v *VPCSpec) IsUnmanaged(clusterName string) bool {
+	return v.ID != "" && !v.Tags.HasOwned(clusterName)
 }
 
 // SubnetSpec configures an AWS Subnet.

pkg/apis/awsprovider/v1alpha1/types.go

@@ -23,6 +23,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/aws/tags"
 )
 
 // AWSResourceReference is a reference to a specific AWS resource by ID, ARN, or filters.
@@ -237,6 +238,9 @@ var (
 
 	// SecurityGroupControlPlane defines a Kubernetes control plane node role
 	SecurityGroupControlPlane = SecurityGroupRole("controlplane")
+
+	// SecurityGroupLB defines a container for the cloud provider to inject its load balancer ingress rules
+	SecurityGroupLB = SecurityGroupRole("lb")
 )
 
 // SecurityGroup defines an AWS security group.
@@ -251,7 +255,7 @@ type SecurityGroup struct {
 	IngressRules IngressRules `json:"ingressRule"`
 
 	// Tags is a map of tags associated with the security group.
-	Tags map[string]string `json:"tags,omitempty"`
+	Tags tags.Map `json:"tags,omitempty"`
 }
 
 // String returns a string representation of the security group.

pkg/cloud/aws/actuators/machine/actuator.go

@@ -374,13 +374,18 @@ func (a *Actuator) Update(ctx context.Context, cluster *clusterv1.Cluster, machi
 		return errors.Errorf("found attempt to change immutable state for machine %q: %+q", machine.Name, errs)
 	}
 
+	existingSecurityGroups, err := ec2svc.GetInstanceSecurityGroups(*scope.MachineStatus.InstanceID)
+	if err != nil {
+		return err
+	}
+
 	// Ensure that the security groups are correct.
 	_, err = a.ensureSecurityGroups(
 		ec2svc,
-		machine,
+		scope,
 		*scope.MachineStatus.InstanceID,
 		scope.MachineConfig.AdditionalSecurityGroups,
-		instanceDescription.SecurityGroupIDs,
+		existingSecurityGroups,
 	)
 	if err != nil {
 		return errors.Errorf("failed to apply security groups: %+v", err)