diff --git a/client.go b/client.go index a640c3a8..452ed65f 100644 --- a/client.go +++ b/client.go @@ -129,3 +129,15 @@ func NewClient(kubeClient kubernetes.Interface, mSpec *cov1.MachineSetSpec, name elbClient: elb.New(s), }, nil } + +func getIgn(kubeClient kubernetes.Interface) (string, error) { + ignConfig, err := kubeClient.CoreV1().ConfigMaps("kube-system").Get("ign-config", metav1.GetOptions{}) + if err != nil { + return "", err + } + ignConfigWorker, ok := ignConfig.Data["worker"] + if !ok { + return "", nil + } + return ignConfigWorker, nil +} diff --git a/examples/machine.yaml b/examples/machine.yaml index 04770953..8aa82828 100644 --- a/examples/machine.yaml +++ b/examples/machine.yaml @@ -17,7 +17,7 @@ spec: value: apiVersion: awsproviderconfig/v1alpha1 kind: AWSClusterProviderConfig - clusterId: test + clusterId: meh.tectonic.kuwit.rocks clusterVersionRef: namespace: test name: test @@ -31,7 +31,7 @@ spec: sslSecret: name: test-certs region: eu-west-1 - keyPairName: test + keyPairName: tectonic defaultHardwareSpec: aws: instanceType: t1.micro @@ -49,7 +49,7 @@ spec: apiVersion: "cluster.k8s.io/v1alpha1" kind: Machine metadata: - name: test + name: extra-worker namespace: test generateName: vs-master- labels: @@ -59,12 +59,12 @@ spec: value: apiVersion: awsproviderconfig/v1alpha1 kind: AWSMachineProviderConfig - clusterId: test + clusterId: meh.tectonic.kuwit.rocks clusterHardware: aws: accountSecret: name: test-aws-creds - keyPairName: test + keyPairName: tectonic region: eu-west-1 sshSecret: name: test-ssh-key @@ -76,8 +76,8 @@ spec: instanceType: t1.micro infra: false vmImage: - # CoreOS-stable-1520.5.0 - awsImage: ami-03f6257a + # CoreOS-beta-1828.3.0-hvm + awsImage: ami-0518e1ac70d8a3389 versions: kubelet: 1.10.1 controlPlane: 1.10.1 diff --git a/machineactuator.go b/machineactuator.go index 599305fe..1b4bf5a8 100644 --- a/machineactuator.go +++ b/machineactuator.go @@ -17,18 +17,16 @@ limitations under the License. package aws import ( - "bytes" "encoding/base64" "fmt" - "io/ioutil" - "text/template" + "strings" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" capicommon "sigs.k8s.io/cluster-api/pkg/apis/cluster/common" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" @@ -37,30 +35,15 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + awsconfigv1 "github.com/enxebre/cluster-api-provider-aws/awsproviderconfig/v1alpha1" cov1 "github.com/enxebre/cluster-api-provider-aws/awsproviderconfig/v1alpha1" "github.com/openshift/cluster-operator/pkg/controller" clustoplog "github.com/openshift/cluster-operator/pkg/logging" - awsconfigv1 "github.com/enxebre/cluster-api-provider-aws/awsproviderconfig/v1alpha1" ) const ( - // Path to bootstrap kubeconfig. This needs to be mounted to the controller pod - // as a secret when running this controller. - bootstrapKubeConfig = "/etc/origin/master/bootstrap.kubeconfig" - - // IAM role for infra/compute - defaultIAMRole = "openshift_node_describe_instances" - - // IAM role for master - masterIAMRole = "openshift_master_launch_instances" - - // Instance ID annotation - instanceIDAnnotation = "cluster-operator.openshift.io/aws-instance-id" - awsCredsSecretIDKey = "awsAccessKeyId" awsCredsSecretAccessKey = "awsSecretAccessKey" - - ec2InstanceIDNotFoundCode = "InvalidInstanceID.NotFound" ) // Instance tag constants @@ -79,15 +62,20 @@ var stateMask int64 = 0xFF // Actuator is the AWS-specific actuator for the Cluster API machine controller type Actuator struct { - kubeClient kubernetes.Interface - clusterClient clusterclient.Interface + kubeClient kubernetes.Interface + clusterClient clusterclient.Interface //codecFactory serializer.CodecFactory defaultAvailabilityZone string logger *log.Entry clientBuilder func(kubeClient kubernetes.Interface, mSpec *cov1.MachineSetSpec, namespace, region string) (Client, error) - userDataGenerator func(master, infra bool) (string, error) - awsProviderConfigCodec *awsconfigv1.AWSProviderConfigCodec - scheme *runtime.Scheme + //userDataGenerator func(master, infra bool) (string, error) + awsProviderConfigCodec *awsconfigv1.AWSProviderConfigCodec + scheme *runtime.Scheme + ignConfig func(kubeClient kubernetes.Interface) (string, error) +} + +func getWorkerRole() { + } // NewActuator returns a new AWS Actuator @@ -103,15 +91,16 @@ func NewActuator(kubeClient kubernetes.Interface, clusterClient clusterclient.In } actuator := &Actuator{ - kubeClient: kubeClient, - clusterClient: clusterClient, + kubeClient: kubeClient, + clusterClient: clusterClient, //codecFactory: coapi.Codecs, defaultAvailabilityZone: defaultAvailabilityZone, logger: logger, clientBuilder: NewClient, - userDataGenerator: generateUserData, - awsProviderConfigCodec: codec, + //userDataGenerator: generateUserData, + awsProviderConfigCodec: codec, scheme: scheme, + ignConfig: getIgn, } return actuator } @@ -185,7 +174,8 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *clusterv1. } // Describe VPC - vpcName := awsClusterProviderConfig.ClusterDeploymentSpec.ClusterID + vpcName := awsProviderConfig.ClusterID + clusterName := strings.Split(vpcName, ".")[0] vpcNameFilter := "tag:Name" describeVpcsRequest := ec2.DescribeVpcsInput{ Filters: []*ec2.Filter{{Name: &vpcNameFilter, Values: []*string{&vpcName}}}, @@ -259,10 +249,9 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *clusterv1. // Add tags to the created machine tagList := []*ec2.Tag{ - {Key: aws.String("clusterid"), Value: aws.String(awsClusterProviderConfig.ClusterDeploymentSpec.ClusterID)}, - {Key: aws.String("host-type"), Value: aws.String(hostType)}, - {Key: aws.String("sub-host-type"), Value: aws.String(subHostType)}, - {Key: aws.String("kubernetes.io/cluster/" + awsClusterProviderConfig.ClusterDeploymentSpec.ClusterID), Value: aws.String(awsClusterProviderConfig.ClusterDeploymentSpec.ClusterID)}, + {Key: aws.String("clusterid"), Value: aws.String(vpcName)}, + {Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", clusterName)), Value: aws.String("owned")}, + {Key: aws.String("tectonicClusterID"), Value: aws.String("447c6a4c-92a9-0266-3a23-9e3495006e24")}, {Key: aws.String("Name"), Value: aws.String(machine.Name)}, } tagInstance := &ec2.TagSpecification{ @@ -274,34 +263,11 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *clusterv1. Tags: tagList[0:1], } - // For now, these are fixed - blkDeviceMappings := []*ec2.BlockDeviceMapping{ - { - DeviceName: aws.String("/dev/sda"), - Ebs: &ec2.EbsBlockDevice{ - DeleteOnTermination: aws.Bool(true), - VolumeSize: aws.Int64(100), - VolumeType: aws.String("gp2"), - }, - }, - //{ - // DeviceName: aws.String("/dev/sdb"), - // Ebs: &ec2.EbsBlockDevice{ - // DeleteOnTermination: aws.Bool(true), - // VolumeSize: aws.Int64(100), - // VolumeType: aws.String("gp2"), - // }, - //}, - } - - // Only compute nodes should get user data, and it's quite important that masters do not as the - // AWS actuator for these is running on the root CO cluster currently, and we do not want to leak - // root CO cluster bootstrap kubeconfigs to the target cluster. - //userData, err := a.userDataGenerator(controller.MachineHasRole(machine, capicommon.MasterRole), coMachineSetSpec.Infra) - //if err != nil { - // return nil, err - //} - userDataEnc := base64.StdEncoding.EncodeToString([]byte("")) + ignConfig, err := a.ignConfig(a.kubeClient) + if err != nil { + return nil, fmt.Errorf("unable to obtain EC2 client: %v", err) + } + userDataEnc := base64.StdEncoding.EncodeToString([]byte(ignConfig)) inputConfig := ec2.RunInstancesInput{ ImageId: describeAMIResult.Images[0].ImageId, @@ -310,12 +276,12 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *clusterv1. MaxCount: aws.Int64(1), KeyName: aws.String(awsClusterProviderConfig.ClusterDeploymentSpec.Hardware.AWS.KeyPairName), IamInstanceProfile: &ec2.IamInstanceProfileSpecification{ - Name: aws.String(iamRole(machine)), + Name: aws.String(iamRole(clusterName)), }, - BlockDeviceMappings: blkDeviceMappings, - TagSpecifications: []*ec2.TagSpecification{tagInstance, tagVolume}, - NetworkInterfaces: networkInterfaces, - UserData: &userDataEnc, + //BlockDeviceMappings: blkDeviceMappings, + TagSpecifications: []*ec2.TagSpecification{tagInstance, tagVolume}, + NetworkInterfaces: networkInterfaces, + UserData: &userDataEnc, InstanceInitiatedShutdownBehavior: aws.String(shutdownBehavior), } @@ -328,6 +294,8 @@ func (a *Actuator) CreateMachine(cluster *clusterv1.Cluster, machine *clusterv1. mLog.Errorf("unexpected reservation creating instances: %v", runResult) return nil, fmt.Errorf("unexpected reservation creating instance") } + + //addInstanceToELB(runResult.Instances[0], "", client) return runResult.Instances[0], nil } @@ -360,7 +328,8 @@ func (a *Actuator) DeleteMachine(machine *clusterv1.Machine) error { return fmt.Errorf("error getting EC2 client: %v", err) } - instances, err := GetRunningInstances(machine, client) + clusterId := awsProviderConfig.ClusterID + instances, err := GetRunningInstances(machine, client, clusterId) if err != nil { return err } @@ -400,7 +369,7 @@ func (a *Actuator) Update(cluster *clusterv1.Cluster, machine *clusterv1.Machine return fmt.Errorf("unable to obtain EC2 client: %v", err) } - instances, err := GetRunningInstances(machine, client) + instances, err := GetRunningInstances(machine, client, awsProviderConfig.ClusterID) mLog.Debugf("found %d instances for machine", len(instances)) if err != nil { return err @@ -456,7 +425,7 @@ func (a *Actuator) Exists(cluster *clusterv1.Cluster, machine *clusterv1.Machine return false, fmt.Errorf("error getting EC2 client: %v", err) } - instances, err := GetRunningInstances(machine, client) + instances, err := GetRunningInstances(machine, client, awsProviderConfig.ClusterID) if err != nil { return false, err } @@ -531,122 +500,13 @@ func (a *Actuator) updateStatus(machine *clusterv1.Machine, instance *ec2.Instan return nil } -func getClusterID(machine *clusterv1.Machine) (string, error) { - //coMachineSetSpec, err := controller.MachineSetSpecFromClusterAPIMachineSpec(&machine.Spec) - //if err != nil { - // return "", err - //} - //return coMachineSetSpec.ClusterID, nil - //TODO: get this dynamically - return "test", nil -} - -// template for user data -// takes the following parameters: -// 1 - type of machine (infra/compute) -// 2 - base64-encoded bootstrap.kubeconfig -const userDataTemplate = `#cloud-config -write_files: -- path: /root/openshift_bootstrap/openshift_settings.yaml - owner: 'root:root' - permissions: '0640' - content: | - openshift_group_type: {{ .NodeType }} -{{- if .IsNode }} -- path: /etc/origin/node/bootstrap.kubeconfig - owner: 'root:root' - permissions: '0640' - encoding: b64 - content: {{ .BootstrapKubeconfig }} -{{- end }} -runcmd: -- [ ansible-playbook, /root/openshift_bootstrap/bootstrap.yml] -{{- if .IsNode }} -- [ systemctl, restart, systemd-hostnamed] -- [ systemctl, restart, NetworkManager] -- [ systemctl, enable, origin-node] -- [ systemctl, start, origin-node] -{{- end }}` - -type userDataParams struct { - NodeType string - BootstrapKubeconfig string - IsNode bool -} - -func executeTemplate(isMaster, isInfra bool, bootstrapKubeconfig string) (string, error) { - var nodeType string - if isMaster { - nodeType = "master" - } else if isInfra { - nodeType = "infra" - } else { - nodeType = "compute" - } - params := userDataParams{ - NodeType: nodeType, - BootstrapKubeconfig: bootstrapKubeconfig, - IsNode: !isMaster, - } - - t, err := template.New("userdata").Parse(userDataTemplate) - if err != nil { - return "", err - } - var buf bytes.Buffer - err = t.Execute(&buf, params) - if err != nil { - return "", err - } - return buf.String(), nil -} - -// generateUserData is a generator function used in the actuator to create the user data for a -// specific type of machine. -func generateUserData(isMaster, isInfra bool) (string, error) { - var bootstrapKubeconfig string - var err error - if !isMaster { - bootstrapKubeconfig, err = getBootstrapKubeconfig() - if err != nil { - return "", fmt.Errorf("cannot get bootstrap kubeconfig: %v", err) - } - } - - return executeTemplate(isMaster, isInfra, bootstrapKubeconfig) -} - -// getBootstrapKubeconfig reads the bootstrap kubeconfig expected to be mounted into the pod. This assumes -// the actuator runs on a master which has such a kubeconfig for joining nodes to the cluster. -func getBootstrapKubeconfig() (string, error) { - content, err := ioutil.ReadFile(bootstrapKubeConfig) - if err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(content), nil -} - -func iamRole(machine *clusterv1.Machine) string { - if controller.MachineHasRole(machine, capicommon.MasterRole) { - return masterIAMRole - } - return defaultIAMRole +func iamRole(clusterName string) string { + return fmt.Sprintf("%s-master-profile", clusterName) } func buildDescribeSecurityGroupsInput(vpcID, vpcName string, isMaster, isInfra bool) *ec2.DescribeSecurityGroupsInput { groupNames := []*string{aws.String(vpcName)} - if isMaster { - groupNames = append(groupNames, aws.String(vpcName+"_master")) - groupNames = append(groupNames, aws.String(vpcName+"_master_k8s")) - } - if isInfra { - groupNames = append(groupNames, aws.String(vpcName+"_infra")) - groupNames = append(groupNames, aws.String(vpcName+"_infra_k8s")) - } - if !isMaster && !isInfra { - groupNames = append(groupNames, aws.String(vpcName+"_compute")) - groupNames = append(groupNames, aws.String(vpcName+"_compute_k8s")) - } + groupNames = append(groupNames, aws.String(vpcName+"_worker_sg")) return &ec2.DescribeSecurityGroupsInput{ Filters: []*ec2.Filter{ diff --git a/utils.go b/utils.go index 610f0949..594e29c4 100644 --- a/utils.go +++ b/utils.go @@ -73,43 +73,43 @@ func chooseNewest(instance1, instance2 *ec2.Instance) *ec2.Instance { // GetInstance returns the AWS instance for a given machine. If multiple instances match our machine, // the most recently launched will be returned. If no instance exists, an error will be returned. -func GetInstance(machine *clusterv1.Machine, client Client) (*ec2.Instance, error) { - instances, err := GetRunningInstances(machine, client) - if err != nil { - return nil, err - } - if len(instances) == 0 { - return nil, fmt.Errorf("no instance found for machine: %s", machine.Name) - } - - instance, _ := SortInstances(instances) - return instance, nil -} +//func GetInstance(machine *clusterv1.Machine, client Client) (*ec2.Instance, error) { +// instances, err := GetRunningInstances(machine, client) +// if err != nil { +// return nil, err +// } +// if len(instances) == 0 { +// return nil, fmt.Errorf("no instance found for machine: %s", machine.Name) +// } +// +// instance, _ := SortInstances(instances) +// return instance, nil +//} // GetRunningInstances returns all running instances that have a tag matching our machine name, // and cluster ID. -func GetRunningInstances(machine *clusterv1.Machine, client Client) ([]*ec2.Instance, error) { +func GetRunningInstances(machine *clusterv1.Machine, client Client, clusterId string) ([]*ec2.Instance, error) { runningInstanceStateFilter := []*string{aws.String(ec2.InstanceStateNameRunning), aws.String(ec2.InstanceStateNamePending)} - return GetInstances(machine, client, runningInstanceStateFilter) + return GetInstances(machine, client, runningInstanceStateFilter, clusterId) } // GetStoppedInstances returns all stopped instances that have a tag matching our machine name, // and cluster ID. -func GetStoppedInstances(machine *clusterv1.Machine, client Client) ([]*ec2.Instance, error) { - stoppedInstanceStateFilter := []*string{aws.String(ec2.InstanceStateNameStopped), aws.String(ec2.InstanceStateNameStopping)} - return GetInstances(machine, client, stoppedInstanceStateFilter) -} +//func GetStoppedInstances(machine *clusterv1.Machine, client Client) ([]*ec2.Instance, error) { +// stoppedInstanceStateFilter := []*string{aws.String(ec2.InstanceStateNameStopped), aws.String(ec2.InstanceStateNameStopping)} +// return GetInstances(machine, client, stoppedInstanceStateFilter) +//} // GetInstances returns all instances that have a tag matching our machine name, // and cluster ID. -func GetInstances(machine *clusterv1.Machine, client Client, instanceStateFilter []*string) ([]*ec2.Instance, error) { +func GetInstances(machine *clusterv1.Machine, client Client, instanceStateFilter []*string, clusterId string) ([]*ec2.Instance, error) { machineName := machine.Name - clusterID, err := getClusterID(machine) - if err != nil { - return []*ec2.Instance{}, fmt.Errorf("unable to get cluster ID for machine %q: %v", machine.Name, err) - } + clusterID := clusterId + //if err != nil { + // return []*ec2.Instance{}, fmt.Errorf("unable to get cluster ID for machine %q: %v", machine.Name, err) + //} requestFilters := []*ec2.Filter{ {