WIP: CAPI AWS implementation

pkg/asset/cluster/cluster.go

@@ -5,30 +5,22 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/util/wait"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
-	utilkubeconfig "sigs.k8s.io/cluster-api/util/kubeconfig"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/cluster/aws"
 	"github.com/openshift/installer/pkg/asset/cluster/azure"
 	"github.com/openshift/installer/pkg/asset/cluster/openstack"
+	"github.com/openshift/installer/pkg/asset/cluster/tfvars"
+	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
+	"github.com/openshift/installer/pkg/asset/ignition/machine"
 	"github.com/openshift/installer/pkg/asset/installconfig"
-	awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	"github.com/openshift/installer/pkg/asset/kubeconfig"
-	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	capimanifests "github.com/openshift/installer/pkg/asset/manifests/clusterapi"
 	"github.com/openshift/installer/pkg/asset/password"
 	"github.com/openshift/installer/pkg/asset/quota"
-	"github.com/openshift/installer/pkg/clusterapi"
 	infra "github.com/openshift/installer/pkg/infrastructure/platform"
 	typesaws "github.com/openshift/installer/pkg/types/aws"
 	typesazure "github.com/openshift/installer/pkg/types/azure"
@@ -67,10 +59,12 @@ func (c *Cluster) Dependencies() []asset.Asset {
 		&installconfig.PlatformPermsCheck{},
 		&installconfig.PlatformProvisionCheck{},
 		&quota.PlatformQuotaCheck{},
-		&TerraformVariables{},
+		&tfvars.TerraformVariables{},
 		&password.KubeadminPassword{},
 		&capimanifests.Cluster{},
 		&kubeconfig.AdminClient{},
+		&bootstrap.Bootstrap{},
+		&machine.Master{},
 	}
 }
 
@@ -82,7 +76,7 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 
 	clusterID := &installconfig.ClusterID{}
 	installConfig := &installconfig.InstallConfig{}
-	terraformVariables := &TerraformVariables{}
+	terraformVariables := &tfvars.TerraformVariables{}
 	parents.Get(clusterID, installConfig, terraformVariables)
 
 	if fs := installConfig.Config.FeatureSet; strings.HasSuffix(string(fs), "NoUpgrade") {
@@ -97,31 +91,14 @@ func (c *Cluster) Generate(parents asset.Parents) (err error) {
 		return errors.New("cluster cannot be created with bootstrapInPlace set")
 	}
 
-	// Check if we're using Cluster API.
-	if capiutils.IsEnabled(installConfig) {
-		// TODO(vincepri): The context should be passed down from the caller,
-		// although today the Asset interface doesn't allow it, refactor once it does.
-		ctx, cancel := context.WithCancel(signals.SetupSignalHandler())
-		go func() {
-			<-ctx.Done()
-			cancel()
-			clusterapi.System().Teardown()
-		}()
-
-		return c.provisionWithClusterAPI(ctx, parents, installConfig, clusterID)
-	}
-
-	// Otherwise, use the normal path.
-	return c.provision(installConfig, clusterID, terraformVariables)
-}
-
-func (c *Cluster) provision(installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID, terraformVariables *TerraformVariables) error {
 	platform := installConfig.Config.Platform.Name()
 
 	if azure := installConfig.Config.Platform.Azure; azure != nil && azure.CloudName == typesazure.StackCloud {
 		platform = typesazure.StackTerraformName
 	}
 
+	// TODO(padillon): determine whether CAPI handles tagging shared subnets, in which case we should be able
+	// to encapsulate these into the terraform package.
 	logrus.Infof("Creating infrastructure resources...")
 	switch platform {
 	case typesaws.Name:
@@ -138,16 +115,11 @@ func (c *Cluster) provision(installConfig *installconfig.InstallConfig, clusterI
 		}
 	}
 
-	tfvarsFiles := []*asset.File{}
-	for _, file := range terraformVariables.Files() {
-		tfvarsFiles = append(tfvarsFiles, file)
-	}
-
 	provider, err := infra.ProviderForPlatform(platform, installConfig.Config.EnabledFeatureGates())
 	if err != nil {
 		return fmt.Errorf("error getting infrastructure provider: %w", err)
 	}
-	files, err := provider.Provision(InstallDir, tfvarsFiles)
+	files, err := provider.Provision(InstallDir, parents)
 	if files != nil {
 		c.FileList = append(c.FileList, files...) // append state files even in case of failure
 	}
@@ -158,137 +130,6 @@ func (c *Cluster) provision(installConfig *installconfig.InstallConfig, clusterI
 	return nil
 }
 
-func (c *Cluster) provisionWithClusterAPI(ctx context.Context, parents asset.Parents, installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID) error {
-	capiManifests := &capimanifests.Cluster{}
-	clusterKubeconfigAsset := &kubeconfig.AdminClient{}
-	parents.Get(
-		capiManifests,
-		clusterKubeconfigAsset,
-	)
-
-	// Only need the objects--not the files.
-	manifests := []client.Object{}
-	for _, m := range capiManifests.RuntimeFiles() {
-		manifests = append(manifests, m.Object)
-	}
-
-	// Run the CAPI system.
-	capiSystem := clusterapi.System()
-	if err := capiSystem.Run(ctx, installConfig); err != nil {
-		return fmt.Errorf("failed to run cluster api system: %w", err)
-	}
-
-	// Grab the client.
-	cl := capiSystem.Client()
-
-	// Create all the manifests and store them.
-	for _, m := range manifests {
-		m.SetNamespace(capiutils.Namespace)
-		if err := cl.Create(context.Background(), m); err != nil {
-			return fmt.Errorf("failed to create manifest: %w", err)
-		}
-		logrus.Infof("Created manifest %+T, namespace=%s name=%s", m, m.GetNamespace(), m.GetName())
-	}
-
-	// Pass cluster kubeconfig and store it in; this is usually the role of a bootstrap provider.
-	{
-		key := client.ObjectKey{
-			Name:      clusterID.InfraID,
-			Namespace: capiutils.Namespace,
-		}
-		cluster := &clusterv1.Cluster{}
-		if err := cl.Get(context.Background(), key, cluster); err != nil {
-			return err
-		}
-		// Create the secret.
-		clusterKubeconfig := clusterKubeconfigAsset.Files()[0].Data
-		secret := utilkubeconfig.GenerateSecret(cluster, clusterKubeconfig)
-		if err := cl.Create(context.Background(), secret); err != nil {
-			return err
-		}
-	}
-
-	// Wait for the load balancer to be ready by checking the control plane endpoint
-	// on the cluster object.
-	var cluster *clusterv1.Cluster
-	{
-		if err := wait.ExponentialBackoff(wait.Backoff{
-			Duration: time.Second * 10,
-			Factor:   float64(1.5),
-			Steps:    32,
-		}, func() (bool, error) {
-			c := &clusterv1.Cluster{}
-			if err := cl.Get(context.Background(), client.ObjectKey{
-				Name:      clusterID.InfraID,
-				Namespace: capiutils.Namespace,
-			}, c); err != nil {
-				if apierrors.IsNotFound(err) {
-					return false, nil
-				}
-				return false, err
-			}
-			cluster = c
-			return cluster.Spec.ControlPlaneEndpoint.IsValid(), nil
-		}); err != nil {
-			return err
-		}
-		if cluster == nil {
-			return errors.New("error occurred during load balancer ready check")
-		}
-		if cluster.Spec.ControlPlaneEndpoint.Host == "" {
-			return errors.New("control plane endpoint is not set")
-		}
-	}
-
-	// Run the post-provisioning steps for the platform we're on.
-	// TODO(vincepri): The following should probably be in a separate package with a clear
-	// interface and multiple hooks at different stages of the cluster lifecycle.
-	switch installConfig.Config.Platform.Name() {
-	case typesaws.Name:
-		ssn, err := installConfig.AWS.Session(context.TODO())
-		if err != nil {
-			return fmt.Errorf("failed to create session: %w", err)
-		}
-		client := awsconfig.NewClient(ssn)
-		r53cfg := awsconfig.GetR53ClientCfg(ssn, "")
-		err = client.CreateOrUpdateRecord(installConfig.Config, cluster.Spec.ControlPlaneEndpoint.Host, r53cfg)
-		if err != nil {
-			return fmt.Errorf("failed to create route53 records: %w", err)
-		}
-		logrus.Infof("Created Route53 records to control plane load balancer.")
-	default:
-	}
-
-	// For each manifest we created, retrieve it and store it in the asset.
-	for _, m := range manifests {
-		key := client.ObjectKey{
-			Name:      m.GetName(),
-			Namespace: m.GetNamespace(),
-		}
-		if err := cl.Get(context.Background(), key, m); err != nil {
-			return fmt.Errorf("failed to get manifest: %w", err)
-		}
-
-		gvk, err := cl.GroupVersionKindFor(m)
-		if err != nil {
-			return fmt.Errorf("failed to get GVK for manifest: %w", err)
-		}
-		fileName := fmt.Sprintf("%s-%s-%s.yaml", gvk.Kind, m.GetNamespace(), m.GetName())
-		objData, err := yaml.Marshal(m)
-		if err != nil {
-			errMsg := fmt.Sprintf("failed to create infrastructure manifest %s from InstallConfig", fileName)
-			return errors.Wrapf(err, errMsg)
-		}
-		c.FileList = append(c.FileList, &asset.File{
-			Filename: fileName,
-			Data:     objData,
-		})
-	}
-
-	logrus.Infof("Cluster API resources have been created. Waiting for cluster to become ready...")
-	return nil
-}
-
 // Files returns the FileList generated by the asset.
 func (c *Cluster) Files() []*asset.File {
 	return c.FileList

pkg/asset/cluster/metadata.go

@@ -2,8 +2,6 @@ package cluster
 
 import (
 	"encoding/json"
-	"os"
-	"path/filepath"
 
 	"github.com/pkg/errors"
 
@@ -16,6 +14,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/cluster/gcp"
 	"github.com/openshift/installer/pkg/asset/cluster/ibmcloud"
 	"github.com/openshift/installer/pkg/asset/cluster/libvirt"
+	clustermetadata "github.com/openshift/installer/pkg/asset/cluster/metadata"
 	"github.com/openshift/installer/pkg/asset/cluster/nutanix"
 	"github.com/openshift/installer/pkg/asset/cluster/openstack"
 	"github.com/openshift/installer/pkg/asset/cluster/ovirt"
@@ -41,10 +40,6 @@ import (
 	vspheretypes "github.com/openshift/installer/pkg/types/vsphere"
 )
 
-const (
-	metadataFileName = "metadata.json"
-)
-
 // Metadata contains information needed to destroy clusters.
 type Metadata struct {
 	File *asset.File
@@ -123,7 +118,7 @@ func (m *Metadata) Generate(parents asset.Parents) (err error) {
 	}
 
 	m.File = &asset.File{
-		Filename: metadataFileName,
+		Filename: clustermetadata.FileName,
 		Data:     data,
 	}
 
@@ -143,19 +138,3 @@ func (m *Metadata) Files() []*asset.File {
 func (m *Metadata) Load(f asset.FileFetcher) (found bool, err error) {
 	return false, nil
 }
-
-// LoadMetadata loads the cluster metadata from an asset directory.
-func LoadMetadata(dir string) (*types.ClusterMetadata, error) {
-	path := filepath.Join(dir, metadataFileName)
-	raw, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	var metadata *types.ClusterMetadata
-	if err = json.Unmarshal(raw, &metadata); err != nil {
-		return nil, errors.Wrapf(err, "failed to Unmarshal data from %q to types.ClusterMetadata", path)
-	}
-
-	return metadata, err
-}

pkg/asset/cluster/metadata/metadata.go

@@ -0,0 +1,30 @@
+package metadata
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/openshift/installer/pkg/types"
+	"github.com/pkg/errors"
+)
+
+const (
+	FileName = "metadata.json"
+)
+
+// LoadMetadata loads the cluster metadata from an asset directory.
+func Load(dir string) (*types.ClusterMetadata, error) {
+	path := filepath.Join(dir, FileName)
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var metadata *types.ClusterMetadata
+	if err = json.Unmarshal(raw, &metadata); err != nil {
+		return nil, errors.Wrapf(err, "failed to Unmarshal data from %q to types.ClusterMetadata", path)
+	}
+
+	return metadata, err
+}

pkg/asset/cluster/tfvars.go → pkg/asset/cluster/tfvars/tfvars.go

@@ -1,4 +1,4 @@
-package cluster
+package tfvars
 
 import (
 	"context"