Swiftless OpenStack: store bootstrap ignition configs in Glance

Gopkg.toml

@@ -27,7 +27,7 @@ required = [
 
 [[constraint]]
   name = "github.com/coreos/ignition"
-  version = "0.26.0"
+  version = "0.35.0"
 
 [[constraint]]
   name = "github.com/libvirt/libvirt-go"

pkg/destroy/bootstrap/bootstrap.go

@@ -74,9 +74,10 @@ func Destroy(dir string) (err error) {
 			return errors.Wrap(err, "Terraform apply")
 		}
 	case openstack.Name:
-		err = osp.DeleteSwiftContainer(metadata.InfraID, metadata.OpenStack.Cloud)
+		imageName := metadata.InfraID + "-ignition"
+		err = osp.DeleteGlanceImage(imageName, metadata.OpenStack.Cloud)
 		if err != nil {
-			return errors.Wrapf(err, "Failed to delete swift container %s", metadata.InfraID)
+			return errors.Wrapf(err, "Failed to delete glance image %s", imageName)
 		}
 	}
 

pkg/destroy/openstack/glance.go

@@ -0,0 +1,40 @@
+package openstack
+
+import (
+	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
+	"github.com/gophercloud/utils/openstack/clientconfig"
+)
+
+// DeleteGlanceImage deletes the image with the specified name
+func DeleteGlanceImage(name string, cloud string) error {
+	opts := clientconfig.ClientOpts{
+		Cloud: cloud,
+	}
+
+	conn, err := clientconfig.NewServiceClient("image", &opts)
+	if err != nil {
+		return err
+	}
+
+	listOpts := images.ListOpts{
+		Name: name,
+	}
+
+	allPages, err := images.List(conn, listOpts).AllPages()
+	if err != nil {
+		return err
+	}
+
+	allImages, err := images.ExtractImages(allPages)
+	if err != nil {
+		return err
+	}
+
+	for _, image := range allImages {
+		err := images.Delete(conn, image.ID).ExtractErr()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/destroy/openstack/openstack.go

@@ -544,6 +544,10 @@ func deleteContainers(opts *clientconfig.ClientOpts, filter Filter, logger logru
 
 	conn, err := clientconfig.NewServiceClient("object-store", opts)
 	if err != nil {
+		if _, ok := err.(*gophercloud.ErrEndpointNotFound); ok {
+			logger.Debug("Swift endpoint is not found")
+			return true, nil
+		}
 		logger.Errorf("%v", err)
 		return false, nil
 	}

pkg/tfvars/openstack/bootstrap_ignition.go

@@ -6,59 +6,51 @@ import (
 	"fmt"
 	"strings"
 
-	ignition "github.com/coreos/ignition/config/v2_2/types"
-	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers"
-	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects"
+	ignition "github.com/coreos/ignition/config/v2_4/types"
+	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/imagedata"
+	"github.com/gophercloud/gophercloud/openstack/imageservice/v2/images"
 	"github.com/gophercloud/utils/openstack/clientconfig"
 	"github.com/sirupsen/logrus"
 	"github.com/vincent-petithory/dataurl"
-	"k8s.io/apimachinery/pkg/util/rand"
 )
 
-// createBootstrapSwiftObject creates a container and object in swift with the bootstrap ignition config.
-func createBootstrapSwiftObject(cloud string, bootstrapIgn string, clusterID string) (string, error) {
-	logrus.Debugln("Creating a Swift container for your bootstrap ignition...")
+// Starting from OpenShift 4.4 we store bootstrap Ignition configs in Glance.
+
+// uploadBootstrapConfig uploads the bootstrap Ignition config in Glance and returns its location
+func uploadBootstrapConfig(cloud string, bootstrapIgn string, clusterID string) (string, error) {
+	logrus.Debugln("Creating a Glance image for your bootstrap ignition config...")
 	opts := clientconfig.ClientOpts{
 		Cloud: cloud,
 	}
 
-	conn, err := clientconfig.NewServiceClient("object-store", &opts)
+	conn, err := clientconfig.NewServiceClient("image", &opts)
 	if err != nil {
 		return "", err
 	}
 
-	containerCreateOpts := containers.CreateOpts{
-		ContainerRead: ".r:*",
-
-		// "kubernetes.io/cluster/${var.cluster_id}" = "owned"
-		Metadata: map[string]string{
-			"Name":               fmt.Sprintf("%s-ignition", clusterID),
-			"openshiftClusterID": clusterID,
-		},
+	imageCreateOpts := images.CreateOpts{
+		Name:            fmt.Sprintf("%s-ignition", clusterID),
+		ContainerFormat: "bare",
+		DiskFormat:      "raw",
+		Tags:            []string{fmt.Sprintf("openshiftClusterID=%s", clusterID)},
+		// TODO(mfedosin): add Description when gophercloud supports it.
 	}
 
-	_, err = containers.Create(conn, clusterID, containerCreateOpts).Extract()
+	img, err := images.Create(conn, imageCreateOpts).Extract()
 	if err != nil {
 		return "", err
 	}
-	logrus.Debugf("Container %s was created.", clusterID)
-
-	logrus.Debugf("Creating a Swift object in container %s containing your bootstrap ignition...", clusterID)
-	objectCreateOpts := objects.CreateOpts{
-		ContentType: "text/plain",
-		Content:     strings.NewReader(bootstrapIgn),
-		DeleteAfter: 3600,
-	}
-
-	objID := rand.String(16)
+	logrus.Debugf("Image %s was created.", img.Name)
 
-	_, err = objects.Create(conn, clusterID, objID, objectCreateOpts).Extract()
-	if err != nil {
+	logrus.Debugf("Uploading bootstrap config to the image %v with ID %v", img.Name, img.ID)
+	res := imagedata.Upload(conn, img.ID, strings.NewReader(bootstrapIgn))
+	if res.Err != nil {
 		return "", err
 	}
-	logrus.Debugf("The object was created.")
+	logrus.Debugf("The config was uploaded.")
 
-	return objID, nil
+	// img.File contains location of the uploaded data
+	return img.File, nil
 }
 
 // To allow Ignition to download its config on the bootstrap machine from a location secured by a
@@ -70,7 +62,7 @@ func createBootstrapSwiftObject(cloud string, bootstrapIgn string, clusterID str
 
 // generateIgnitionShim is used to generate an ignition file that contains a user ca bundle
 // in its Security section.
-func generateIgnitionShim(userCA string, clusterID string, swiftObject string) (string, error) {
+func generateIgnitionShim(userCA string, clusterID string, bootstrapConfigURL string, tokenID string) (string, error) {
 	fileMode := 420
 
 	// Hostname Config
@@ -100,14 +92,22 @@ func generateIgnitionShim(userCA string, clusterID string, swiftObject string) (
 		}
 	}
 
+	headers := []ignition.HTTPHeader{
+		{
+			Name:  "X-Auth-Token",
+			Value: tokenID,
+		},
+	}
+
 	ign := ignition.Config{
 		Ignition: ignition.Ignition{
 			Version:  ignition.MaxVersion.String(),
 			Security: security,
 			Config: ignition.IgnitionConfig{
 				Append: []ignition.ConfigReference{
 					{
-						Source: swiftObject,
+						Source:      bootstrapConfigURL,
+						HTTPHeaders: headers,
 					},
 				},
 			},
@@ -126,3 +126,22 @@ func generateIgnitionShim(userCA string, clusterID string, swiftObject string) (
 
 	return string(data), nil
 }
+
+// getAuthToken fetches valid OpenStack authentication token ID
+func getAuthToken(cloud string) (string, error) {
+	opts := &clientconfig.ClientOpts{
+		Cloud: cloud,
+	}
+
+	conn, err := clientconfig.NewServiceClient("identity", opts)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := conn.GetAuthResult().ExtractTokenID()
+	if err != nil {
+		return "", err
+	}
+
+	return token, nil
+}

pkg/tfvars/openstack/openstack.go

@@ -72,18 +72,23 @@ func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, cloud string, external
 		}
 	}
 
-	swiftPublicURL, err := getSwiftPublicURL(cloud)
+	glancePublicURL, err := getGlancePublicURL(cloud)
 	if err != nil {
 		return nil, err
 	}
 
-	objectID, err := createBootstrapSwiftObject(cloud, bootstrapIgn, infraID)
+	configLocation, err := uploadBootstrapConfig(cloud, bootstrapIgn, infraID)
 	if err != nil {
 		return nil, err
 	}
 
-	objectAddress := fmt.Sprintf("%s/%s/%s", swiftPublicURL, infraID, objectID)
-	userCAIgnition, err := generateIgnitionShim(userCA, infraID, objectAddress)
+	tokenID, err := getAuthToken(cloud)
+	if err != nil {
+		return nil, err
+	}
+
+	bootstrapConfigURL := fmt.Sprintf("%s%s", glancePublicURL, configLocation)
+	userCAIgnition, err := generateIgnitionShim(userCA, infraID, bootstrapConfigURL, tokenID)
 	if err != nil {
 		return nil, err
 	}
@@ -133,7 +138,7 @@ func validateOverriddenImageName(imageName, cloud string) error {
 	return nil
 }
 
-// We need to obtain Swift public endpoint that will be used by Ignition to download bootstrap ignition files.
+// We need to obtain Glance public endpoint that will be used by Ignition to download bootstrap ignition files.
 // By design this should be done by using https://www.terraform.io/docs/providers/openstack/d/identity_endpoint_v3.html
 // but OpenStack default policies forbid to use this API for regular users.
 // On the other hand when a user authenticates in OpenStack (i.e. gets a token), it includes the whole service
@@ -143,33 +148,33 @@ func validateOverriddenImageName(imageName, cloud string) error {
 // We do next:
 // 1. In "getServiceCatalog" we authenticate in OpenStack (tokens.Create(..)),
 //    parse the token and extract the service catalog: (ExtractServiceCatalog())
-// 2. In getSwiftPublicURL we iterate through the catalog and find "public" endpoint for "object-store".
+// 2. In getGlancePublicURL we iterate through the catalog and find "public" endpoint for "image".
 
-// getSwiftPublicURL obtains Swift public endpoint URL
-func getSwiftPublicURL(cloud string) (string, error) {
-	var swiftPublicURL string
+// getGlancePublicURL obtains Glance public endpoint URL
+func getGlancePublicURL(cloud string) (string, error) {
+	var glancePublicURL string
 	serviceCatalog, err := getServiceCatalog(cloud)
 	if err != nil {
 		return "", err
 	}
 
 	for _, svc := range serviceCatalog.Entries {
-		if svc.Type == "object-store" {
+		if svc.Type == "image" {
 			for _, e := range svc.Endpoints {
 				if e.Interface == "public" {
-					swiftPublicURL = e.URL
+					glancePublicURL = e.URL
 					break
 				}
 			}
 			break
 		}
 	}
 
-	if swiftPublicURL == "" {
-		return "", errors.Errorf("cannot retrieve Swift URL from the service catalog")
+	if glancePublicURL == "" {
+		return "", errors.Errorf("cannot retrieve Glance URL from the service catalog")
 	}
 
-	return swiftPublicURL, nil
+	return glancePublicURL, nil
 }
 
 // getServiceCatalog fetches OpenStack service catalog with service endpoints