openstack: get ignition via IP

data/data/openstack/main.tf

@@ -67,6 +67,8 @@ module "masters" {
   master_port_names   = module.topology.master_port_names
   user_data_ign       = var.ignition_master
   service_vm_fixed_ip = module.topology.service_vm_fixed_ip
+  api_int_ip          = var.openstack_api_int_ip
+  node_dns_ip         = var.openstack_node_dns_ip
   master_sg_ids = concat(
     var.openstack_master_extra_sg_ids,
     [module.topology.master_sg_id],
@@ -84,6 +86,8 @@ module "topology" {
   external_network_id = var.openstack_external_network_id
   masters_count       = var.master_count
   lb_floating_ip      = var.openstack_lb_floating_ip
+  api_int_ip          = var.openstack_api_int_ip
+  node_dns_ip         = var.openstack_node_dns_ip
   trunk_support       = var.openstack_trunk_support
   octavia_support     = var.openstack_octavia_support
 }

data/data/openstack/masters/main.tf

@@ -28,6 +28,8 @@ data "ignition_file" "clustervars" {
 
   content {
     content = <<EOF
+export API_VIP=${var.api_int_ip}
+export DNS_VIP=${var.node_dns_ip}
 export FLOATING_IP=${var.lb_floating_ip}
 export BOOTSTRAP_IP=${var.bootstrap_ip}
 ${replace(join("\n", formatlist("export MASTER_FIXED_IPS_%s=%s", var.master_port_names, var.master_ips)), "${var.cluster_id}-master-port-", "")}

data/data/openstack/masters/variables.tf

@@ -51,6 +51,14 @@ variable "user_data_ign" {
   type = string
 }
 
+variable "api_int_ip" {
+  type = string
+}
+
+variable "node_dns_ip" {
+  type = string
+}
+
 variable "service_vm_fixed_ip" {
   type = string
 }

data/data/openstack/topology/private-network.tf

@@ -30,6 +30,14 @@ resource "openstack_networking_subnet_v2" "nodes" {
   network_id      = openstack_networking_network_v2.openshift-private.id
   tags            = ["openshiftClusterID=${var.cluster_id}"]
   dns_nameservers = [openstack_networking_port_v2.service_port.all_fixed_ips[0]]
+
+  # We reserve some space at the beginning of the CIDR to use for the VIPs
+  # It would be good to make this more dynamic by calculating the number of
+  # addresses in the provided CIDR. This currently assumes at least a /18.
+  allocation_pool {
+    start = cidrhost(local.nodes_cidr_block, 10)
+    end   = cidrhost(local.nodes_cidr_block, 16000)
+  }
 }
 
 resource "openstack_networking_port_v2" "masters" {
@@ -44,6 +52,14 @@ resource "openstack_networking_port_v2" "masters" {
   fixed_ip {
     subnet_id = openstack_networking_subnet_v2.nodes.id
   }
+
+  allowed_address_pairs {
+    ip_address = var.api_int_ip
+  }
+
+  allowed_address_pairs {
+    ip_address = var.node_dns_ip
+  }
 }
 
 resource "openstack_networking_trunk_v2" "masters" {

data/data/openstack/topology/variables.tf

@@ -28,6 +28,14 @@ variable "masters_count" {
   type = string
 }
 
+variable "api_int_ip" {
+  type = string
+}
+
+variable "node_dns_ip" {
+  type = string
+}
+
 variable "trunk_support" {
   type = string
 }

data/data/openstack/variables-openstack.tf

@@ -269,6 +269,16 @@ EOF
 
 }
 
+variable "openstack_api_int_ip" {
+  type        = string
+  description = "IP on the node subnet reserved for api-int VIP."
+}
+
+variable "openstack_node_dns_ip" {
+  type        = string
+  description = "IP on the nodes subnet reserved for node dns VIP."
+}
+
 variable "openstack_master_flavor_name" {
   type        = string
   description = "Instance size for the master node(s). Example: `m1.medium`."

pkg/asset/cluster/tfvars.go

@@ -248,6 +248,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			installConfig.Config.Platform.OpenStack.Region,
 			installConfig.Config.Platform.OpenStack.ExternalNetwork,
 			installConfig.Config.Platform.OpenStack.LbFloatingIP,
+			installConfig.Config.Platform.OpenStack.APIVIP,
+			installConfig.Config.Platform.OpenStack.DNSVIP,
 			installConfig.Config.Platform.OpenStack.TrunkSupport,
 			installConfig.Config.Platform.OpenStack.OctaviaSupport,
 		)

pkg/asset/ignition/machine/node.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/openshift/installer/pkg/types"
 	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 )
 
 // pointerIgnitionConfig generates a config which references the remote config
@@ -20,6 +21,11 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro
 		// Baremetal needs to point directly at the VIP because we don't have a
 		// way to configure DNS before Ignition runs.
 		ignitionHost = fmt.Sprintf("%s:22623", installConfig.BareMetal.APIVIP)
+	case openstacktypes.Name:
+		// We can't actually set this to the VIP until we have keepalived patches
+		// merged to machine-config-operator
+		// ignitionHost = fmt.Sprintf("%s:22623", installConfig.Config.OpenStack.APIVIP)
+		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 	default:
 		ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain())
 	}

pkg/asset/tls/mcscertkey.go

@@ -8,6 +8,7 @@ import (
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
+	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 )
 
 // MCSCertKey is the asset that generates the MCS key/cert pair.
@@ -45,6 +46,9 @@ func (a *MCSCertKey) Generate(dependencies asset.Parents) error {
 	case baremetaltypes.Name:
 		cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.BareMetal.APIVIP)}
 		cfg.DNSNames = []string{hostname, installConfig.Config.BareMetal.APIVIP}
+	case openstacktypes.Name:
+		cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.OpenStack.APIVIP)}
+		cfg.DNSNames = []string{hostname, installConfig.Config.OpenStack.APIVIP}
 	default:
 		cfg.DNSNames = []string{hostname}
 	}

pkg/tfvars/openstack/openstack.go

@@ -14,19 +14,23 @@ type config struct {
 	Cloud           string `json:"openstack_credentials_cloud,omitempty"`
 	FlavorName      string `json:"openstack_master_flavor_name,omitempty"`
 	LbFloatingIP    string `json:"openstack_lb_floating_ip,omitempty"`
+	APIVIP          string `json:"openstack_api_int_ip,omitempty"`
+	DNSVIP          string `json:"openstack_node_dns_ip,omitempty"`
 	TrunkSupport    string `json:"openstack_trunk_support,omitempty"`
 	OctaviaSupport  string `json:"openstack_octavia_support,omitempty"`
 }
 
 // TFVars generates OpenStack-specific Terraform variables.
-func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externalNetwork string, lbFloatingIP string, trunkSupport string, octaviaSupport string) ([]byte, error) {
+func TFVars(masterConfig *v1alpha1.OpenstackProviderSpec, region string, externalNetwork string, lbFloatingIP string, apiVIP string, dnsVIP string, trunkSupport string, octaviaSupport string) ([]byte, error) {
 	cfg := &config{
 		Region:          region,
 		BaseImage:       masterConfig.Image,
 		ExternalNetwork: externalNetwork,
 		Cloud:           masterConfig.CloudName,
 		FlavorName:      masterConfig.Flavor,
 		LbFloatingIP:    lbFloatingIP,
+		APIVIP:          apiVIP,
+		DNSVIP:          dnsVIP,
 		TrunkSupport:    trunkSupport,
 		OctaviaSupport:  octaviaSupport,
 	}

pkg/types/defaults/installconfig.go

@@ -67,7 +67,7 @@ func SetInstallConfigDefaults(c *types.InstallConfig) {
 	case c.Platform.Libvirt != nil:
 		libvirtdefaults.SetPlatformDefaults(c.Platform.Libvirt)
 	case c.Platform.OpenStack != nil:
-		openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack)
+		openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack, c)
 	case c.Platform.VSphere != nil:
 		vspheredefaults.SetPlatformDefaults(c.Platform.VSphere, c)
 	case c.Platform.BareMetal != nil:

pkg/types/defaults/installconfig_test.go

@@ -65,7 +65,7 @@ func defaultLibvirtInstallConfig() *types.InstallConfig {
 func defaultOpenStackInstallConfig() *types.InstallConfig {
 	c := defaultInstallConfig()
 	c.Platform.OpenStack = &openstack.Platform{}
-	openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack)
+	openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack, c)
 	return c
 }
 

pkg/types/openstack/defaults/platform.go

@@ -1,9 +1,37 @@
 package defaults
 
 import (
+	"fmt"
+
+	"github.com/apparentlymart/go-cidr/cidr"
+
+	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/openstack"
 )
 
+// Defaults for the openstack platform.
+const (
+	APIVIP = ""
+	DNSVIP = ""
+)
+
 // SetPlatformDefaults sets the defaults for the platform.
-func SetPlatformDefaults(p *openstack.Platform) {
+func SetPlatformDefaults(p *openstack.Platform, c *types.InstallConfig) {
+	if p.APIVIP == "" {
+		vip, err := cidr.Host(&c.Networking.MachineCIDR.IPNet, 5)
+		if err != nil {
+			p.APIVIP = fmt.Sprintf("failed to get APIVIP from MachineCIDR: %s", err.Error())
+		} else {
+			p.APIVIP = vip.String()
+		}
+	}
+
+	if p.DNSVIP == "" {
+		vip, err := cidr.Host(&c.Networking.MachineCIDR.IPNet, 6)
+		if err != nil {
+			p.DNSVIP = fmt.Sprintf("failed to get DNSVIP from MachineCIDR: %s", err.Error())
+		} else {
+			p.DNSVIP = vip.String()
+		}
+	}
 }

pkg/types/openstack/platform.go

@@ -28,6 +28,14 @@ type Platform struct {
 	// Existing Floating IP to associate with the OpenStack load balancer.
 	LbFloatingIP string `json:"lbFloatingIP"`
 
+	// APIVIP
+	// IP in the machineCIDR to use for api-int.
+	APIVIP string `json:"apiVIP"`
+
+	// DNSVIP
+	// IP in the machineCIDR to use for simulated route53.
+	DNSVIP string `json:"dnsVIP"`
+
 	// TrunkSupport
 	// Whether OpenStack ports can be trunked
 	TrunkSupport string `json:"trunkSupport"`