baremetal: make provisioning network optional

data/data/baremetal/bootstrap/main.tf

@@ -37,12 +37,11 @@ resource "libvirt_domain" "bootstrap" {
     mode = "host-passthrough"
   }
 
-  network_interface {
-    bridge = var.external_bridge
-  }
-
-  network_interface {
-    bridge = var.provisioning_bridge
+  dynamic "network_interface" {
+    for_each = var.bridges
+    content {
+      bridge = network_interface.value
+    }
   }
 }
 

data/data/baremetal/bootstrap/variables.tf

@@ -13,12 +13,7 @@ variable "ignition" {
   description = "The content of the bootstrap ignition file."
 }
 
-variable "external_bridge" {
-  type        = string
-  description = "The name of the bridge providing external access"
-}
-
-variable "provisioning_bridge" {
-  type        = string
-  description = "The name of the bridge used for provisioning"
+variable "bridges" {
+  type        = list(string)
+  description = "A list of network interfaces to attach to the bootstrap virtual machine"
 }

data/data/baremetal/main.tf

@@ -12,11 +12,10 @@ provider "ironic" {
 module "bootstrap" {
   source = "./bootstrap"
 
-  cluster_id          = var.cluster_id
-  image               = var.bootstrap_os_image
-  ignition            = var.ignition_bootstrap
-  external_bridge     = var.external_bridge
-  provisioning_bridge = var.provisioning_bridge
+  cluster_id = var.cluster_id
+  image      = var.bootstrap_os_image
+  ignition   = var.ignition_bootstrap
+  bridges    = compact([var.external_bridge, var.provisioning_bridge])
 }
 
 module "masters" {

data/data/bootstrap/baremetal/files/usr/local/bin/startironic.sh.template

@@ -23,10 +23,16 @@ for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb i
 done
 
 # Start the provisioning nic if not already started
-PROVISIONING_NIC=ens4
-if ! nmcli -t device | grep "$PROVISIONING_NIC:ethernet:connected:provisioning"; then
+PROVISIONING_NIC={{.PlatformData.BareMetal.ProvisioningInterface}}
+
+if ! nmcli -t device | grep "$PROVISIONING_NIC:ethernet:connected"; then
     nmcli c add type ethernet ifname $PROVISIONING_NIC con-name provisioning {{ if .PlatformData.BareMetal.ProvisioningIPv6 }} ip6 {{ else }} ip4 {{ end }} {{.PlatformData.BareMetal.ProvisioningIP}}/{{.PlatformData.BareMetal.ProvisioningCIDR}}
     nmcli c up provisioning
+else
+  connection=$(nmcli -t device show $PROVISIONING_NIC | grep GENERAL.CONNECTION | cut -d: -f2)
+  nmcli con modify "$connection" ifname $PROVISIONING_NIC {{ if .PlatformData.BareMetal.ProvisioningIPv6 }} ip6 {{ else }} ip4 {{ end }} {{.PlatformData.BareMetal.ProvisioningIP}}/{{.PlatformData.BareMetal.ProvisioningCIDR}}
+  nmcli con reload "$connection"
+  nmcli con up "$connection"
 fi
 
 # Wait for the interface to come up
@@ -53,14 +59,19 @@ done
 # Start dnsmasq, http, mariadb, and ironic containers using same image
 # Currently we do this outside of a pod because we need to ensure the images
 # are downloaded before starting the API pods
-podman run -d --net host --privileged --name mariadb \
-     -v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/runmariadb \
-     --env MARIADB_PASSWORD=$mariadb_password ${IRONIC_IMAGE}
-
-podman run -d --net host --privileged --name dnsmasq \
+{{ if .PlatformData.BareMetal.ProvisioningDNSMasq }}
+dnsmasq_container_name="dnsmasq"
+podman run -d --net host --privileged --name $dnsmasq_container_name \
      --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
      --env DHCP_RANGE=$DHCP_RANGE \
      -v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/rundnsmasq ${IRONIC_IMAGE}
+{{ else }}
+dnsmasq_container_name=""
+{{ end }}
+
+podman run -d --net host --privileged --name mariadb \
+     -v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/runmariadb \
+     --env MARIADB_PASSWORD=$mariadb_password ${IRONIC_IMAGE}
 
 podman run -d --net host --privileged --name httpd \
      --env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
@@ -146,7 +157,7 @@ sudo podman run -d --net host --privileged --name ironic-api \
 # Now loop so the service remains active and restart everything should one of the containers exit unexpectedly.
 # The alternative would be RemainAfterExit=yes but then we lose the ability to restart if something crashes.
 while true; do
-    for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb; do
+    for name in ironic-api ironic-conductor ironic-inspector $dnsmasq_container_name httpd mariadb; do
         # Note there are two levels of go templating here, the outer braces
         # are for the templating done in openshift-install, to escape the
         # templating input to the --format option of podman inspect

data/data/install.openshift.io_installconfigs.yaml

@@ -1119,11 +1119,9 @@ spec:
                       on the host that will run the bootstrap VM.
                     type: string
                   provisioningDHCPExternal:
-                    description: ProvisioningDHCPExternal indicates that DHCP is provided
-                      by an external service, appropriately configured with next-server
-                      set to BootstrapProvisioningIP for the control plane, and ClusterProvisioningIP
-                      for workers. The default for this field is false, which means
-                      we will start and manage a DHCP server on the provisioning network.
+                    description: DeprecatedProvisioningDHCPExternal indicates that
+                      DHCP is provided by an external service. This parameter is replaced
+                      by ProvisioningNetwork being set to "Unmanaged".
                     type: boolean
                   provisioningDHCPRange:
                     description: ProvisioningDHCPRange is used to provide DHCP services
@@ -1135,6 +1133,16 @@ spec:
                       services, and an http server to cache some downloaded content
                       e.g RHCOS/IPA images
                     type: string
+                  provisioningNetwork:
+                    default: Managed
+                    description: ProvisioningNetwork is used to indicate if we will
+                      have a provisioning network, and how it will be managed.
+                    enum:
+                    - ""
+                    - Managed
+                    - Unmanaged
+                    - Disabled
+                    type: string
                   provisioningNetworkCIDR:
                     description: ProvisioningNetworkCIDR defines the network to use
                       for provisioning.

data/data/manifests/openshift/baremetal-provisioning-config.yaml.template

@@ -6,6 +6,6 @@ spec:
   provisioningInterface: "{{.Baremetal.ProvisioningNetworkInterface}}"
   provisioningIP: "{{.Baremetal.ClusterProvisioningIP}}"
   provisioningNetworkCIDR: "{{.Baremetal.ProvisioningNetworkCIDR}}"
-  provisioningDHCPExternal: {{.Baremetal.ProvisioningDHCPExternal}}
+  provisioningNetwork: "{{.Baremetal.ProvisioningNetwork}}"
   provisioningDHCPRange: "{{.Baremetal.ProvisioningDHCPRange}}"
   provisioningOSDownloadURL: "{{.ProvisioningOSDownloadURL}}"

docs/user/metal/install_ipi.md

@@ -17,8 +17,10 @@ deployments, see [install_upi.md](install_upi.md).
 
 ### Network Requirements
 
-It is assumed that all hosts have at least 2 NICs, used for the following
-purposes:
+You have the choice of a single or dual NIC setup, depending on whether
+you would like to use PXE/DHCP-based provisioning or not. Please note
+that disabling the provisioning network means that host BMC's must be
+accessible over the external network which may not be desirable.
 
 * **NIC #1 - External Network**
   * This network is the main network used by the cluster, including API traffic
@@ -46,12 +48,15 @@ purposes:
     * `api.<cluster-name>.<base-domain>` - pointing to the API VIP
     * `*.apps.<cluster-name>.<base-domain>` - pointing to the Ingress VIP
 
-* **NIC #2 - Provisioning Network**
+* **NIC #2 - Provisioning Network (optional) **
   * A private network used for PXE based provisioning.
   * You must specify `provisioningNetworkInterface` to indicate which
-    interface is connected to this network.
-  * DHCP is automated for this network by default, to rely on external
-    DHCP, set the platform's `provisioningDHCPExternal` option to `true`
+    interface is connected to this network on the control plane nodes.
+  * The provisioning network may be "Managed" (default), "Unmanaged," or
+    "Disabled."
+  * In managed mode, DHCP and TFTP are configured to run in the cluster. In
+    unmanaged mode, TFTP is still available but you must configure DHCP
+    externally.
   * Addressing for this network defaults to `172.22.0.0/24`, but is
     configurable by setting the `provisioningNetworkCIDR` option.
   * Two IP's are required to be available for use, one for the bootstrap
@@ -278,6 +283,9 @@ both required.  For example
 To use virtual media instead of PXE for attaching the provisioning
 image to the host, use `redfish-virtualmedia://` or `idrac-virtualmedia://`
 
+Please note that when the provisioning network is disabled, the only
+supported BMC's are virtual media.
+
 ## Work in Progress
 
 Integration of the `baremetal` platform is still a work-in-progress across

pkg/asset/ignition/bootstrap/baremetal/template.go

@@ -1,12 +1,19 @@
 package baremetal
 
 import (
-	"github.com/openshift/installer/pkg/types/baremetal"
+	"net"
 	"strings"
+
+	"github.com/openshift/installer/pkg/types"
+	"github.com/openshift/installer/pkg/types/baremetal"
 )
 
 // TemplateData holds data specific to templates used for the baremetal platform.
 type TemplateData struct {
+	// ProvisioningInterface holds the interface the bootstrap node will use to host the ProvisioningIP below.
+	// When the provisioning network is disabled, this is the external baremetal network interface.
+	ProvisioningInterface string
+
 	// ProvisioningIP holds the IP the bootstrap node will use to service Ironic, TFTP, etc.
 	ProvisioningIP string
 
@@ -16,6 +23,9 @@ type TemplateData struct {
 	// ProvisioningCIDR has the integer CIDR notation, e.g. 255.255.255.0 should be "24"
 	ProvisioningCIDR int
 
+	// ProvisioningDNSMasq determines if we start the dnsmasq service on the bootstrap node.
+	ProvisioningDNSMasq bool
+
 	// ProvisioningDHCPRange has the DHCP range, if DHCP is not external. Otherwise it
 	// should be blank.
 	ProvisioningDHCPRange string
@@ -26,17 +36,22 @@ type TemplateData struct {
 }
 
 // GetTemplateData returns platform-specific data for bootstrap templates.
-func GetTemplateData(config *baremetal.Platform) *TemplateData {
+func GetTemplateData(config *baremetal.Platform, networks []types.MachineNetworkEntry) *TemplateData {
 	var templateData TemplateData
 
 	templateData.ProvisioningIP = config.BootstrapProvisioningIP
 
-	cidr, _ := config.ProvisioningNetworkCIDR.Mask.Size()
-	templateData.ProvisioningCIDR = cidr
-
-	templateData.ProvisioningIPv6 = config.ProvisioningNetworkCIDR.IP.To4() == nil
+	if config.ProvisioningNetwork != baremetal.DisabledProvisioningNetwork {
+		cidr, _ := config.ProvisioningNetworkCIDR.Mask.Size()
+		templateData.ProvisioningCIDR = cidr
+		templateData.ProvisioningIPv6 = config.ProvisioningNetworkCIDR.IP.To4() == nil
+		templateData.ProvisioningInterface = "ens4"
+		templateData.ProvisioningDNSMasq = true
+	}
 
-	if !config.ProvisioningDHCPExternal {
+	switch config.ProvisioningNetwork {
+	case baremetal.ManagedProvisioningNetwork:
+		// When provisioning network is managed, we set a DHCP range:
 		templateData.ProvisioningDHCPRange = config.ProvisioningDHCPRange
 
 		var dhcpAllowList []string
@@ -46,6 +61,19 @@ func GetTemplateData(config *baremetal.Platform) *TemplateData {
 			}
 		}
 		templateData.ProvisioningDHCPAllowList = strings.Join(dhcpAllowList, " ")
+	case baremetal.DisabledProvisioningNetwork:
+		templateData.ProvisioningInterface = "ens3"
+		templateData.ProvisioningDNSMasq = false
+
+		for _, network := range networks {
+			if network.CIDR.Contains(net.ParseIP(templateData.ProvisioningIP)) {
+				templateData.ProvisioningIPv6 = network.CIDR.IP.To4() == nil
+
+				cidr, _ := network.CIDR.Mask.Size()
+				templateData.ProvisioningCIDR = cidr
+			}
+
+		}
 	}
 
 	return &templateData

pkg/asset/ignition/bootstrap/baremetal/template_test.go

@@ -9,10 +9,10 @@ import (
 
 func TestTemplatingIPv4(t *testing.T) {
 	bareMetalConfig := baremetal.Platform{
-		ProvisioningNetworkCIDR:  ipnet.MustParseCIDR("172.22.0.0/24"),
-		BootstrapProvisioningIP:  "172.22.0.2",
-		ProvisioningDHCPExternal: false,
-		ProvisioningDHCPRange:    "172.22.0.10,172.22.0.100",
+		ProvisioningNetworkCIDR: ipnet.MustParseCIDR("172.22.0.0/24"),
+		BootstrapProvisioningIP: "172.22.0.2",
+		ProvisioningNetwork:     baremetal.ManagedProvisioningNetwork,
+		ProvisioningDHCPRange:   "172.22.0.10,172.22.0.100",
 		Hosts: []*baremetal.Host{
 			{
 				Role:           "master",
@@ -33,7 +33,7 @@ func TestTemplatingIPv4(t *testing.T) {
 		},
 	}
 
-	result := GetTemplateData(&bareMetalConfig)
+	result := GetTemplateData(&bareMetalConfig, nil)
 
 	assert.Equal(t, result.ProvisioningDHCPRange, "172.22.0.10,172.22.0.100")
 	assert.Equal(t, result.ProvisioningCIDR, 24)
@@ -44,12 +44,12 @@ func TestTemplatingIPv4(t *testing.T) {
 
 func TestTemplatingIPv6(t *testing.T) {
 	bareMetalConfig := baremetal.Platform{
-		ProvisioningNetworkCIDR:  ipnet.MustParseCIDR("fd2e:6f44:5dd8:b856::0/64"),
-		BootstrapProvisioningIP:  "fd2e:6f44:5dd8:b856::2",
-		ProvisioningDHCPExternal: true,
+		ProvisioningNetworkCIDR: ipnet.MustParseCIDR("fd2e:6f44:5dd8:b856::0/64"),
+		BootstrapProvisioningIP: "fd2e:6f44:5dd8:b856::2",
+		ProvisioningNetwork:     baremetal.UnmanagedProvisioningNetwork,
 	}
 
-	result := GetTemplateData(&bareMetalConfig)
+	result := GetTemplateData(&bareMetalConfig, nil)
 
 	assert.Equal(t, result.ProvisioningDHCPRange, "")
 	assert.Equal(t, result.ProvisioningCIDR, 64)

pkg/asset/ignition/bootstrap/bootstrap.go

@@ -245,7 +245,7 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, releaseI
 
 	switch installConfig.Platform.Name() {
 	case baremetaltypes.Name:
-		platformData.BareMetal = baremetal.GetTemplateData(installConfig.Platform.BareMetal)
+		platformData.BareMetal = baremetal.GetTemplateData(installConfig.Platform.BareMetal, installConfig.MachineNetwork)
 	case vspheretypes.Name:
 		platformData.VSphere = vsphere.GetTemplateData(installConfig.Platform.VSphere)
 	}