*: implement new bootstrap process

Documentation/dev/libvirt-howto.md

@@ -80,7 +80,7 @@ echo server=/tt.testing/192.168.124.1 | sudo tee /etc/NetworkManager/dnsmasq.d/t
 1. Make sure you have the `virsh` binary installed: `sudo dnf install libvirt-client libvirt-devel`
 2. Install the libvirt terraform provider:
 ```sh
-go get github.com/dmacvicar/terraform-provider-libvirt
+go get github.com/crawford/terraform-provider-libvirt
 mkdir -p ~/.terraform.d/plugins
 cp $GOPATH/bin/terraform-provider-libvirt ~/.terraform.d/plugins/
 ```

Documentation/dev/node-bootstrap-flow.md

@@ -47,12 +47,6 @@ WantedBy=multi-user.target
 This service is enabled by default and can crash-loop until success.
 It is started on every boot.
 
-### `rm-assets.service`
-
-This service waits for the bootkube and tectonic process to be completed.
-It is a oneshot service, thus marked as started only once the script returns with success.
-This is an optional service only present on platforms which pull assets from block storage.
-
 ## Diagram
 
 This is a visual simplified representation of the overall bootstrapping flow.
@@ -64,30 +58,29 @@ Legend:
  * k.s   -> kubelet.service
  * b.s   -> bootkube.service
  * t.s   -> tectonic.service
- * rm.s  -> rm-assets.service
-
-.--------------------------------------------------------------------------------------------------------------------------------+
-|                                                                                                                                |
-|           Provision cloud/userdata           +----------+                                                                      |
-|     ,---------------------------------------o|    TF    |                                                                      |
-|     |                                        +----------+                                                                      |
-|     |                                                                                                                          |
-|     |                                                                                                                          |
-|     |                                                                                                                          |
-|     |                                                                                                                          |
-|     V                                                                                                                          |
-| +-------+                          Before   +------------+   Before                                                            |
-| |  IGN  |                  .--------------->|    k.s     |o--------.                                                           |
-| +-------+                  |                +------------+         |                                                           |
-|     |                      |                   ^      |            |    +-----+      Before     +-------+   Before +-----+     |
-|     '----------------------'                   |      v            '--->| b.s |o--------------->|  t.s  |--------> |rm.s |     |
-|   Enable                                       '------'                 +-----+                 +-------+          +-----+     |
-|                                                                                                                                |
-|                                                                                                                                |
-|                                       o                            o                                                           |
-|                                       |                            |                                                           |
-|                                       |        * Each boot         |         * First boot                                      |
-|                                       |        * All nodes         |         * Bootkube master                                 |
-|                                       |                            |                                                           |
-'---------------------------------------o----------------------------o-----------------------------------------------------------+
+
+.-----------------------------------------------------------------------------------------------------------+
+|                                                                                                           |
+|           Provision cloud/userdata           +----------+                                                 |
+|     ,---------------------------------------o|    TF    |                                                 |
+|     |                                        +----------+                                                 |
+|     |                                                                                                     |
+|     |                                                                                                     |
+|     |                                                                                                     |
+|     |                                                                                                     |
+|     V                                                                                                     |
+| +-------+                          Before   +------------+   Before                                       |
+| |  IGN  |                  .--------------->|    k.s     |o--------.                                      |
+| +-------+                  |                +------------+         |                                      |
+|     |                      |                   ^      |            |    +-----+      Before     +-------+ |
+|     '----------------------'                   |      v            '--->| b.s |o--------------->|  t.s  | |
+|   Enable                                       '------'                 +-----+                 +-------+ |
+|                                                                                                           |
+|                                                                                                           |
+|                                       o                            o                                      |
+|                                       |                            |                                      |
+|                                       |        * Each boot         |         * First boot                 |
+|                                       |        * All nodes         |         * Bootkube master            |
+|                                       |                            |                                      |
+'---------------------------------------o----------------------------o--------------------------------------+
 ```

config.tf

@@ -58,20 +58,19 @@ variable "tectonic_container_images" {
 
   default = {
     addon_resizer                        = "gcr.io/google_containers/addon-resizer:2.1"
-    awscli                               = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600"
     bootkube                             = "quay.io/coreos/bootkube:v0.10.0"
-    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
+    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
     etcd_cert_signer                     = "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6"
     etcd                                 = "quay.io/coreos/etcd:v3.2.14"
     hyperkube                            = "openshift/origin-node:latest"
-    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
-    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
-    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
-    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
+    kube_core_renderer                   = "quay.io/coreos/kube-core-renderer-dev:32681d92536d3e180604622825ee76935d0a5692"
+    kube_core_operator                   = "quay.io/coreos/kube-core-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
+    tectonic_channel_operator            = "quay.io/coreos/tectonic-channel-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
+    kube_addon_operator                  = "quay.io/coreos/kube-addon-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
     tectonic_alm_operator                = "quay.io/coreos/tectonic-alm-operator:v0.3.1"
-    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
-    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
-    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:6705af4bcf26f899e347c88ffddc892a236648e4"
+    tectonic_ingress_controller_operator = "quay.io/coreos/tectonic-ingress-controller-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
+    tectonic_utility_operator            = "quay.io/coreos/tectonic-utility-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
+    tectonic_network_operator            = "quay.io/coreos/tectonic-network-operator-dev:32681d92536d3e180604622825ee76935d0a5692"
   }
 }
 
@@ -333,12 +332,12 @@ variable "tectonic_kubelet_debug_config" {
   description = "(internal) debug flags for the kubelet (used in CI only)"
 }
 
-variable "tectonic_ignition_master" {
-  type    = "string"
-  default = ""
+variable "tectonic_ignition_masters" {
+  type    = "list"
+  default = []
 
   description = <<EOF
-(internal) Ignition config file path. This is automatically generated by the installer.
+(internal) Ignition config file paths. This is automatically generated by the installer.
 EOF
 }
 

installer/pkg/config-generator/fixtures/test-aws.yaml

@@ -6,15 +6,12 @@ networking:
   mtu: 1480
   podCIDR: 10.2.0.0/16
   serviceCIDR: 10.3.0.0/16
-masters:
+master:
   nodePools:
     - master
-workers:
+worker:
   nodePools:
     - worker
-etcd:
-  nodePools:
-    - etcd
 pullSecretPath: /path/config.json
 licensePath: /path/tectonic-license.txt
 containerLinux:
@@ -27,12 +24,6 @@ aws:
   region: eu-west-1
   sshKey: tectonic
   vpcCIDRBlock: 10.0.0.0/16
-  etcd:
-    ec2Type: t2.medium
-    rootVolume:
-      iops: 100
-      size: 30
-      type: gp2
   master:
     ec2Type: t2.medium
     rootVolume:
@@ -47,8 +38,6 @@ aws:
       type: gp2
 nodePools:
   - name: master
-    count: 2
-  - name: worker
     count: 3
-  - name: etcd
+  - name: worker
     count: 3

installer/pkg/config-generator/fixtures/test.yaml

@@ -1,9 +1,9 @@
 name: test
 platform: aws
 baseDomain: cluster.com
-etcd:
+master:
   nodePools:
-    - etcd
+    - master
 nodePools:
-  - name: etcd
+  - name: master
     count: 3

installer/pkg/config-generator/generator.go

@@ -203,7 +203,7 @@ func (c *ConfigGenerator) tncoConfig() (*tnco.OperatorConfig, error) {
 	tncoConfig.ControllerConfig.CloudProviderConfig = "" // TODO(yifan): Get CloudProviderConfig.
 	tncoConfig.ControllerConfig.ClusterName = c.Cluster.Name
 	tncoConfig.ControllerConfig.BaseDomain = c.Cluster.BaseDomain
-	tncoConfig.ControllerConfig.EtcdInitialCount = c.Cluster.NodeCount(c.Cluster.Etcd.NodePools)
+	tncoConfig.ControllerConfig.EtcdInitialCount = c.Cluster.NodeCount(c.Cluster.Master.NodePools)
 	tncoConfig.ControllerConfig.AdditionalConfigs = []string{} // TODO(yifan): Get additional configs.
 	tncoConfig.ControllerConfig.NodePoolUpdateLimit = nil      // TODO(yifan): Get the node pool update limit.
 
@@ -271,7 +271,7 @@ func marshalYAML(obj interface{}) (string, error) {
 }
 
 func (c *ConfigGenerator) getEtcdServersURLs() string {
-	etcdServers := make([]string, c.Cluster.NodeCount(c.Cluster.Etcd.NodePools))
+	etcdServers := make([]string, c.Cluster.NodeCount(c.Cluster.Master.NodePools))
 	for i := range etcdServers {
 		etcdServers[i] = fmt.Sprintf("https://%s-etcd-%v.%s:2379", c.Cluster.Name, i, c.Cluster.BaseDomain)
 	}

installer/pkg/config-generator/ignition.go

@@ -42,29 +42,50 @@ func (c *ConfigGenerator) poolToRoleMap() map[string]string {
 func (c *ConfigGenerator) GenerateIgnConfig(clusterDir string) error {
 	poolToRole := c.poolToRoleMap()
 	for _, p := range c.NodePools {
-		ignFile := p.IgnitionFile
-		ignCfg, err := parseIgnFile(ignFile)
+		role := poolToRole[p.Name]
+		if _, ok := ignFilesPath[role]; !ok {
+			return fmt.Errorf("unrecognized pool: %s", p.Name)
+		}
+
+		ignCfg, err := parseIgnFile(p.IgnitionFile)
 		if err != nil {
 			return fmt.Errorf("failed to GenerateIgnConfig for pool %s and file %s: %v", p.Name, p.IgnitionFile, err)
 		}
-		role := poolToRole[p.Name]
-		// TODO(alberto): Append block need to be different for each etcd node.
-		// add loop over count if role is etcd
-		c.embedAppendBlock(ignCfg, role)
 
-		ca := filepath.Join(clusterDir, caPath)
-		if err = c.appendCertificateAuthority(ignCfg, ca); err != nil {
+		var ignCfgs []ignconfigtypes.Config
+		for i := 0; i < p.Count; i++ {
+			ignCfgs = append(ignCfgs, *ignCfg)
+		}
+
+		ca, err := ioutil.ReadFile(filepath.Join(clusterDir, caPath))
+		if err != nil {
 			return err
 		}
 
+		for i := range ignCfgs {
+			c.appendCertificateAuthority(&ignCfgs[i], ca)
+		}
+
 		// XXX(crawford): The SSH key should only be added to the bootstrap
 		//                node. After that, MCO should be responsible for
 		//                distributing SSH keys.
-		c.embedUserBlock(ignCfg)
+		for i := range ignCfgs {
+			c.embedUserBlock(&ignCfgs[i])
+		}
 
 		fileTargetPath := filepath.Join(clusterDir, ignFilesPath[role])
-		if err = ignCfgToFile(*ignCfg, fileTargetPath); err != nil {
-			return err
+		if role == "master" {
+			for i := range ignCfgs {
+				c.embedAppendBlock(&ignCfgs[i], role, fmt.Sprintf("etcd_index=%d", i))
+				if err = ignCfgToFile(ignCfgs[i], fmt.Sprintf(fileTargetPath, i)); err != nil {
+					return err
+				}
+			}
+		} else {
+			c.embedAppendBlock(&ignCfgs[0], role, "")
+			if err = ignCfgToFile(ignCfgs[0], fileTargetPath); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -91,25 +112,18 @@ func parseIgnFile(filePath string) (*ignconfigtypes.Config, error) {
 	return &cfg, nil
 }
 
-func (c *ConfigGenerator) embedAppendBlock(ignCfg *ignconfigtypes.Config, role string) {
+func (c *ConfigGenerator) embedAppendBlock(ignCfg *ignconfigtypes.Config, role string, query string) {
 	appendBlock := ignconfigtypes.ConfigReference{
-		Source:       c.getTNCURL(role),
+		Source:       c.getTNCURL(role, query),
 		Verification: ignconfigtypes.Verification{Hash: nil},
 	}
 	ignCfg.Ignition.Config.Append = append(ignCfg.Ignition.Config.Append, appendBlock)
 }
 
-func (c *ConfigGenerator) appendCertificateAuthority(ignCfg *ignconfigtypes.Config, caPath string) error {
-	ca, err := ioutil.ReadFile(caPath)
-	if err != nil {
-		return err
-	}
-
+func (c *ConfigGenerator) appendCertificateAuthority(ignCfg *ignconfigtypes.Config, ca []byte) {
 	ignCfg.Ignition.Security.TLS.CertificateAuthorities = append(ignCfg.Ignition.Security.TLS.CertificateAuthorities, ignconfigtypes.CaReference{
 		Source: dataurl.EncodeBytes(ca),
 	})
-
-	return nil
 }
 
 func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {
@@ -123,7 +137,7 @@ func (c *ConfigGenerator) embedUserBlock(ignCfg *ignconfigtypes.Config) {
 	ignCfg.Passwd.Users = append(ignCfg.Passwd.Users, userBlock)
 }
 
-func (c *ConfigGenerator) getTNCURL(role string) string {
+func (c *ConfigGenerator) getTNCURL(role string, query string) string {
 	var u string
 
 	// cloud platforms put this behind a load balancer which remaps ports;
@@ -133,19 +147,13 @@ func (c *ConfigGenerator) getTNCURL(role string) string {
 		port = 49500
 	}
 
-	// XXX: The bootstrap node on AWS uses a CNAME to redirect TNC-bound
-	// traffic to S3. Because of this, HTTPS cannot be used.
-	scheme := "https"
-	if c.Platform == config.PlatformAWS && role == "master" {
-		scheme = "http"
-	}
-
 	if role == "master" || role == "worker" {
 		u = func() *url.URL {
 			return &url.URL{
-				Scheme: scheme,
-				Host:   fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
-				Path:   fmt.Sprintf("/config/%s", role),
+				Scheme:   "https",
+				Host:     fmt.Sprintf("%s-tnc.%s:%d", c.Name, c.BaseDomain, port),
+				Path:     fmt.Sprintf("/config/%s", role),
+				RawQuery: query,
 			}
 		}().String()
 	}

installer/pkg/config/cluster.go

@@ -13,7 +13,8 @@ import (
 
 const (
 	// IgnitionMaster is the relative path to the ign master cfg from the tf working directory
-	IgnitionMaster = "master.ign"
+	// This is a format string so that the index can be populated later
+	IgnitionMaster = "master-%d.ign"
 	// IgnitionWorker is the relative path to the ign worker cfg from the tf working directory
 	IgnitionWorker = "worker.ign"
 	// IgnitionEtcd is the relative path to the ign etcd cfg from the tf working directory
@@ -81,9 +82,9 @@ type Cluster struct {
 	CA              `json:",inline" yaml:"CA,omitempty"`
 	ContainerLinux  `json:",inline" yaml:"containerLinux,omitempty"`
 	Etcd            `json:",inline" yaml:"etcd,omitempty"`
-	IgnitionEtcd    string `json:"tectonic_ignition_etcd,omitempty" yaml:"-"`
-	IgnitionMaster  string `json:"tectonic_ignition_master,omitempty" yaml:"-"`
-	IgnitionWorker  string `json:"tectonic_ignition_worker,omitempty" yaml:"-"`
+	IgnitionEtcd    string   `json:"tectonic_ignition_etcd,omitempty" yaml:"-"`
+	IgnitionMasters []string `json:"tectonic_ignition_masters,omitempty" yaml:"-"`
+	IgnitionWorker  string   `json:"tectonic_ignition_worker,omitempty" yaml:"-"`
 	Internal        `json:",inline" yaml:"-"`
 	libvirt.Libvirt `json:",inline" yaml:"libvirt,omitempty"`
 	LicensePath     string `json:"tectonic_license_path,omitempty" yaml:"licensePath,omitempty"`
@@ -117,13 +118,16 @@ func (c *Cluster) TFVars() (string, error) {
 	c.Master.Count = c.NodeCount(c.Master.NodePools)
 	c.Worker.Count = c.NodeCount(c.Worker.NodePools)
 
-	c.IgnitionMaster = IgnitionMaster
+	for i := 0; i < c.Master.Count; i++ {
+		c.IgnitionMasters = append(c.IgnitionMasters, fmt.Sprintf(IgnitionMaster, i))
+	}
+
 	c.IgnitionWorker = IgnitionWorker
 	c.IgnitionEtcd = IgnitionEtcd
 
 	// fill in master ips
 	if c.Platform == PlatformLibvirt {
-		if err := c.Libvirt.TFVars(c.Master.Count); err != nil {
+		if err := c.Libvirt.TFVars(c.Master.Count, c.Worker.Count, c.Etcd.Count); err != nil {
 			return "", err
 		}
 	}