Libvirt support for bootstrap + etcd phases

Documentation/dev/libvirt-howto.md

@@ -0,0 +1,29 @@
+# Libvirt howto
+
+Tectonic has limited support for installing a Libvirt cluster. This is useful especially
+for operator development.
+
+## HOW TO:
+
+### Setup and preparation
+* Make sure you have the `virsh` binary on your path.
+* Decide on an IP range. In this example, `192.168.124.0/24`
+* Decide on a domain. In this example, `tt.testing`
+* Download the latest CoreOS image. This is not done automatically to avoid unnecessary downloads. e.g.
+```
+wget https://beta.release.core-os.net/amd64-usr/current/coreos_production_qemu_image.img.bz2
+bunzip2 coreos_production_qemu_image.img.bz2
+```
+
+Now, copy `examples/tectonic.libvirt.yaml` and customize it. You're ready to begin! The workflow is the same, but only the `install assets` and `install bootstrap` steps are supported.
+
+
+## Differences between libvirt and aws:
+
+1. We use the Libvirt DNS server. So, if you want to resolve those names on your host, you'll need to configure NetworkManager's dns overlay mode (dnsmasq mode)
+1. There isn't a load balancer. We need to manually remap port 6443 to 443
+1. We may not support changing the number of workers.
+
+## Remaining tasks
+1. Provision the masters and update the DNS names
+1. Provision the workers and update the ingress names

config.tf

@@ -71,7 +71,7 @@ variable "tectonic_container_images" {
     awscli                               = "quay.io/coreos/awscli:025a357f05242fdad6a81e8a6b520098aa65a600"
     gcloudsdk                            = "google/cloud-sdk:178.0.0-alpine"
     bootkube                             = "quay.io/coreos/bootkube:v0.10.0"
-    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator:beryllium-m2"
+    tnc_operator                         = "quay.io/coreos/tectonic-node-controller-operator-dev:6b54f3e13862531c33b44b322a05a06ec84d074d"
     etcd_cert_signer                     = "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6"
     etcd                                 = "quay.io/coreos/etcd:v3.2.14"
     hyperkube                            = "quay.io/coreos/hyperkube:v1.9.3_coreos.0"
@@ -535,3 +535,17 @@ variable "tectonic_ignition_worker" {
 (internal) Ignition config file path. This is automatically generated by the installer.
 EOF
 }
+
+variable "tectonic_platform" {
+  type = "string"
+
+  description = <<EOF
+(internal) The internal Terraform platform type, e.g. aws or libvirt
+EOF
+}
+
+// This variable is generated by tectonic internally. Do not modify
+variable "tectonic_cluster_id" {
+  type        = "string"
+  description = "(internal) The Tectonic cluster id."
+}

examples/tectonic.libvirt.yaml

@@ -0,0 +1,210 @@
+admin:
+  email: "[email protected]"
+  password: "verysecure"
+# The base DNS domain of the cluster. It must NOT contain a trailing period. Some
+# DNS providers will automatically add this if necessary.
+#
+# Example: `openstack.dev.coreos.systems`.
+#
+# Note: This field MUST be set manually prior to creating the cluster.
+#
+# [Azure-specific NOTE]
+# To use Azure-provided DNS, `BaseDomain` should be set to `""`
+# If using DNS records, ensure that `BaseDomain` is set to a properly configured external DNS zone.
+# Instructions for configuring delegated domains for Azure DNS can be found here: https://docs.microsoft.com/en-us/azure/dns/dns-delegate-domain-azure-dns
+baseDomain: 
+
+libvirt:
+  uri: "qemu:///system"
+  network:
+    name: tectonic
+    ifName: tt0
+    dnsServer: "8.8.8.8"
+    ipRange: "192.168.124.0/24"
+  sshKey: "ssh-rsa ..."
+  imagePath: /path/to/image
+
+ca:
+  # (optional) The content of the PEM-encoded CA certificate, used to generate Tectonic Console's server certificate.
+  # If left blank, a CA certificate will be automatically generated.
+  # cert:
+
+  # (optional) The content of the PEM-encoded CA key, used to generate Tectonic Console's server certificate.
+  # This field is mandatory if `ca_cert` is set.
+  # key:
+
+  # (optional) The algorithm used to generate ca_key.
+  # The default value is currently recommended.
+  # This field is mandatory if `ca_cert` is set.
+  # keyAlg: RSA
+
+containerLinux:
+  # (optional) The Container Linux update channel.
+  #
+  # Examples: `stable`, `beta`, `alpha`
+  channel: beta
+
+  # The Container Linux version to use. Set to `latest` to select the latest available version for the selected update channel.
+  #
+  # Examples: `latest`, `1465.6.0`
+  version: latest
+
+  # (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+  # customCAPEMList:
+
+ddns:
+  key:
+    # (optional) This only applies if you use the modules/dns/ddns module.
+    #
+    # Specifies the RFC2136 Dynamic DNS server key algorithm.
+    # algorithm:
+
+    # (optional) This only applies if you use the modules/dns/ddns module.
+    #
+    # Specifies the RFC2136 Dynamic DNS server key name.
+    # name:
+
+    # (optional) This only applies if you use the modules/dns/ddns module.
+    #
+    # Specifies the RFC2136 Dynamic DNS server key secret.
+    # secret:
+
+  # (optional) This only applies if you use the modules/dns/ddns module.
+  #
+  # Specifies the RFC2136 Dynamic DNS server IP/host to register IP addresses to.
+  # server:
+
+# (optional) DNS prefix used to construct the console and API server endpoints.
+# dnsName:
+
+etcd:
+  nodePools:
+    - etcd
+  external:
+    # (optional) The path of the file containing the CA certificate for TLS communication with etcd.
+    #
+    # Note: This works only when used in conjunction with an external etcd cluster.
+    # If set, the variable `servers` must also be set.
+    # caCertPath: /dev/null
+
+    # (optional) The path of the file containing the client certificate for TLS communication with etcd.
+    #
+    # Note: This works only when used in conjunction with an external etcd cluster.
+    # If set, the variables `servers`, `caCertPath`, and `clientKeyPath` must also be set.
+    # clientCertPath: /dev/null
+
+    # (optional) The path of the file containing the client key for TLS communication with etcd.
+    #
+    # Note: This works only when used in conjunction with an external etcd cluster.
+    # If set, the variables `servers`, `caCertPath`, and `clientCertPath` must also be set.
+    # clientKeyPath: /dev/null
+
+    # (optional) List of external etcd v3 servers to connect with (hostnames/IPs only).
+    # Needs to be set if using an external etcd cluster.
+    # Note: If this variable is defined, the installer will not create self-signed certs.
+    # To provide a CA certificate to trust the etcd servers, set "caCertPath".
+    #
+    # Example: `["etcd1", "etcd2", "etcd3"]`
+    # servers:
+
+iscsi:
+  # (optional) Start iscsid.service to enable iscsi volume attachment.
+  # enabled: false
+
+# The path to the tectonic licence file.
+# You can download the Tectonic license file from your Account overview page at [1].
+#
+# [1] https://account.coreos.com/overview
+licensePath:
+
+master:
+  nodePools:
+    - master
+
+# The name of the cluster.
+# If used in a cloud-environment, this will be prepended to `baseDomain` resulting in the URL to the Tectonic console.
+#
+# Note: This field MUST be set manually prior to creating the cluster.
+# Warning: Special characters in the name like '.' may cause errors on OpenStack platforms due to resource name constraints.
+name:
+
+networking:
+  # (optional) This declares the MTU used by Calico.
+  # mtu:
+
+  # (optional) This declares the IP range to assign Kubernetes pod IPs in CIDR notation.
+  podCIDR: 10.2.0.0/16
+
+  # (optional) This declares the IP range to assign Kubernetes service cluster IPs in CIDR notation.
+  # The maximum size of this IP range is /12
+  serviceCIDR: 10.3.0.0/16
+
+  # (optional) Configures the network to be used in Tectonic. One of the following values can be used:
+  #
+  # - "flannel": enables overlay networking only. This is implemented by flannel using VXLAN.
+  #
+  # - "canal": enables overlay networking including network policy. Overlay is implemented by flannel using VXLAN. Network policy is implemented by Calico.
+  #
+  # - "calico-ipip": [ALPHA] enables BGP based networking. Routing and network policy is implemented by Calico. Note this has been tested on baremetal installations only.
+  #
+  # - "none": disables the installation of any Pod level networking layer provided by Tectonic. By setting this value, users are expected to deploy their own solution to enable network connectivity for Pods and Services.
+  type: canal
+  mtu: 1480
+
+nodePools:
+    # The number of etcd nodes to be created.
+    # If set to zero, the count of etcd nodes will be determined automatically.
+    #
+    # Note: This is not supported on bare metal.
+  - count: 1
+    name: etcd
+
+    # The number of master nodes to be created.
+    # This applies only to cloud platforms.
+  - count: 1
+    name: master
+
+    # The number of worker nodes to be created.
+    # This applies only to cloud platforms.
+  - count: 2
+    name: worker
+
+# The platform used for deploying.
+platform: libvirt
+
+proxy:
+  # (optional) HTTP proxy address.
+  #
+  # Example: `http://myproxy.example.com`
+  # http:
+
+  # (optional) HTTPS proxy address.
+  #
+  # Example: `http://myproxy.example.com`
+  # https:
+
+  # (optional) List of local endpoints that will not use HTTP proxy.
+  #
+  # Example: `["127.0.0.1","localhost",".example.com","10.3.0.1"]`
+  # no:
+
+# The path the pull secret file in JSON format.
+# This is known to be a "Docker pull secret" as produced by the docker login [1] command.
+# A sample JSON content is shown in [2].
+# You can download the pull secret from your Account overview page at [3].
+#
+# [1] https://docs.docker.com/engine/reference/commandline/login/
+#
+# [2] https://coreos.com/os/docs/latest/registry-authentication.html#manual-registry-auth-setup
+#
+# [3] https://account.coreos.com/overview
+pullSecretPath:
+
+# Validity period of the self-signed certificates (in hours).
+# Default is 3 years.
+# This setting is ignored if user provided certificates are used.
+tlsValidityPeriod: 26280
+
+worker:
+  nodePools:
+    - worker

installer/pkg/config-generator/generator.go

@@ -119,7 +119,7 @@ func (c ConfigGenerator) addonConfig() (*kubeaddon.OperatorConfig, error) {
 		return nil, err
 	}
 	addonConfig.DNSConfig.ClusterIP = cidrhost
-	addonConfig.CloudProvider = c.Platform
+	addonConfig.CloudProvider = cloudProvider(c.Platform)
 	return &addonConfig, nil
 }
 
@@ -137,7 +137,7 @@ func (c ConfigGenerator) coreConfig() *kubecore.OperatorConfig {
 	coreConfig.AuthConfig.OIDCUsernameClaim = authConfigOIDCUsernameClaim
 
 	coreConfig.CloudProviderConfig.CloudConfigPath = ""
-	coreConfig.CloudProviderConfig.CloudProviderProfile = strings.ToLower(c.Cluster.Platform)
+	coreConfig.CloudProviderConfig.CloudProviderProfile = cloudProvider(c.Cluster.Platform)
 
 	coreConfig.NetworkConfig.ClusterCIDR = c.Cluster.Networking.PodCIDR
 	coreConfig.NetworkConfig.ServiceCIDR = c.Cluster.Networking.ServiceCIDR
@@ -183,8 +183,8 @@ func (c ConfigGenerator) tncoConfig() (*tnco.OperatorConfig, error) {
 	}
 
 	tncoConfig.ControllerConfig.ClusterDNSIP = cidrhost
-	tncoConfig.ControllerConfig.CloudProvider = strings.ToLower(c.Cluster.Platform)
-	tncoConfig.ControllerConfig.CloudProviderConfig = "" // TODO(yifan): Get CloudProviderConfig.
+	tncoConfig.ControllerConfig.CloudProvider = strings.ToLower(c.Platform) // This is not actually the cloud provider
+	tncoConfig.ControllerConfig.CloudProviderConfig = ""                    // TODO(yifan): Get CloudProviderConfig.
 	tncoConfig.ControllerConfig.ClusterName = c.Cluster.Name
 	tncoConfig.ControllerConfig.BaseDomain = c.Cluster.BaseDomain
 	tncoConfig.ControllerConfig.EtcdInitialCount = c.Cluster.NodeCount(c.Cluster.Etcd.NodePools)
@@ -366,3 +366,12 @@ func cidrhost(iprange string, hostNum int) (string, error) {
 
 	return ip.String(), nil
 }
+
+// Converts a platform to the cloudProvider that k8s understands
+func cloudProvider(platform string) string {
+	switch strings.ToLower(platform) {
+	case "aws":
+		return "aws"
+	}
+	return ""
+}

installer/pkg/config/BUILD.bazel

@@ -23,6 +23,7 @@ go_library(
         "//installer/pkg/config/azure:go_default_library",
         "//installer/pkg/config/gcp:go_default_library",
         "//installer/pkg/config/govcloud:go_default_library",
+        "//installer/pkg/config/libvirt:go_default_library",
         "//installer/pkg/config/metal:go_default_library",
         "//installer/pkg/config/openstack:go_default_library",
         "//installer/pkg/config/vmware:go_default_library",

installer/pkg/config/cluster.go

@@ -9,6 +9,7 @@ import (
 	"github.com/coreos/tectonic-installer/installer/pkg/config/azure"
 	"github.com/coreos/tectonic-installer/installer/pkg/config/gcp"
 	"github.com/coreos/tectonic-installer/installer/pkg/config/govcloud"
+	"github.com/coreos/tectonic-installer/installer/pkg/config/libvirt"
 	"github.com/coreos/tectonic-installer/installer/pkg/config/metal"
 	"github.com/coreos/tectonic-installer/installer/pkg/config/openstack"
 	"github.com/coreos/tectonic-installer/installer/pkg/config/vmware"
@@ -48,6 +49,7 @@ type Cluster struct {
 	azure.Azure         `json:",inline" yaml:"azure,omitempty"`
 	gcp.GCP             `json:",inline" yaml:"gcp,omitempty"`
 	govcloud.GovCloud   `json:",inline" yaml:"govcloud,omitempty"`
+	libvirt.Libvirt     `json:",inline" yaml:"libvirt,omitempty"`
 	metal.Metal         `json:",inline" yaml:"metal,omitempty"`
 	openstack.OpenStack `json:",inline" yaml:"openstack,omitempty"`
 	vmware.VMware       `json:",inline" yaml:"vmware,omitempty"`
@@ -80,6 +82,14 @@ func (c *Cluster) TFVars() (string, error) {
 	c.IgnitionMaster = IgnitionMaster
 	c.IgnitionWorker = IgnitionWorker
 	c.IgnitionEtcd = IgnitionEtcd
+
+	// fill in master ips
+	if c.Platform == "libvirt" {
+		if err := c.Libvirt.TFVars(c.Master.Count); err != nil {
+			return "", err
+		}
+	}
+
 	data, err := json.MarshalIndent(&c, "", "  ")
 	if err != nil {
 		return "", err

installer/pkg/config/libvirt/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["libvirt.go"],
+    importpath = "github.com/coreos/tectonic-installer/installer/pkg/config/libvirt",
+    visibility = ["//visibility:public"],
+    deps = ["//installer/vendor/github.com/apparentlymart/go-cidr/cidr:go_default_library"],
+)