Use TLS authentication for cluster-api libvirt connection

docs/dev/libvirt-howto.md

@@ -62,28 +62,34 @@ polkit.addRule(function(action, subject) {
 EOF
 ```
 
-### 1.7 Configure libvirt to accept TCP connections
+### 1.7 Configure libvirt to accept TLS connections
 
 The Kubernetes [cluster-api](https://github.com/kubernetes-sigs/cluster-api)
 components drive deployment of worker machines.  The libvirt cluster-api
 provider will run inside the local cluster, and will need to connect back to
 the libvirt instance on the host machine to deploy workers.
 
-In order for this to work, you'll need to enable TCP connections for libvirt.
+In order for this to work, you'll need to enable TLS connections for libvirt.
+To do this, first generate the TLS assets:
 
-#### Configure libvirtd.conf
-To do this, first modify your `/etc/libvirt/libvirtd.conf` and set the
-following:
 ```
-listen_tls = 0
-listen_tcp = 1
-auth_tcp="none"
-tcp_port = "16509"
+$ go run ./hack/libvirt-ca/main.go --network="192.168.124.0/24" --out $HOME
 ```
 
-Note that authentication is not currently supported, but should be soon.
+You can omit the `--network` flag if you're using the default
+`192.168.124.0/24` network, and of course store the resulting
+certificates and keys wherever you like.
+
+Next, modify your `/etc/libvirt/libvirtd.conf` and set the following:
+
+```
+listen_tls = 1
+tls_port = "16514"
+key_file = "/path/to/serverkey.pem"
+cert_file = "/path/to/servercert.pem"
+ca_file = "/path/to/cacert.pem"
+```
 
-#### Configure the service runner to pass `--listen` to libvirtd
 In addition to the config, you'll have to pass an additional command-line
 argument to libvirtd. On Fedora, modify `/etc/sysconfig/libvirtd` and set:
 
@@ -99,49 +105,6 @@ libvirtd_opts="--listen"
 
 Next, restart libvirt: `systemctl restart libvirtd`
 
-#### Firewall
-Finally, if you have a firewall, you may have to allow connections to the
-libvirt daemon from the IP range used by your cluster nodes.
-
-#### Manual management
-The following example rule works for the suggested cluster ipRange of `192.168.124.0/24` and a libvirt *default* subnet of `192.168.122.0/24`, which might be different in your configuration:
-
-```
-iptables -I INPUT -p tcp -s 192.168.124.0/24 -d 192.168.122.1 --dport 16509 \
-  -j ACCEPT -m comment --comment "Allow insecure libvirt clients"
-```
-
-#### Firewalld
-
-If using `firewalld`, simply obtain the name of the existing active zone which
-can be used to integrate the appropriate source and ports to allow connections from
-the IP range used by your cluster nodes. An example is shown below.
-
-```console
-$ sudo firewall-cmd --get-active-zones
-FedoraWorkstation
-  interfaces: enp0s25 tun0
-```
-With the name of the active zone, include the source and port to allow connections
-from the IP range used by your cluster nodes. The default subnet is `192.168.124.0/24`
-unless otherwise specified.
-
-```sh
-sudo firewall-cmd --zone=FedoraWorkstation --add-source=192.168.124.0/24
-sudo firewall-cmd --zone=FedoraWorkstation --add-port=16509/tcp
-```
-
-Verification of the source and port can be done listing the zone
-
-```sh
-sudo firewall-cmd --zone=FedoraWorkstation --list-ports
-sudo firewall-cmd --zone=FedoraWorkstation --list-sources
-```
-
-NOTE: When the firewall rules are no longer needed, `sudo firewalld-cmd --reload`
-will remove the changes made as they were not permanently added. For persistence,
-add `--permanent` to the `firewall-cmd` commands and run them a second time.
-
 ### 1.8 Configure default libvirt storage pool
 
 Check to see if a default storage pool has been defined in Libvirt by running

examples/libvirt.yaml

@@ -11,10 +11,11 @@ admin:
 baseDomain:
 
 libvirt:
-  # You must specify an IP address here that libvirtd is listening on,
-  # and that the cluster-api controller pod will be able to connect
-  # to.  Often 192.168.122.1 is the default for the virbr0 interface.
-  uri: qemu+tcp://192.168.122.1/system
+  uri: qemu:///system
+  tls:
+    caPath: /path/to/cacert.pem
+    certPath: /path/to/clientcert.pem
+    keyPath: /path/to/clientkey.pem
   network:
     name: tectonic
     ifName: tt0

hack/libvirt-ca/main.go

@@ -0,0 +1,206 @@
+package main
+
+import (
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/apparentlymart/go-cidr/cidr"
+	"github.com/openshift/installer/pkg/asset/tls"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+const (
+	orgUnit          = "openshift"
+	caCommonName     = "libvirt"
+	serverCommonName = "libvirt"
+	clientCommonName = "openshift-cluster-api"
+	defaultNetwork   = "192.168.124.0/24"
+	defaultOutDir    = "."
+
+	// Libvirt is picky about the naming here:
+	// https://libvirt.org/remote.html
+	caCertFile     = "cacert.pem"
+	caKeyFile      = "cakey.pem"
+	serverCertFile = "servercert.pem"
+	serverKeyFile  = "serverkey.pem"
+	clientCertFile = "clientcert.pem"
+	clientKeyFile  = "clientkey.pem"
+)
+
+var (
+	network = kingpin.Flag("network", "Cluster network CIDR.").
+		Short('n').Default(defaultNetwork).String()
+
+	outDir = kingpin.Flag("out", "Output directory.").
+		Short('o').Default(defaultOutDir).ExistingDir()
+)
+
+type certificate struct {
+	key  *rsa.PrivateKey
+	cert *x509.Certificate
+}
+
+func (c *certificate) WritePEMs(certPath, keyPath string) error {
+	if err := ioutil.WriteFile(keyPath, []byte(tls.PrivateKeyToPem(c.key)), 0600); err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(certPath, []byte(tls.CertToPem(c.cert)), 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getGateway(network string) (net.IP, error) {
+	_, ipNet, err := net.ParseCIDR(network)
+	if err != nil {
+		return nil, err
+	}
+
+	gateway, err := cidr.Host(ipNet, 1)
+	if err != nil {
+		return nil, err
+	}
+
+	return gateway, nil
+}
+
+func generateCA() (*certificate, error) {
+	var ca certificate
+	var err error
+
+	cfg := &tls.CertCfg{
+		Subject: pkix.Name{
+			CommonName:         caCommonName,
+			OrganizationalUnit: []string{orgUnit},
+		},
+		KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		Validity:  tls.ValidityTenYears,
+		IsCA:      true,
+	}
+
+	ca.key, ca.cert, err = tls.GenerateRootCertKey(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ca, err
+}
+
+func generateServerCert(dnsNames []string, ips []net.IP, ca *certificate) (*certificate, error) {
+	var server certificate
+	var err error
+
+	cfg := &tls.CertCfg{
+		KeyUsages:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		Subject: pkix.Name{
+			CommonName:         serverCommonName,
+			OrganizationalUnit: []string{orgUnit},
+		},
+		DNSNames:    dnsNames,
+		Validity:    tls.ValidityTenYears,
+		IPAddresses: ips,
+		IsCA:        false,
+	}
+
+	server.key, server.cert, err = tls.GenerateCert(ca.key, ca.cert, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &server, err
+}
+
+func generateClientCert(ca *certificate) (*certificate, error) {
+	var client certificate
+	var err error
+
+	cfg := &tls.CertCfg{
+		Subject: pkix.Name{
+			CommonName:         clientCommonName,
+			OrganizationalUnit: []string{orgUnit},
+		},
+		KeyUsages:    x509.KeyUsageKeyEncipherment,
+		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		Validity:     tls.ValidityTenYears,
+	}
+
+	client.key, client.cert, err = tls.GenerateCert(ca.key, ca.cert, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &client, nil
+}
+
+func writeCertificates(dir string, ca, server, client *certificate) error {
+	// Certificate Authority
+	caCertPath := filepath.Join(dir, caCertFile)
+	caKeyPath := filepath.Join(dir, caKeyFile)
+	if err := ca.WritePEMs(caCertPath, caKeyPath); err != nil {
+		return err
+	}
+
+	// Server Certificate
+	serverCertPath := filepath.Join(dir, serverCertFile)
+	serverKeyPath := filepath.Join(dir, serverKeyFile)
+	if err := server.WritePEMs(serverCertPath, serverKeyPath); err != nil {
+		return err
+	}
+
+	// Client Certificate
+	clientCertPath := filepath.Join(dir, clientCertFile)
+	clientKeyPath := filepath.Join(dir, clientKeyFile)
+	if err := server.WritePEMs(clientCertPath, clientKeyPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	kingpin.Parse()
+
+	hostname, err := os.Hostname()
+	if err != nil {
+		fmt.Printf("Failed to determine hostname: %v\n", err)
+		os.Exit(1)
+	}
+
+	gateway, err := getGateway(*network)
+	if err != nil {
+		fmt.Printf("Failed to read network: %v\n", err)
+		os.Exit(1)
+	}
+
+	ca, err := generateCA()
+	if err != nil {
+		fmt.Printf("Failed to generate CA: %v\n", err)
+		os.Exit(1)
+	}
+
+	server, err := generateServerCert([]string{hostname}, []net.IP{gateway}, ca)
+	if err != nil {
+		fmt.Printf("Failed to generate server certificate: %v\n", err)
+		os.Exit(1)
+	}
+
+	client, err := generateClientCert(ca)
+	if err != nil {
+		fmt.Printf("Failed to generate client certificate: %v\n", err)
+		os.Exit(1)
+	}
+
+	if err := writeCertificates(*outDir, ca, server, client); err != nil {
+		fmt.Printf("Failed to write certificate files: %v\n", err)
+		os.Exit(1)
+	}
+}

installer/pkg/config-generator/generator.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/url"
 	"path/filepath"
 	"strings"
 
@@ -37,8 +38,11 @@ const (
 	certificatesStrategy          = "userProvidedCA"
 	identityAPIService            = "tectonic-identity-api.tectonic-system.svc.cluster.local"
 	maoTargetNamespace            = "openshift-cluster-api"
+	libvirtPKIPath                = "/etc/pki/libvirt"
 )
 
+var errBadLibvirtScheme = errors.New("bad libvirt URI scheme")
+
 // ConfigGenerator defines the cluster config generation for a cluster.
 type ConfigGenerator struct {
 	config.Cluster
@@ -136,9 +140,14 @@ func (c *ConfigGenerator) maoConfig(clusterDir string) (*maoOperatorConfig, erro
 		}
 
 	case config.PlatformLibvirt:
+		uri, err := libvirtURI(c.Libvirt.URI, c.Libvirt.IPRange)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create libvirt URI: %v", err)
+		}
+
 		cfg.Libvirt = &libvirtConfig{
+			URI:         uri.String(),
 			ClusterName: c.Name,
-			URI:         c.Libvirt.URI,
 			NetworkName: c.Libvirt.Network.Name,
 			IPRange:     c.Libvirt.IPRange,
 			Replicas:    c.NodeCount(c.Worker.NodePools),
@@ -519,3 +528,41 @@ func tectonicCloudProvider(platform config.Platform) string {
 	}
 	panic("invalid platform")
 }
+
+// Returns a libvirt URI given for the machine-api-operator given the
+// URI in the config and the cluster network in CIDR format.
+func libvirtURI(configURI, networkCIDR string) (*url.URL, error) {
+	_, ipNet, err := net.ParseCIDR(networkCIDR)
+	if err != nil {
+		return nil, err
+	}
+
+	gateway, err := cidr.Host(ipNet, 1)
+	if err != nil {
+		return nil, err
+	}
+
+	libvirtURI, err := url.Parse(configURI)
+	if err != nil {
+		return nil, err
+	}
+
+	// If there's a transport in the configured URI, replace it with
+	// TLS.  Otherwise, if there's none, explicityly add TLS.
+	scheme := strings.Split(libvirtURI.Scheme, "+")
+	switch len(scheme) {
+	case 1, 2: // Replace or add the transport.
+		libvirtURI.Scheme = scheme[0] + "+tls"
+
+	default:
+		return nil, errBadLibvirtScheme
+	}
+
+	query := libvirtURI.Query()
+	query.Set("pkipath", libvirtPKIPath)
+
+	libvirtURI.Host = gateway.String()
+	libvirtURI.RawQuery = query.Encode()
+
+	return libvirtURI, nil
+}