diff --git a/docs/dev/libvirt-howto.md b/docs/dev/libvirt-howto.md index d92dadb2d9b..2a750024850 100644 --- a/docs/dev/libvirt-howto.md +++ b/docs/dev/libvirt-howto.md @@ -62,28 +62,34 @@ polkit.addRule(function(action, subject) { EOF ``` -### 1.7 Configure libvirt to accept TCP connections +### 1.7 Configure libvirt to accept TLS connections The Kubernetes [cluster-api](https://github.com/kubernetes-sigs/cluster-api) components drive deployment of worker machines. The libvirt cluster-api provider will run inside the local cluster, and will need to connect back to the libvirt instance on the host machine to deploy workers. -In order for this to work, you'll need to enable TCP connections for libvirt. +In order for this to work, you'll need to enable TLS connections for libvirt. +To do this, first generate the TLS assets: -#### Configure libvirtd.conf -To do this, first modify your `/etc/libvirt/libvirtd.conf` and set the -following: ``` -listen_tls = 0 -listen_tcp = 1 -auth_tcp="none" -tcp_port = "16509" +$ go run ./hack/libvirt-ca/main.go --network="192.168.124.0/24" --out $HOME ``` -Note that authentication is not currently supported, but should be soon. +You can omit the `--network` flag if you're using the default +`192.168.124.0/24` network, and of course store the resulting +certificates and keys wherever you like. + +Next, modify your `/etc/libvirt/libvirtd.conf` and set the following: + +``` +listen_tls = 1 +tls_port = "16514" +key_file = "/path/to/serverkey.pem" +cert_file = "/path/to/servercert.pem" +ca_file = "/path/to/cacert.pem" +``` -#### Configure the service runner to pass `--listen` to libvirtd In addition to the config, you'll have to pass an additional command-line argument to libvirtd. On Fedora, modify `/etc/sysconfig/libvirtd` and set: @@ -99,49 +105,6 @@ libvirtd_opts="--listen" Next, restart libvirt: `systemctl restart libvirtd` -#### Firewall -Finally, if you have a firewall, you may have to allow connections to the -libvirt daemon from the IP range used by your cluster nodes. - -#### Manual management -The following example rule works for the suggested cluster ipRange of `192.168.124.0/24` and a libvirt *default* subnet of `192.168.122.0/24`, which might be different in your configuration: - -``` -iptables -I INPUT -p tcp -s 192.168.124.0/24 -d 192.168.122.1 --dport 16509 \ - -j ACCEPT -m comment --comment "Allow insecure libvirt clients" -``` - -#### Firewalld - -If using `firewalld`, simply obtain the name of the existing active zone which -can be used to integrate the appropriate source and ports to allow connections from -the IP range used by your cluster nodes. An example is shown below. - -```console -$ sudo firewall-cmd --get-active-zones -FedoraWorkstation - interfaces: enp0s25 tun0 -``` -With the name of the active zone, include the source and port to allow connections -from the IP range used by your cluster nodes. The default subnet is `192.168.124.0/24` -unless otherwise specified. - -```sh -sudo firewall-cmd --zone=FedoraWorkstation --add-source=192.168.124.0/24 -sudo firewall-cmd --zone=FedoraWorkstation --add-port=16509/tcp -``` - -Verification of the source and port can be done listing the zone - -```sh -sudo firewall-cmd --zone=FedoraWorkstation --list-ports -sudo firewall-cmd --zone=FedoraWorkstation --list-sources -``` - -NOTE: When the firewall rules are no longer needed, `sudo firewalld-cmd --reload` -will remove the changes made as they were not permanently added. For persistence, -add `--permanent` to the `firewall-cmd` commands and run them a second time. - ### 1.8 Configure default libvirt storage pool Check to see if a default storage pool has been defined in Libvirt by running diff --git a/examples/libvirt.yaml b/examples/libvirt.yaml index 7065c8ad9ae..c2bdd1ab724 100644 --- a/examples/libvirt.yaml +++ b/examples/libvirt.yaml @@ -11,10 +11,11 @@ admin: baseDomain: libvirt: - # You must specify an IP address here that libvirtd is listening on, - # and that the cluster-api controller pod will be able to connect - # to. Often 192.168.122.1 is the default for the virbr0 interface. - uri: qemu+tcp://192.168.122.1/system + uri: qemu:///system + tls: + caPath: /path/to/cacert.pem + certPath: /path/to/clientcert.pem + keyPath: /path/to/clientkey.pem network: name: tectonic ifName: tt0 diff --git a/hack/libvirt-ca/main.go b/hack/libvirt-ca/main.go new file mode 100644 index 00000000000..43f2fcb4c45 --- /dev/null +++ b/hack/libvirt-ca/main.go @@ -0,0 +1,206 @@ +package main + +import ( + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + + "github.com/apparentlymart/go-cidr/cidr" + "github.com/openshift/installer/pkg/asset/tls" + "gopkg.in/alecthomas/kingpin.v2" +) + +const ( + orgUnit = "openshift" + caCommonName = "libvirt" + serverCommonName = "libvirt" + clientCommonName = "openshift-cluster-api" + defaultNetwork = "192.168.124.0/24" + defaultOutDir = "." + + // Libvirt is picky about the naming here: + // https://libvirt.org/remote.html + caCertFile = "cacert.pem" + caKeyFile = "cakey.pem" + serverCertFile = "servercert.pem" + serverKeyFile = "serverkey.pem" + clientCertFile = "clientcert.pem" + clientKeyFile = "clientkey.pem" +) + +var ( + network = kingpin.Flag("network", "Cluster network CIDR."). + Short('n').Default(defaultNetwork).String() + + outDir = kingpin.Flag("out", "Output directory."). + Short('o').Default(defaultOutDir).ExistingDir() +) + +type certificate struct { + key *rsa.PrivateKey + cert *x509.Certificate +} + +func (c *certificate) WritePEMs(certPath, keyPath string) error { + if err := ioutil.WriteFile(keyPath, []byte(tls.PrivateKeyToPem(c.key)), 0600); err != nil { + return err + } + + if err := ioutil.WriteFile(certPath, []byte(tls.CertToPem(c.cert)), 0644); err != nil { + return err + } + + return nil +} + +func getGateway(network string) (net.IP, error) { + _, ipNet, err := net.ParseCIDR(network) + if err != nil { + return nil, err + } + + gateway, err := cidr.Host(ipNet, 1) + if err != nil { + return nil, err + } + + return gateway, nil +} + +func generateCA() (*certificate, error) { + var ca certificate + var err error + + cfg := &tls.CertCfg{ + Subject: pkix.Name{ + CommonName: caCommonName, + OrganizationalUnit: []string{orgUnit}, + }, + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + Validity: tls.ValidityTenYears, + IsCA: true, + } + + ca.key, ca.cert, err = tls.GenerateRootCertKey(cfg) + if err != nil { + return nil, err + } + + return &ca, err +} + +func generateServerCert(dnsNames []string, ips []net.IP, ca *certificate) (*certificate, error) { + var server certificate + var err error + + cfg := &tls.CertCfg{ + KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + Subject: pkix.Name{ + CommonName: serverCommonName, + OrganizationalUnit: []string{orgUnit}, + }, + DNSNames: dnsNames, + Validity: tls.ValidityTenYears, + IPAddresses: ips, + IsCA: false, + } + + server.key, server.cert, err = tls.GenerateCert(ca.key, ca.cert, cfg) + if err != nil { + return nil, err + } + + return &server, err +} + +func generateClientCert(ca *certificate) (*certificate, error) { + var client certificate + var err error + + cfg := &tls.CertCfg{ + Subject: pkix.Name{ + CommonName: clientCommonName, + OrganizationalUnit: []string{orgUnit}, + }, + KeyUsages: x509.KeyUsageKeyEncipherment, + ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + Validity: tls.ValidityTenYears, + } + + client.key, client.cert, err = tls.GenerateCert(ca.key, ca.cert, cfg) + if err != nil { + return nil, err + } + + return &client, nil +} + +func writeCertificates(dir string, ca, server, client *certificate) error { + // Certificate Authority + caCertPath := filepath.Join(dir, caCertFile) + caKeyPath := filepath.Join(dir, caKeyFile) + if err := ca.WritePEMs(caCertPath, caKeyPath); err != nil { + return err + } + + // Server Certificate + serverCertPath := filepath.Join(dir, serverCertFile) + serverKeyPath := filepath.Join(dir, serverKeyFile) + if err := server.WritePEMs(serverCertPath, serverKeyPath); err != nil { + return err + } + + // Client Certificate + clientCertPath := filepath.Join(dir, clientCertFile) + clientKeyPath := filepath.Join(dir, clientKeyFile) + if err := server.WritePEMs(clientCertPath, clientKeyPath); err != nil { + return err + } + + return nil +} + +func main() { + kingpin.Parse() + + hostname, err := os.Hostname() + if err != nil { + fmt.Printf("Failed to determine hostname: %v\n", err) + os.Exit(1) + } + + gateway, err := getGateway(*network) + if err != nil { + fmt.Printf("Failed to read network: %v\n", err) + os.Exit(1) + } + + ca, err := generateCA() + if err != nil { + fmt.Printf("Failed to generate CA: %v\n", err) + os.Exit(1) + } + + server, err := generateServerCert([]string{hostname}, []net.IP{gateway}, ca) + if err != nil { + fmt.Printf("Failed to generate server certificate: %v\n", err) + os.Exit(1) + } + + client, err := generateClientCert(ca) + if err != nil { + fmt.Printf("Failed to generate client certificate: %v\n", err) + os.Exit(1) + } + + if err := writeCertificates(*outDir, ca, server, client); err != nil { + fmt.Printf("Failed to write certificate files: %v\n", err) + os.Exit(1) + } +} diff --git a/installer/pkg/config-generator/generator.go b/installer/pkg/config-generator/generator.go index 68e9278437b..26e224b2fae 100644 --- a/installer/pkg/config-generator/generator.go +++ b/installer/pkg/config-generator/generator.go @@ -9,6 +9,7 @@ import ( "fmt" "io/ioutil" "net" + "net/url" "path/filepath" "strings" @@ -37,8 +38,11 @@ const ( certificatesStrategy = "userProvidedCA" identityAPIService = "tectonic-identity-api.tectonic-system.svc.cluster.local" maoTargetNamespace = "openshift-cluster-api" + libvirtPKIPath = "/etc/pki/libvirt" ) +var errBadLibvirtScheme = errors.New("bad libvirt URI scheme") + // ConfigGenerator defines the cluster config generation for a cluster. type ConfigGenerator struct { config.Cluster @@ -136,9 +140,14 @@ func (c *ConfigGenerator) maoConfig(clusterDir string) (*maoOperatorConfig, erro } case config.PlatformLibvirt: + uri, err := libvirtURI(c.Libvirt.URI, c.Libvirt.IPRange) + if err != nil { + return nil, fmt.Errorf("failed to create libvirt URI: %v", err) + } + cfg.Libvirt = &libvirtConfig{ + URI: uri.String(), ClusterName: c.Name, - URI: c.Libvirt.URI, NetworkName: c.Libvirt.Network.Name, IPRange: c.Libvirt.IPRange, Replicas: c.NodeCount(c.Worker.NodePools), @@ -519,3 +528,41 @@ func tectonicCloudProvider(platform config.Platform) string { } panic("invalid platform") } + +// Returns a libvirt URI given for the machine-api-operator given the +// URI in the config and the cluster network in CIDR format. +func libvirtURI(configURI, networkCIDR string) (*url.URL, error) { + _, ipNet, err := net.ParseCIDR(networkCIDR) + if err != nil { + return nil, err + } + + gateway, err := cidr.Host(ipNet, 1) + if err != nil { + return nil, err + } + + libvirtURI, err := url.Parse(configURI) + if err != nil { + return nil, err + } + + // If there's a transport in the configured URI, replace it with + // TLS. Otherwise, if there's none, explicityly add TLS. + scheme := strings.Split(libvirtURI.Scheme, "+") + switch len(scheme) { + case 1, 2: // Replace or add the transport. + libvirtURI.Scheme = scheme[0] + "+tls" + + default: + return nil, errBadLibvirtScheme + } + + query := libvirtURI.Query() + query.Set("pkipath", libvirtPKIPath) + + libvirtURI.Host = gateway.String() + libvirtURI.RawQuery = query.Encode() + + return libvirtURI, nil +} diff --git a/installer/pkg/config-generator/generator_test.go b/installer/pkg/config-generator/generator_test.go index eb5e67b89fa..63b4e99ed61 100644 --- a/installer/pkg/config-generator/generator_test.go +++ b/installer/pkg/config-generator/generator_test.go @@ -4,6 +4,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "io/ioutil" + "net/url" "os" "testing" @@ -169,3 +170,75 @@ func TestGenerateCert(t *testing.T) { } } } + +func TestLibvirtURI(t *testing.T) { + escapedPKIPath := url.QueryEscape(libvirtPKIPath) + + cases := []struct { + label string + uri string + network string + expected string + err error + }{ + { + label: "defaults", + uri: "qemu:///system", + network: "192.168.124.0/24", + expected: "qemu+tls://192.168.124.1/system?pkipath=" + escapedPKIPath, + err: nil, + }, + { + label: "qemu-localhost", + uri: "qemu://127.0.0.1/system", + network: "192.168.124.0/24", + expected: "qemu+tls://192.168.124.1/system?pkipath=" + escapedPKIPath, + err: nil, + }, + { + label: "custom-network", + uri: "qemu:///system", + network: "172.16.128.0/17", + expected: "qemu+tls://172.16.128.1/system?pkipath=" + escapedPKIPath, + err: nil, + }, + { + label: "preserve-query", + uri: "qemu:///system?foo=bar", + network: "192.168.124.0/24", + expected: "qemu+tls://192.168.124.1/system?foo=bar&pkipath=" + escapedPKIPath, + err: nil, + }, + { + label: "ssh-scheme", + uri: "qemu+ssh://127.0.0.1/system", + network: "192.168.124.0/24", + expected: "qemu+tls://192.168.124.1/system?pkipath=" + escapedPKIPath, + err: nil, + }, + { + label: "bad-scheme", + uri: "qemu+foo+bar:///system", + network: "192.168.124.0/24", + expected: "", + err: errBadLibvirtScheme, + }, + } + + for _, tt := range cases { + t.Run(tt.label, func(t *testing.T) { + uri, err := libvirtURI(tt.uri, tt.network) + if err != tt.err { + t.Errorf("unexpected error: %v", err) + } + + if uri == nil || err != nil { + return + } + + if uri.String() != tt.expected { + t.Errorf("got %q, want %q", uri.String(), tt.expected) + } + }) + } +} diff --git a/modules/bootkube/manifests.tf b/modules/bootkube/manifests.tf index 9c80d1addc9..fbaf149dcb6 100644 --- a/modules/bootkube/manifests.tf +++ b/modules/bootkube/manifests.tf @@ -92,3 +92,32 @@ resource "local_file" "manifest_files" { filename = "./generated/manifests/${var.manifest_names[count.index]}" content = "${data.template_file.manifest_file_list.*.rendered[count.index]}" } + +# Conditionally include the libvirt-certs secret. +data "template_file" "libvirt_certs" { + template = "${file("${path.module}/resources/manifests/libvirt-certs-secret.yaml")}" + + vars { + ca_cert = "${base64encode(var.libvirt_tls_ca_pem)}" + client_cert = "${base64encode(var.libvirt_tls_cert_pem)}" + client_key = "${base64encode(var.libvirt_tls_key_pem)}" + } +} + +data "ignition_file" "libvirt_certs_secret" { + count = "${var.libvirt_tls_cert_pem != "" ? 1 : 0}" + filesystem = "root" + mode = "0644" + + path = "/opt/tectonic/manifests/libvirt-certs-secret.yaml" + + content { + content = "${data.template_file.libvirt_certs.rendered}" + } +} + +resource "local_file" "libvirt_certs" { + count = "${var.libvirt_tls_cert_pem != "" ? 1 : 0}" + filename = "./generated/manifests/libvirt-certs-secret.yaml" + content = "${data.template_file.libvirt_certs.rendered}" +} diff --git a/modules/bootkube/outputs.tf b/modules/bootkube/outputs.tf index 90f9412960b..91c0ca207e9 100644 --- a/modules/bootkube/outputs.tf +++ b/modules/bootkube/outputs.tf @@ -26,5 +26,6 @@ output "ignition_file_id_list" { data.ignition_file.kubeconfig-kubelet.id, ), data.ignition_file.manifest_file_list.*.id, + data.ignition_file.libvirt_certs_secret.*.id, ))}"] } diff --git a/modules/bootkube/resources/manifests/libvirt-certs-secret.yaml b/modules/bootkube/resources/manifests/libvirt-certs-secret.yaml new file mode 100644 index 00000000000..e1658cb753a --- /dev/null +++ b/modules/bootkube/resources/manifests/libvirt-certs-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: libvirt-certs + namespace: openshift-cluster-api +type: Opaque +data: + cacert.pem: ${ca_cert} + clientcert.pem: ${client_cert} + clientkey.pem: ${client_key} diff --git a/modules/bootkube/resources/manifests/machine-api-operator.yaml b/modules/bootkube/resources/manifests/machine-api-operator.yaml index 125b870bc3d..1a83e5adb07 100644 --- a/modules/bootkube/resources/manifests/machine-api-operator.yaml +++ b/modules/bootkube/resources/manifests/machine-api-operator.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: machine-api-operator - image: quay.io/coreos/machine-api-operator:b6a04c2 + image: quay.io/coreos/machine-api-operator:6ce7a77 command: - "/machine-api-operator" resources: diff --git a/modules/bootkube/variables.tf b/modules/bootkube/variables.tf index 0935eda3443..ea10f8be169 100644 --- a/modules/bootkube/variables.tf +++ b/modules/bootkube/variables.tf @@ -166,3 +166,21 @@ variable "worker_ign_config" { type = "string" default = "" } + +variable "libvirt_tls_ca_pem" { + type = "string" + description = "The libvirt CA certificate in PEM format" + default = "" +} + +variable "libvirt_tls_cert_pem" { + type = "string" + description = "The libvirt client certificate in PEM format" + default = "" +} + +variable "libvirt_tls_key_pem" { + type = "string" + description = "The libvirt client private key in PEM format" + default = "" +} diff --git a/pkg/types/config/libvirt/libvirt.go b/pkg/types/config/libvirt/libvirt.go index 3d0f9c80061..56cb209790e 100644 --- a/pkg/types/config/libvirt/libvirt.go +++ b/pkg/types/config/libvirt/libvirt.go @@ -16,12 +16,20 @@ const ( type Libvirt struct { URI string `json:"tectonic_libvirt_uri,omitempty" yaml:"uri"` Image string `json:"tectonic_os_image,omitempty" yaml:"image"` + TLS `json:",inline" yaml:"tls"` Network `json:",inline" yaml:"network"` MasterIPs []string `json:"tectonic_libvirt_master_ips,omitempty" yaml:"masterIPs"` WorkerIPs []string `json:"tectonic_libvirt_worker_ips,omitempty" yaml:"workerIPs"` BootstrapIP string `json:"tectonic_libvirt_bootstrap_ip,omitempty" yaml:"bootstrapIP"` } +// TLS represents paths to TLS assets for libvirt clients. +type TLS struct { + CAPath string `json:"tectonic_libvirt_tls_ca_path,omitempty" yaml:"caPath"` + CertPath string `json:"tectonic_libvirt_tls_cert_path,omitempty" yaml:"certPath"` + KeyPath string `json:"tectonic_libvirt_tls_key_path,omitempty" yaml:"keyPath"` +} + // Network describes a libvirt network configuration. type Network struct { Name string `json:"tectonic_libvirt_network_name,omitempty" yaml:"name"` diff --git a/steps/assets/base/tectonic.tf b/steps/assets/base/tectonic.tf index a111a0e32a4..94b8697e543 100644 --- a/steps/assets/base/tectonic.tf +++ b/steps/assets/base/tectonic.tf @@ -52,6 +52,10 @@ module "bootkube" { etcd_endpoints = "${data.template_file.etcd_hostname_list.*.rendered}" worker_ign_config = "${var.aws_worker_ign_config}" + + libvirt_tls_ca_pem = "${var.libvirt_tls_ca_pem}" + libvirt_tls_cert_pem = "${var.libvirt_tls_cert_pem}" + libvirt_tls_key_pem = "${var.libvirt_tls_key_pem}" } module "tectonic" { diff --git a/steps/assets/base/variables.tf b/steps/assets/base/variables.tf index ef8b9702a5a..8d599523222 100644 --- a/steps/assets/base/variables.tf +++ b/steps/assets/base/variables.tf @@ -12,3 +12,18 @@ variable "aws_worker_ign_config" { type = "string" default = "" } + +variable "libvirt_tls_ca_pem" { + type = "string" + default = "" +} + +variable "libvirt_tls_cert_pem" { + type = "string" + default = "" +} + +variable "libvirt_tls_key_pem" { + type = "string" + default = "" +} diff --git a/steps/assets/libvirt/main.tf b/steps/assets/libvirt/main.tf index 1382a6ecbff..d42017aab08 100644 --- a/steps/assets/libvirt/main.tf +++ b/steps/assets/libvirt/main.tf @@ -1,3 +1,9 @@ +locals { + libvirt_tls_ca_pem = "${file("${var.tectonic_libvirt_tls_ca_path}")}" + libvirt_tls_cert_pem = "${file("${var.tectonic_libvirt_tls_cert_path}")}" + libvirt_tls_key_pem = "${file("${var.tectonic_libvirt_tls_key_path}")}" +} + module assets_base { source = "../base" @@ -20,4 +26,8 @@ module assets_base { tectonic_service_cidr = "${var.tectonic_service_cidr}" tectonic_update_channel = "${var.tectonic_update_channel}" tectonic_versions = "${var.tectonic_versions}" + + libvirt_tls_ca_pem = "${local.libvirt_tls_ca_pem}" + libvirt_tls_cert_pem = "${local.libvirt_tls_cert_pem}" + libvirt_tls_key_pem = "${local.libvirt_tls_key_pem}" } diff --git a/steps/variables-libvirt.tf b/steps/variables-libvirt.tf index 88bed42827e..f0877093fe0 100644 --- a/steps/variables-libvirt.tf +++ b/steps/variables-libvirt.tf @@ -3,6 +3,21 @@ variable "tectonic_libvirt_uri" { description = "libvirt connection URI" } +variable "tectonic_libvirt_tls_ca_path" { + type = "string" + description = "path to the libvirt CA certificate" +} + +variable "tectonic_libvirt_tls_cert_path" { + type = "string" + description = "path to the libvirt client certificate" +} + +variable "tectonic_libvirt_tls_key_path" { + type = "string" + description = "path to the libvirt client private key" +} + variable "tectonic_libvirt_network_name" { type = "string" description = "Name of the libvirt network to create"