diff --git a/hack/generate-placeholder-crl.sh b/hack/generate-placeholder-crl.sh new file mode 100755 index 000000000..be230cc3f --- /dev/null +++ b/hack/generate-placeholder-crl.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Generate a placeholder CRL, like the one included in pkg/router/crl/crl.go as dummyCRL. In order to create an already +# expired CRL, this script requires the utility faketime, which is provided by libfaketime on fedora. +set -e +cwd=$(dirname $0) +tmpdir=$(mktemp -d) +sed -e "s@%%tmpdir%%@${tmpdir}@" ${cwd}/placeholder-ca.cnf.template > ${tmpdir}/placeholder-ca.cnf +openssl genrsa -out ${tmpdir}/placeholder-ca.key 2048 +openssl req -new -key ${tmpdir}/placeholder-ca.key -out ${tmpdir}/placeholder-ca.csr -subj "/C=US/ST=NC/L=Raleigh/O=OS4/OU=Eng/CN=Placeholder CA" +openssl x509 -req -in ${tmpdir}/placeholder-ca.csr -out ${tmpdir}/placeholder-ca.crt -days 3650 -signkey ${tmpdir}/placeholder-ca.key -extfile ${tmpdir}/placeholder-ca.cnf +touch ${tmpdir}/placeholder-crl-index.txt +faketime 'Jan 1, 2000 12:00AM GMT' openssl ca -gencrl -crlhours 1 -out ${tmpdir}/placeholder-ca.crl -config ${tmpdir}/placeholder-ca.cnf + +echo "new placeholder crl at ${tmpdir}/placeholder-ca.crl" >&2 +cat ${tmpdir}/placeholder-ca.crl diff --git a/hack/placeholder-ca.cnf.template b/hack/placeholder-ca.cnf.template new file mode 100644 index 000000000..12a520003 --- /dev/null +++ b/hack/placeholder-ca.cnf.template @@ -0,0 +1,18 @@ +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid, issuer +basicConstraints=CA:TRUE + +[ca] +default_ca=placeholder_ca + +[placeholder_ca] +authorityKeyIdentifier=keyid,issuer +certificate=%%tmpdir%%/placeholder-ca.crt +database=%%tmpdir%%/placeholder-crl-index.txt +default_crl_hours=1 +default_md=sha256 +private_key=%%tmpdir%%/placeholder-ca.key +crl_extensions=crl_exts + +[crl_exts] +authorityKeyIdentifier=keyid, issuer diff --git a/images/router/haproxy/Dockerfile b/images/router/haproxy/Dockerfile index a765bcfbe..cb149ee47 100644 --- a/images/router/haproxy/Dockerfile +++ b/images/router/haproxy/Dockerfile @@ -4,7 +4,7 @@ RUN INSTALL_PKGS="haproxy22 rsyslog sysvinit-tools" && \ rpm -V $INSTALL_PKGS && \ yum clean all && \ mkdir -p /var/lib/haproxy/router/{certs,cacerts,whitelists} && \ - mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log} && \ + mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log,mtls} && \ touch /var/lib/haproxy/conf/{{os_http_be,os_edge_reencrypt_be,os_tcp_be,os_sni_passthrough,os_route_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \ setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy && \ chown -R :0 /var/lib/haproxy && \ diff --git a/images/router/haproxy/Dockerfile.rhel b/images/router/haproxy/Dockerfile.rhel index 4957025e0..9b8ffe01c 100644 --- a/images/router/haproxy/Dockerfile.rhel +++ b/images/router/haproxy/Dockerfile.rhel @@ -4,7 +4,7 @@ RUN INSTALL_PKGS="haproxy22 rsyslog sysvinit-tools" && \ rpm -V $INSTALL_PKGS && \ yum clean all && \ mkdir -p /var/lib/haproxy/router/{certs,cacerts,whitelists} && \ - mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log} && \ + mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log,mtls} && \ touch /var/lib/haproxy/conf/{{os_http_be,os_edge_reencrypt_be,os_tcp_be,os_sni_passthrough,os_route_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \ setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy && \ chown -R :0 /var/lib/haproxy && \ diff --git a/images/router/haproxy/Dockerfile.rhel8 b/images/router/haproxy/Dockerfile.rhel8 index 4dbe2445a..f1e1fdeb9 100644 --- a/images/router/haproxy/Dockerfile.rhel8 +++ b/images/router/haproxy/Dockerfile.rhel8 @@ -4,7 +4,7 @@ RUN INSTALL_PKGS="haproxy22 rsyslog procps-ng util-linux" && \ rpm -V $INSTALL_PKGS && \ yum clean all && \ mkdir -p /var/lib/haproxy/router/{certs,cacerts,whitelists} && \ - mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log} && \ + mkdir -p /var/lib/haproxy/{conf/.tmp,run,bin,log,mtls} && \ touch /var/lib/haproxy/conf/{{os_http_be,os_edge_reencrypt_be,os_tcp_be,os_sni_passthrough,os_route_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \ setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy && \ chown -R :0 /var/lib/haproxy && \ diff --git a/images/router/haproxy/conf/haproxy-config.template b/images/router/haproxy/conf/haproxy-config.template index 618ee89dc..b1612c96d 100644 --- a/images/router/haproxy/conf/haproxy-config.template +++ b/images/router/haproxy/conf/haproxy-config.template @@ -9,6 +9,8 @@ {{- $dynamicConfigManager := .DynamicConfigManager }} {{- $router_ip_v4_v6_mode := env "ROUTER_IP_V4_V6_MODE" "v4" }} {{- $router_disable_http2 := env "ROUTER_DISABLE_HTTP2" "false" }} +{{- $haveClientCA := .HaveClientCA }} +{{- $haveCRLs := .HaveCRLs }} {{- /* A bunch of regular expressions. Each should be wrapped in (?:) so that it is safe to include bare */}} @@ -290,8 +292,13 @@ frontend fe_sni {{- "" }} crt-list /var/lib/haproxy/conf/cert_config.map accept-proxy {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} {{- "" }} verify {{. }} - {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{. }} {{ else }} ca-file /etc/ssl/certs/ca-bundle.trust.crt {{ end }} - {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{. }} {{ end }} + {{- if (ne (env "ROUTER_MUTUAL_TLS_AUTH_CRL") "") }} + {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{. }} {{ else }} ca-file /etc/ssl/certs/ca-bundle.trust.crt {{ end }} + {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{. }} {{ end }} + {{- else }} + {{- if $haveClientCA }} ca-file /var/lib/haproxy/mtls/latest/ca-bundle.pem {{ else }} ca-file /etc/ssl/certs/ca-bundle.trust.crt {{ end }} + {{- if $haveCRLs }} crl-file /var/lib/haproxy/mtls/latest/crls.pem {{ end }} + {{- end }} {{- end }} mode http @@ -376,8 +383,13 @@ frontend fe_no_sni bind unix@/var/lib/haproxy/run/haproxy-no-sni.sock ssl crt {{ firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem" }} accept-proxy {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} {{- "" }} verify {{. }} - {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{. }} {{ else }} ca-file /etc/ssl/certs/ca-bundle.trust.crt {{ end }} - {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{. }} {{ end }} + {{- if (ne (env "ROUTER_MUTUAL_TLS_AUTH_CRL") "") }} + {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{. }} {{ else }} ca-file /etc/ssl/certs/ca-bundle.trust.crt {{ end }} + {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{. }} {{ end }} + {{- else }} + {{- if $haveClientCA }} ca-file /var/lib/haproxy/mtls/latest/ca-bundle.pem {{ else }} ca-file /etc/ssl/certs/ca-bundle.trust.crt {{ end }} + {{- if $haveCRLs }} crl-file /var/lib/haproxy/mtls/latest/crls.pem {{ end }} + {{- end }} {{- end }} mode http diff --git a/pkg/router/crl/crl.go b/pkg/router/crl/crl.go new file mode 100644 index 000000000..281800bad --- /dev/null +++ b/pkg/router/crl/crl.go @@ -0,0 +1,508 @@ +package crl + +import ( + "bytes" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/hex" + "encoding/pem" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + logf "github.com/openshift/router/log" + "github.com/openshift/router/pkg/util" + kerrors "k8s.io/apimachinery/pkg/util/errors" +) + +var log = logf.Logger.WithName("crl") + +const ( + // crlFilePermissions is the permission bits used for the CRL file. + crlFilePermissions = 0644 + // stagingDirPermissions is the permission bits used for staging directories + stagingDirPermissions = 0755 + // crlFallbackTime is how long to wait before retrying if nextUpdate can't be determined. + crlFallbackTime = 5 * time.Minute + // errorBackoffTime is how long to wait before retrying if a generic error happens during CRL refresh. + errorBackoffTime = 5 * time.Minute + // mtlsBaseDirectory is the directory where all crl temp directories and symlinks will live. + mtlsBaseDirectory = "/var/lib/haproxy/mtls" + // crlBasename is the name of the crl file + crlBasename = "crls.pem" + // caBundleBasename is the name of the CA bundle file copied from the CA bundle configmap. + caBundleBasename = "ca-bundle.pem" + // dummyCRL is a placeholder CRL so that HAProxy can start serving non-mTLS traffic while CRLs are downloaded. This + // CRL is for a CA cert/key that are intentionally not included, and was generated with an expiration (nextUpdate) + // at 1:00AM GMT on Jan 1, 2000 so that in the extremely unlikely case that someone is able to generate a matching + // cert and key, HAProxy will still reject the connection due to the expired CRL. + dummyCRL = `-----BEGIN X509 CRL----- +MIIBzzCBuAIBATANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJVUzELMAkGA1UE +CAwCTkMxEDAOBgNVBAcMB1JhbGVpZ2gxDDAKBgNVBAoMA09TNDEMMAoGA1UECwwD +RW5nMRcwFQYDVQQDDA5QbGFjZWhvbGRlciBDQRcNMDAwMTAxMDAwMDAwWhcNMDAw +MTAxMDEwMDAwWqAjMCEwHwYDVR0jBBgwFoAUbaqyU8VAswFgefsu6pOdvqK1nfgw +DQYJKoZIhvcNAQELBQADggEBAD8W+OmWHp9Pg7914rA5QOk+pUCZ4F7++fbmGPpc +9gdNOxkeCrZ3sdBeEs0P3+tSf8dLcpI5PKEbL+bC3wrIM3yzsD+mIZkvV/FGhgE1 +s7b6IA/8FYsmNWIjgAWBAp13zh0AH3qhpI01tm+cQETz6r249TWQ+p04pEA89+XT +7CE99nHd8yNDOESs1xZreSFkIF/Hmm8y4I0o/+8wpjA9e3PJ7O25ZB2OGX4FufMf +tVa0xfWd9czWFqM1DjU3ME0mVi6lr38AhUDoG6sFbHk+TfzTp4ykVUpXIHu4bJTG +DPfV3SE277EvsrsGFYIsxWgXskITjzb9no9fnodd/jG46tw= +-----END X509 CRL----- +` +) + +var ( + // mtlsLatestSymlink is the fully qualified path to the symlink used in the template. + mtlsLatestSymlink = filepath.Join(mtlsBaseDirectory, "latest") + // mtlsNextSymlink is the fully qualified path to the staging symlink, used to atomically replace the existing + // mtlsLatestSymlink. + mtlsNextSymlink = filepath.Join(mtlsBaseDirectory, "next") + // CRLFilename is the fully qualified path to the currently in use crl file. + CRLFilename = filepath.Join(mtlsLatestSymlink, crlBasename) + // CABundleFilename is the fully qualified path to the currently in use CA bundle. + CABundleFilename = filepath.Join(mtlsLatestSymlink, caBundleBasename) +) + +// authorityKeyIdentifier is a certificate's authority key identifier. +type authorityKeyIdentifier struct { + KeyIdentifier []byte `asn1:"optional,tag:0"` +} + +// authorityKeyIdentifierOID is the ASN.1 object identifier for the authority key identifier extension. +var authorityKeyIdentifierOID = asn1.ObjectIdentifier{2, 5, 29, 35} + +// InitMTLSDirectory creates an initial directory for HAProxy to use to complete startup and serve non-mTLS traffic +// while CRLs are being downloaded in the background. Returns an error if any of the filesystem operations fail. +func InitMTLSDirectory(caBundleFilename string) error { + stagingDirectory, err := makeStagingDirectory() + if err != nil { + return err + } + // Copy the CA bundle from the configmap mount location to the staging directory + if err := util.CopyFile(caBundleFilename, filepath.Join(stagingDirectory, caBundleBasename)); err != nil { + return err + } + // Write out the dummyCRL as a placeholder. With this, HAProxy will reject connections that require CRLs until after + // all CRL downloads are complete, but other traffic should be handled correctly. + if err := os.WriteFile(filepath.Join(stagingDirectory, crlBasename), []byte(dummyCRL), crlFilePermissions); err != nil { + return err + } + // At any time other than startup, we need to be careful overwrite the previous mtlsLatestSymlink in an atomic way, + // but at initialization, mtlsLatestSymlink shouldn't exist yet, so it should be safe to just directly create the + // symlink. + if err := os.Symlink(stagingDirectory, mtlsLatestSymlink); err != nil { + return err + } + return nil +} + +// CABundleHasCRLs returns true if any of the certificates in caBundleFilename specify a CRL distribution point. +// Returns an error if the CA Bundle could not be parsed. +func CABundleHasCRLs(caBundleFilename string) (bool, error) { + clientCAData, err := os.ReadFile(caBundleFilename) + if err != nil { + return false, err + } + for len(clientCAData) > 0 { + block, data := pem.Decode(clientCAData) + if block == nil { + break + } + clientCAData = data + if block.Type != "CERTIFICATE" { + log.Info("found non-certificate data in client CA bundle. skipping.", "type", block.Type) + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return false, fmt.Errorf("client CA bundle has an invalid certificate: %w", err) + } + if len(cert.CRLDistributionPoints) != 0 { + return true, nil + } + } + return false, nil +} + +// ManageCRLs spins off a goroutine that ensures that any CRLs specified in caBundleFilename are downloaded and kept +// up-to-date. It will automatically refresh expired CRLs and download missing CRLs when it receives a message on +// caUpdateChannel (indicating the CA bundle has been updated), or when any existing CRL expires. Whenever either the CA +// bundle or the CRL file has changed, updateCallback is called, with a boolean indicating whether crl-file needs to be +// specified in the HAProxy config. +func ManageCRLs(caBundleFilename string, caUpdateChannel <-chan struct{}, updateCallback func(bool)) { + go func() { + caUpdated := false + nextUpdate := time.Now() + shouldHaveCRLs, err := CABundleHasCRLs(caBundleFilename) + if err != nil { + log.Error(err, "failed to parse CA bundle", "CA bundle filename", caBundleFilename) + nextUpdate = time.Now().Add(errorBackoffTime) + } + for { + updated := false + if nextUpdate.IsZero() { + log.V(4).Info("no nextUpdate. only watching for CA updates") + select { + case <-caUpdateChannel: + caUpdated = true + } + } else { + log.V(4).Info("nextUpdate is at " + nextUpdate.Format(time.RFC3339)) + select { + case <-time.After(time.Until(nextUpdate)): + case <-caUpdateChannel: + caUpdated = true + } + } + + if caUpdated { + shouldHaveCRLs, err = CABundleHasCRLs(caBundleFilename) + if err != nil { + log.Error(err, "failed to parse CA bundle", "CA bundle filename", caBundleFilename) + nextUpdate = time.Now().Add(errorBackoffTime) + continue + } + } + + nextUpdate, updated, err = updateCRLFile(caBundleFilename, caUpdated) + if err != nil { + log.Error(err, "failed to update CRLs") + nextUpdate = time.Now().Add(errorBackoffTime) + continue + } + // After successfully updating the CRL file, reset caUpdated + caUpdated = false + if updated { + updateCallback(shouldHaveCRLs) + } + } + }() +} + +// updateCRLFile creates a new staging directory, updates CRLs, and updates mtlsLatestSymlink to point to the new +// staging directory. Returns the next update time and a boolean for if anything changed. Returns an error if there was +// an issue during the update. +func updateCRLFile(caBundleFilename string, caUpdated bool) (time.Time, bool, error) { + stagingDirectory, err := makeStagingDirectory() + if err != nil { + log.Error(err, "failed to create staging directory") + return time.Time{}, false, err + } + + defer reapStaleDirectories() + + stagingCRLFilename := filepath.Join(stagingDirectory, crlBasename) + + nextUpdate, crlsUpdated, err := writeCRLFile(caBundleFilename, CRLFilename, stagingCRLFilename) + if err != nil { + log.Error(err, "failed to update CRLs") + return time.Time{}, false, err + } + + if caUpdated || crlsUpdated { + if err := commitCACRLUpdate(stagingDirectory, caBundleFilename, crlsUpdated); err != nil { + log.Error(err, "failed to commit CRL update") + return time.Time{}, false, err + } + return nextUpdate, true, nil + } + return nextUpdate, false, nil +} + +// commitCACRLUpdate makes sure stagingDirectory contains up-to-date versions of both the CA bundle and the CRL file, +// then updates the symlink so that HAProxy will reference the new versions on reload. Returns an error if any of the +// file operations fail. +func commitCACRLUpdate(stagingDirectory, caBundleFilename string, stagingCRLUpdated bool) error { + // Copy CA bundle to the new directory. + if err := util.CopyFile(caBundleFilename, filepath.Join(stagingDirectory, caBundleBasename)); err != nil { + return err + } + // If stagingCRLUpdated is true, then the CRL file has already been written to the new directory, so nothing needs + // to be done for it. However, if stagingCRLUpdated is false, then we need to copy the existing crl file to the new + // directory. + if !stagingCRLUpdated { + if err := util.CopyFile(CRLFilename, filepath.Join(stagingDirectory, crlBasename)); err != nil { + if os.IsNotExist(err) { + // Even if CRLFilename doesn't currently exist, a crl file may need to be supplied to HAProxy when this + // update is committed. Using the dummyCRL allows http traffic to be handled as normal, but essentially + // guarantees that any mTLS connections will fail until new CRLs are downloaded. + if err := os.WriteFile(filepath.Join(stagingDirectory, crlBasename), []byte(dummyCRL), crlFilePermissions); err != nil { + return err + } + } else { + return err + } + } + } + + // os.Symlink() will return an error if a file exists with the same name, so to avoid that, create a symlink with a + // temporary name, then use os.Rename() to replace any existing file with the name we actually want. + + // Remove the staging symlink if it exists. It should not exist at all if the last commitCACRLUpdate call was + // successful, and should never be referred to outside of this function, so it should be safe to remove. + if err := os.Remove(mtlsNextSymlink); err != nil && !os.IsNotExist(err) { + // Successfully removing mtlsNextSymlink and receiving ErrNotExist are both acceptible; we just need + // mtlsNextSymlink to not be there. However, any other error should be returned. + return err + } + if err := os.Symlink(stagingDirectory, mtlsNextSymlink); err != nil { + return err + } + if err := os.Rename(mtlsNextSymlink, mtlsLatestSymlink); err != nil { + return err + } + return nil +} + +var existingCRLs map[string]*pkix.CertificateList + +// writeCRLFile reads the CA bundle at caBundleFilename, and makes sure all CRLs specified in the CA bundle are written +// into the crl file at newCRLFilename. If any of the specified CRLs are in existingCRLFilename and have not expired, +// writeCRLFile will prefer to use those over downloading them again from their distribution points. +// +// Returns the time of the next CRL expiration (zero if no CRLs are in use), and whether or not the CRL file was +// updated. Returns an error if parsing data, encoding data, or a file operation fails. +func writeCRLFile(caBundleFilename, existingCRLFilename, newCRLFilename string) (time.Time, bool, error) { + clientCAData, err := os.ReadFile(caBundleFilename) + if err != nil { + return time.Time{}, false, err + } + + crls, nextCRLUpdate, updated, err := downloadMissingCRLs(existingCRLs, clientCAData) + if err != nil { + return time.Time{}, false, err + } + + existingCRLs = crls + + if len(crls) == 0 { + // If there are no CRLs, still write out dummyCRL as a placeholder. + return time.Time{}, updated, os.WriteFile(newCRLFilename, []byte(dummyCRL), crlFilePermissions) + } + + // If any CRLs changed, encode the CRLs and write to newCRLFilename. + if !updated { + return nextCRLUpdate, updated, nil + } + + buf := &bytes.Buffer{} + for subjectKeyId, crl := range crls { + asn1Data, err := asn1.Marshal(*crl) + if err != nil { + return time.Time{}, false, fmt.Errorf("failed to encode ASN.1 for CRL for certificate key %s: %w", subjectKeyId, err) + } + block := &pem.Block{ + Type: "X509 CRL", + Bytes: asn1Data, + } + if err := pem.Encode(buf, block); err != nil { + return time.Time{}, false, fmt.Errorf("failed to encode PEM for CRL for certificate key %s: %w", subjectKeyId, err) + } + } + + if err := os.WriteFile(newCRLFilename, buf.Bytes(), crlFilePermissions); err != nil { + return time.Time{}, false, err + } + + return nextCRLUpdate, updated, nil +} + +// downloadMissingCRLs parses the certificates in the CA bundle, clientCAData, and returns a map of all CRLs that were +// specified. downloadMissingCRLs will prefer to use CRLs from existingCRLs if they are still valid, but otherwise, CRLs +// are downloaded from the distribution points from the CA bundle. +// +// Returns: +// - a map of all CRLs keyed by their subject key ID +// - the time at which the next CRL will expire, or a fallback time if any CRLs have already expired. If clientCAData +// specifies no CRL distribution points, this time will be zero. +// - whether the crl map has been updated, either because new CRLs were downloaded, or because some CRLs in +// existingCRLs are no longer required +// +// Returns an error if CRL downloading or parsing fails. +func downloadMissingCRLs(existingCRLs map[string]*pkix.CertificateList, clientCAData []byte) (map[string]*pkix.CertificateList, time.Time, bool, error) { + var nextCRLUpdate time.Time + crls := make(map[string]*pkix.CertificateList) + updated := false + now := time.Now() + for len(clientCAData) > 0 { + block, data := pem.Decode(clientCAData) + if block == nil { + break + } + clientCAData = data + if block.Type != "CERTIFICATE" { + log.Info("found non-certificate data in client CA bundle. skipping.", "type", block.Type) + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, time.Time{}, false, fmt.Errorf("client CA bundle has an invalid certificate: %w", err) + } + subjectKeyId := hex.EncodeToString(cert.SubjectKeyId) + if len(cert.CRLDistributionPoints) == 0 { + continue + } + if crl, ok := existingCRLs[subjectKeyId]; ok { + if crl.TBSCertList.NextUpdate.Before(now) { + log.Info("certificate revocation list has expired", "subject key identifier", subjectKeyId, "next update", crl.TBSCertList.NextUpdate.Format(time.RFC3339)) + } else { + crls[subjectKeyId] = existingCRLs[subjectKeyId] + if nextCRLUpdate.IsZero() || crl.TBSCertList.NextUpdate.Before(nextCRLUpdate) { + nextCRLUpdate = crl.TBSCertList.NextUpdate + } + continue + } + } + log.Info("retrieving certificate revocation list", "subject key identifier", subjectKeyId) + if crl, err := getCRL(cert.CRLDistributionPoints, now); err != nil { + // Creating or updating the crl file with incomplete data would compromise security by potentially + // permitting revoked certificates. + return nil, time.Time{}, false, fmt.Errorf("failed to get certificate revocation list for certificate key %s: %w", subjectKeyId, err) + } else { + crls[subjectKeyId] = crl + log.Info("new certificate revocation list", "subject key identifier", subjectKeyId, "next update", crl.TBSCertList.NextUpdate.Format(time.RFC3339)) + if nextCRLUpdate.IsZero() || crl.TBSCertList.NextUpdate.Before(nextCRLUpdate) { + nextCRLUpdate = crl.TBSCertList.NextUpdate + } + updated = true + } + } + // If updated is still false, no new CRLs have been downloaded, but it's possible that some existing CRLs are no + // longer necessary. If that's the case, then existingCRLs will contain more items than crls, so we can compare + // their lengths to determine if an update is necessary. + updated = updated || (len(existingCRLs) != len(crls)) + // If nextCRLUpdate is non-zero but is still in the past, that means at least one CRL has already expired, but a + // non-expired version wasn't able to be downloaded. If that's the case, use the fallback time for the nextCRLUpdate + // time instead. + if !nextCRLUpdate.IsZero() && nextCRLUpdate.Before(now) { + nextCRLUpdate = now.Add(crlFallbackTime) + } + return crls, nextCRLUpdate, updated, nil +} + +// getCRL gets a certificate revocation list using the provided distribution points and returns the certificate list. +// Returns an error if the CRL could not be downloaded. +func getCRL(distributionPoints []string, now time.Time) (*pkix.CertificateList, error) { + var errs []error + for _, distributionPoint := range distributionPoints { + // The distribution point is typically a URL with the "http" scheme. "https" is generally not used because the + // certificate list is signed, and because using TLS to get the certificate list could introduce a circular + // dependency (cannot use TLS without the revocation list, and cannot get the revocation list without using + // TLS). + // + // TODO Support ldap. + switch { + case strings.HasPrefix(distributionPoint, "http:"): + log.Info("retrieving CRL distribution point", "distribution point", distributionPoint) + crl, err := getHTTPCRL(distributionPoint) + if err != nil { + errs = append(errs, fmt.Errorf("error getting %q: %w", distributionPoint, err)) + continue + } + if crl.TBSCertList.NextUpdate.Before(now) { + log.Info("CRL expired. trying next distribution point", "nextUpdate", crl.TBSCertList.NextUpdate.Format(time.RFC3339)) + errs = append(errs, fmt.Errorf("retrieved expired CRL from %s", distributionPoint)) + continue + } + return crl, nil + default: + errs = append(errs, fmt.Errorf("unsupported distribution point type: %s", distributionPoint)) + } + } + log.Info("failed to get valid CRL after trying all distribution points") + return nil, kerrors.NewAggregate(errs) +} + +// getHTTPCRL gets a certificate revocation list using the provided HTTP URL. Returns an error if the CRL could not be +// downloaded, or if parsing the CRL fails. +func getHTTPCRL(url string) (*pkix.CertificateList, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("http.Get failed: %w", err) + } + defer resp.Body.Close() + // If http.Get returned anything other than 200 OK, we can't rely on the response body to actually be a CRL. Return + // an error with the status code rather than failing at the parsing stage. + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("got unexpected status %s", resp.Status) + } + crlBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response: %w", err) + } + // Try to decode the CRL from PEM to DER. If pemBlock comes back nil, assume the CRL was already in DER format, and + // try to parse anyway. + pemBlock, _ := pem.Decode(crlBytes) + if pemBlock != nil { + if pemBlock.Type == "X509 CRL" { + crlBytes = pemBlock.Bytes + } else { + return nil, fmt.Errorf("error parsing response: file is not CRL type") + } + } + crl, err := x509.ParseCRL(crlBytes) + if err != nil { + return nil, fmt.Errorf("error parsing response: %w", err) + } + return crl, nil +} + +// reapStaleDirectories deletes any subdirectories of mtlsBaseDirectory that are no longer necessary +func reapStaleDirectories() { + log.V(4).Info("cleaning up stale mtls files...") + // list all entries in the base directory. + dirEntries, err := os.ReadDir(mtlsBaseDirectory) + if err != nil { + log.Error(err, "Failed to read directory", "directory", mtlsBaseDirectory) + return + } + // The base directory should only contain 2 entries: the latest mTLS files, and the "latest" symlink. If it only + // contains 2 items, assume it's those two, so nothing needs to be done. + if len(dirEntries) <= 2 { + log.V(2).Info("no stale directories") + return + } + // Get the name of the directory that mtlsLatestSymlink points to so we don't delete it. If os.Readlink() returns an + // error, it'll be because mtlsLatestSymlink doesn't exist. That shouldn't happen, but if it does, we can't + // determine what we need to keep, so just exit and hope it's fixed in a future run. + inUseDirName, err := os.Readlink(mtlsLatestSymlink) + if err != nil { + log.V(1).Error(err, "failed to read symlink to determine which files to clean", "name", mtlsLatestSymlink) + return + } + // Walk through all entries in mtlsBaseDirectory. Only 2 things need to stay: mtlsLatestSymlink, and the directory + // that mtlsLatestSymlink points to. Everything else can be deleted. + for _, dirEntry := range dirEntries { + entryName := dirEntry.Name() + // entryName is the bare file/directory name, but mtlsLatestSymlink and inUseDirName are fully-qualified paths. + // Since we know that all the directory entries are the files/directories in mtlsBaseDirectory, we can prepend + // that to entryName to get the full name to compare against. + fullPath := filepath.Join(mtlsBaseDirectory, entryName) + log.V(4).Info("checking for staleness", "filename", fullPath) + // Leave mtlsLatestSymlink and the current directory pointed to by mtlsLatestSymlink alone. + if fullPath == mtlsLatestSymlink || fullPath == inUseDirName { + continue + } + log.V(4).Info("removing stale file", "path", fullPath) + if err := os.RemoveAll(fullPath); err != nil { + log.Error(err, "failed to remove stale file or directory", "path", fullPath) + } + } + log.V(4).Info("cleanup done") +} + +// makeStagingDirectory creates a new staging directory, with the name format +// mtls-[year][month][day]-[hour][minute][second], so that directories are unique but still sortable for debugging +// purposes. The exact directory name is returned. Returns an error if the directory could not be created. +func makeStagingDirectory() (string, error) { + stagingDirName := filepath.Join(mtlsBaseDirectory, fmt.Sprintf("mtls-%s", time.Now().Format("20060102-150405"))) + if err := os.MkdirAll(stagingDirName, stagingDirPermissions); err != nil { + return "", err + } + return stagingDirName, nil +} diff --git a/pkg/router/template/router.go b/pkg/router/template/router.go index eb9ca11d7..a2c490c29 100644 --- a/pkg/router/template/router.go +++ b/pkg/router/template/router.go @@ -24,6 +24,7 @@ import ( routev1 "github.com/openshift/api/route/v1" logf "github.com/openshift/router/log" + "github.com/openshift/router/pkg/router/crl" "github.com/openshift/router/pkg/router/template/limiter" ) @@ -119,6 +120,10 @@ type templateRouter struct { captureHTTPCookie *CaptureHTTPCookie // httpHeaderNameCaseAdjustments specifies HTTP header name case adjustments. httpHeaderNameCaseAdjustments []HTTPHeaderNameCaseAdjustment + // haveClientCA specifies if the user provided their own CA for client auth in mTLS + haveClientCA bool + // haveCRLs specifies if the crl file has been generated for client auth + haveCRLs bool } // templateRouterCfg holds all configuration items required to initialize the template router @@ -183,6 +188,10 @@ type templateData struct { // HTTPHeaderNameCaseAdjustments specifies HTTP header name adjustments // performed on HTTP headers. HTTPHeaderNameCaseAdjustments []HTTPHeaderNameCaseAdjustment + // HaveClientCA specifies if the user provided their own CA for client auth in mTLS + HaveClientCA bool + // HaveCRLs specifies if the crl file is present + HaveCRLs bool } func newTemplateRouter(cfg templateRouterCfg) (*templateRouter, error) { @@ -430,23 +439,29 @@ func (r *templateRouter) writeDefaultCert() error { func (r *templateRouter) watchMutualTLSCert() error { caPath := os.Getenv("ROUTER_MUTUAL_TLS_AUTH_CA") if len(caPath) != 0 { - reloadFn := func() { - log.V(0).Info("reloading to get updated client CA", "name", caPath) - r.rateLimitedCommitFunction.RegisterChange() + r.haveClientCA = true + if err := crl.InitMTLSDirectory(caPath); err != nil { + return err } - if err := r.watchVolumeMountDir(filepath.Dir(caPath), reloadFn); err != nil { - log.V(0).Info("failed to establish watch on mTLS certificate directory", "error", err) - return nil + haveCRLs, err := crl.CABundleHasCRLs(caPath) + if err != nil { + log.V(0).Error(err, "failed to parse CA Bundle", "path", caPath) + return err } - } - crlPath := os.Getenv("ROUTER_MUTUAL_TLS_AUTH_CRL") - if len(crlPath) != 0 && filepath.Dir(caPath) != filepath.Dir(crlPath) { - reloadFn := func() { - log.V(0).Info("reloading to get updated client CA CRL", "name", crlPath) + r.haveCRLs = haveCRLs + caUpdateChannel := make(chan struct{}) + crlReloadFn := func(haveCRLs bool) { + r.haveCRLs = haveCRLs + log.V(0).Info("reloading to get updated client CA CRL", "name", crl.CRLFilename, "have CRLs", haveCRLs) r.rateLimitedCommitFunction.RegisterChange() } - if err := r.watchVolumeMountDir(filepath.Dir(crlPath), reloadFn); err != nil { - log.V(0).Info("failed to establish watch on mTLS certificate directory", "error", err) + crl.ManageCRLs(caPath, caUpdateChannel, crlReloadFn) + caReloadFn := func() { + // Send signal to CRL management goroutine that client CA has been changed + caUpdateChannel <- struct{}{} + } + if err := r.watchVolumeMountDir(filepath.Dir(caPath), caReloadFn); err != nil { + log.V(0).Error(err, "failed to establish watch on mTLS certificate directory") return nil } } @@ -582,6 +597,8 @@ func (r *templateRouter) writeConfig() error { CaptureHTTPResponseHeaders: r.captureHTTPResponseHeaders, CaptureHTTPCookie: r.captureHTTPCookie, HTTPHeaderNameCaseAdjustments: r.httpHeaderNameCaseAdjustments, + HaveClientCA: r.haveClientCA, + HaveCRLs: r.haveCRLs, } if err := template.Execute(file, data); err != nil { file.Close() diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..040fa7c64 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,34 @@ +package util + +import ( + "os" + + logf "github.com/openshift/router/log" +) + +var log = logf.Logger.WithName("util") + +// CopyFile copies a file from src to dest. It returns an error on failure +func CopyFile(src, dest string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { + if err := in.Close(); err != nil { + log.Error(err, "Failed to close input file", "filename", src) + } + }() + out, err := os.Create(dest) + if err != nil { + return err + } + _, err = out.ReadFrom(in) + if err != nil { + if closeErr := out.Close(); closeErr != nil { + log.Error(closeErr, "Failed to close output file", "filename", dest) + } + return err + } + return out.Close() +}