diff --git a/manifests/00-cluster-role.yaml b/manifests/00-cluster-role.yaml index f514a00ef8..209b221125 100644 --- a/manifests/00-cluster-role.yaml +++ b/manifests/00-cluster-role.yaml @@ -100,6 +100,7 @@ rules: - dnses - apiservers - networks + - proxies verbs: - get @@ -109,6 +110,7 @@ rules: - dnses - infrastructures - ingresses + - proxies verbs: - list - watch diff --git a/pkg/manifests/bindata.go b/pkg/manifests/bindata.go index 59de27edb0..028a1c17e8 100644 --- a/pkg/manifests/bindata.go +++ b/pkg/manifests/bindata.go @@ -15,7 +15,7 @@ // assets/router/service-account.yaml (213B) // assets/router/service-cloud.yaml (631B) // assets/router/service-internal.yaml (432B) -// manifests/00-cluster-role.yaml (3.181kB) +// manifests/00-cluster-role.yaml (3.205kB) // manifests/00-custom-resource-definition-internal.yaml (7.756kB) // manifests/00-custom-resource-definition.yaml (121.33kB) // manifests/00-ingress-credentials-request.yaml (4.824kB) @@ -402,7 +402,7 @@ func assetsRouterServiceInternalYaml() (*asset, error) { return a, nil } -var _manifests00ClusterRoleYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\x4d\x8f\xe3\x36\x0c\xbd\xfb\x57\x08\x33\x87\x05\x16\xb0\x83\xde\x8a\xdc\x8a\x16\xe8\xa9\x5d\xa0\x28\x7a\x67\x24\x26\x66\x47\x16\x0d\x92\xca\xac\xfb\xeb\x0b\xf9\x63\xb2\x93\xef\xd9\xcd\x29\x91\x4c\x3e\x3e\xf2\x91\x84\x9e\xdd\xaf\x31\xab\xa1\x38\xe1\x88\x6e\xcb\xe2\xac\x45\xc7\x3d\x0a\x18\x8b\x23\x53\x8c\xdb\xa6\x7a\x76\x7f\x7f\xf9\xed\xcb\xda\xfd\xe2\x22\x9b\xe3\x6d\xb1\x52\x74\xda\x72\x8e\xc1\x6d\xd0\x09\xf6\x11\x3c\x06\xb7\x19\x46\x28\x75\x94\x46\xa8\x04\x1d\x6a\x0f\x1e\x75\x44\x7f\x6d\xc9\xb7\xd5\xf3\xfb\x28\xe0\x2d\x43\x8c\x83\x4b\x88\x41\x1d\x78\x8f\xaa\x4d\xf5\x42\x29\xac\x17\x82\x7f\x71\xc4\x0a\x7a\xfa\x07\x45\x89\xd3\xda\xc9\x06\x7c\x03\xd9\x5a\x16\xfa\x0f\x8c\x38\x35\x2f\x3f\x6b\x43\xbc\xda\xff\x54\x75\x68\x10\xc0\x60\x5d\xb9\x91\xc1\xba\x04\x4b\xda\xd2\xd6\x6a\x4a\x3b\x41\xd5\x7a\x09\x5f\x39\x07\x29\xb1\x8d\x18\x5a\x3c\x9c\xa3\xe4\x63\x0e\xd8\x08\x46\x04\xc5\xe6\xcd\xbb\xe0\xd3\xa6\xab\x7d\xe4\x1c\xea\x0e\x12\xec\x30\xac\xdd\x93\x49\xc6\xa7\xdb\xae\xa5\x9a\x8b\x57\xdd\xd2\xae\xad\x61\x0f\x14\x61\x43\x91\x6c\xf8\x00\x0e\xa5\x5d\xc4\x3a\x71\xc0\x3a\xe0\x1e\x63\x49\xe6\xcd\x5d\x72\x44\x5d\x57\xb5\x83\x9e\x7e\x17\xce\xfd\x98\x55\xed\x9e\x0a\xb2\xa0\x72\x16\x8f\xf3\x9d\xe7\xb4\xa5\x5d\x07\xbd\x8e\xc7\x83\x5c\xe3\x51\x51\xf6\xe4\x11\xbc\xe7\x9c\x6c\xba\xc3\x14\x7a\xa6\xe5\x34\x5b\x2c\x07\x2f\x38\x7f\xe8\x39\xcc\xf6\x7b\x9c\x8c\xf7\x28\x9b\x85\xc9\xe7\xa7\xea\x3e\x7e\x05\x66\x85\x7b\xf2\x45\x9d\x23\x10\x2f\x08\x86\xf7\x22\x95\x62\x1d\xd1\x88\xa4\x76\xc6\x1b\xfa\xb1\x1a\x47\xfe\x01\xfb\xc8\x43\x87\x4b\xe6\x01\xb0\xe3\xa4\x78\x5f\x6e\x3d\x47\xf2\xc3\xd9\xfc\x02\xa9\xe4\xbe\xe4\xb7\xc9\x61\x77\x27\x5e\xc7\x89\x8c\x85\xd2\xae\xf1\x2c\xc8\xda\x78\xee\x4e\xe1\x67\x79\x66\xeb\x23\xe4\xa9\x7e\xe3\xdf\x1d\xda\xf8\x9b\xfb\x50\xae\x4e\xe3\x5d\x1c\xb7\x33\x2d\x35\x4d\xec\xb8\x06\x8e\x2f\x36\x94\x02\xa5\xdd\x74\x7f\xb0\x38\xfa\x74\x9d\xe3\xa8\x5a\xf9\xf3\x0a\xe6\xdb\xeb\xb4\x97\x21\x7f\x37\x3e\xa7\x94\xe7\x9d\xe0\x39\x99\x70\x8c\x28\x7a\xe1\x7a\xa5\x06\x96\xef\x52\x68\x76\x6e\xee\xa4\x10\x92\x0a\x7a\x96\x79\x6a\x0e\xc7\x0f\x84\x9c\x86\xf9\x66\xae\x5b\x01\x35\xc9\xde\xb2\xe0\xbb\x44\xf1\x2d\xf6\xfc\x0f\x7a\x2a\x1d\xb4\xd4\x23\xa1\xbd\xb2\xbc\x1c\x71\x29\xba\x7c\x27\x97\x43\xa4\x5b\xac\x8e\xa7\xf6\xa0\xff\x77\x86\x9e\x9b\x72\x51\xe7\xc3\x6d\xf7\xa0\xb0\x67\xd5\xbd\xd8\xce\x77\x0a\x3c\x97\xed\x2c\x76\x7f\x81\xfd\xac\x6d\x59\x28\x97\x06\x7b\x19\x87\x08\xa7\xa2\x7c\xfa\xfc\xa9\xaa\x9e\xdd\x1f\x24\xc2\x82\xc1\x6d\x85\x3b\x57\xec\x4c\x57\xc2\xd9\x50\x56\x1d\x9a\x90\xd7\xd5\x5c\x82\xba\x0c\x7d\x33\x40\x17\xcf\x6c\x9b\xe2\x71\x23\xcd\x09\x55\x17\xd8\x33\x3d\x79\x9d\xce\x1d\x34\xca\xbe\xc3\x64\xe4\xaf\x2f\x3c\xe3\x17\x4c\x82\x7b\xc2\xd7\xf3\x6d\xf4\x18\x26\xb7\x37\xaf\xe6\xcd\xbf\xe8\x6d\x7a\x40\x3d\x94\xd0\xb3\x83\x14\x1c\x7e\xed\x21\x85\xe2\x33\x3f\x14\x3d\x24\x90\xa1\x3e\x2c\xc8\xe6\x07\xb4\xfc\x78\x47\x3d\xb2\x93\xae\x4f\xe2\x0f\xf3\x50\xf4\x59\xc8\x86\x1b\x54\x16\xb3\x52\x51\xfc\x6a\x9e\x93\x9a\x00\x9d\x3c\xa0\xb2\xe2\x37\xce\x7f\x96\x57\xdb\xf4\xa1\x65\xb5\x79\x94\x1f\xc0\x3a\x90\x7a\xde\xa3\x0c\x17\x5b\xee\xed\x35\x18\xe7\x57\xe0\xe5\x45\xfd\x7f\x00\x00\x00\xff\xff\x82\x99\xe5\x95\x6d\x0c\x00\x00") +var _manifests00ClusterRoleYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\xcd\x8e\xe3\x36\x0c\xbe\xfb\x29\x84\xc9\x61\x81\x05\xec\xa0\xb7\x22\xb7\xa2\x05\x7a\x6a\x17\x28\x8a\xde\x19\x89\x89\xd9\x91\x45\x83\xa4\x32\x9b\x3e\x7d\x21\x47\x9e\xec\x64\xf2\x37\xbb\x39\xd9\x92\xc8\x8f\x1f\x7f\xc1\x85\xfb\x35\x66\x35\x14\x27\x1c\xd1\x6d\x58\x9c\xf5\xe8\x78\x44\x01\x63\x71\x64\x8a\x71\xd3\x35\x0b\xf7\xf7\x97\xdf\xbe\xac\xdc\x2f\x2e\xb2\x39\xde\x14\x29\x45\xa7\x3d\xe7\x18\xdc\x1a\x9d\xe0\x18\xc1\x63\x70\xeb\xfd\x04\xa5\x8e\xd2\x04\x95\x60\x40\x1d\xc1\xa3\x4e\xe8\x2f\x3d\xf9\xbe\x59\xbc\xb5\x02\xde\x32\xc4\xb8\x77\x09\x31\xa8\x03\xef\x51\xb5\x6b\x9e\x29\x85\xd5\x4c\xf0\x2f\x8e\xd8\xc0\x48\xff\xa0\x28\x71\x5a\x39\x59\x83\xef\x20\x5b\xcf\x42\xff\x81\x11\xa7\xee\xf9\x67\xed\x88\x97\xbb\x9f\x9a\x01\x0d\x02\x18\xac\x1a\x37\x31\x58\x15\x63\x49\x7b\xda\x58\x4b\x69\x2b\xa8\xda\xce\xe6\x1b\xe7\x20\x25\xb6\x09\x43\x8b\x86\x73\x94\x7c\xcc\x01\x3b\xc1\x88\xa0\xd8\xbd\x6a\x17\x7c\x5a\x0f\xad\x8f\x9c\x43\x3b\x40\x82\x2d\x86\x95\x7b\x32\xc9\xf8\x74\x5b\xb5\x44\x73\xd6\x6a\x7b\xda\xf6\x2d\xec\x80\x22\xac\x29\x92\xed\x3f\x80\x43\x69\x1b\xb1\x4d\x1c\xb0\x0d\xb8\xc3\x58\x9c\x79\x55\x97\x1c\x51\x57\x4d\xeb\x60\xa4\xdf\x85\xf3\x38\x79\xd5\xba\xa7\x82\x2c\xa8\x9c\xc5\x63\xbd\xf3\x9c\x36\xb4\x1d\x60\xd4\xe9\x78\x4c\xd7\x74\x54\x94\x1d\x79\x04\xef\x39\x27\x3b\xdc\x61\x0a\x23\xd3\x7c\xaa\x12\xf3\xc1\x0b\xd6\x87\x91\x43\x95\xdf\xe1\x41\x78\x87\xb2\x9e\x99\x7c\x7e\x6a\xee\xe3\x57\x60\x96\xb8\x23\x5f\xb2\x73\x02\xe2\x05\xc1\xf0\x5e\xa4\x12\xac\x13\x1a\x91\xd4\xce\x68\xc3\x38\x45\xe3\x44\x3f\xe0\x18\x79\x3f\xe0\xec\x79\x00\x1c\x38\x29\xde\xe7\xdb\xc8\x91\xfc\xfe\xac\x7f\x81\x54\xf2\x58\xfc\x5b\xe7\xb0\xbd\x13\x6f\xe0\x44\xc6\x42\x69\xdb\x79\x16\x64\xed\x3c\x0f\xef\xe1\x6b\x7a\xaa\xf4\x09\xf2\x21\x7e\xd3\xef\x16\x6d\xfa\xe6\x31\x94\xab\xf7\xf6\x2e\xb6\xdb\x99\x92\x3a\x74\xec\x34\x06\x4e\x2f\xd6\x94\x02\xa5\xed\xe1\xfe\x28\x71\xf2\x74\x9d\xe3\x94\xb5\xf2\xf3\x02\xe6\xfb\xeb\xb4\xe7\x26\x7f\xd3\x3e\xef\x29\xd7\x99\xe0\x39\x99\x70\x8c\x28\x7a\xe1\x7a\xa9\x06\x96\xef\xca\x50\x55\xee\xee\xa4\x10\x92\x0a\x7a\x96\xda\x35\xc7\xe3\x07\x4c\x1e\x9a\xf9\xa6\xaf\x1b\x01\x35\xc9\xde\xb2\xe0\x1b\x47\xf1\xd5\x76\xfd\x83\x91\x4a\x05\xcd\xf1\x48\x68\x2f\x2c\xcf\xb5\xc1\x85\xbf\xd2\x69\x47\x95\x1c\x7d\x27\xaf\xa3\xd5\xdb\x0c\xcf\xda\x3e\xa9\x8b\xef\xa4\x51\x8b\x75\xce\xda\x87\xcb\xf1\x41\x66\xcf\x66\xfd\x62\x99\xdf\x99\xf8\x1a\xc2\xb3\xd8\xe3\x05\xf6\x35\xe7\x65\xd0\x5c\x6a\xf8\xb9\x4d\x22\xd4\x04\x7d\x03\xfb\xe9\xf3\xa7\xa6\x59\xb8\x3f\x48\x84\x05\x83\xdb\x08\x0f\xae\xc8\x99\x2e\x85\xb3\xa1\x2c\x07\x34\x21\xaf\xcb\x1a\x82\xb6\x0c\x83\x6e\x0f\x43\x3c\x33\x85\x8a\xc6\x0d\x37\x0f\xa8\x3a\xc3\x9e\xa9\xcf\xeb\x74\xee\xa0\x51\xe6\x20\x26\x23\x7f\x7d\x10\x1a\x3f\x63\x12\xdc\x11\xbe\x9c\x2f\xa3\xc7\x30\xb9\x3d\x91\x35\xaf\xff\x45\x6f\x87\xc5\xea\xa1\x84\x16\x0e\x52\x70\xf8\x75\x84\x14\x8a\x4e\x5d\x20\x3d\x24\x90\x7d\x7b\x1c\x9c\xdd\x0f\xe4\xf2\xe3\x15\xf5\xc8\x4a\xba\xde\x89\x3f\xcc\x43\xd1\x67\x21\xdb\xdf\xa0\x32\x8b\x95\x88\xe2\x57\xf3\x9c\xd4\x04\xe8\xdd\x62\x95\x15\xbf\x51\xfe\xb3\x6c\x73\x87\x87\x9e\xd5\x6a\x2b\x3f\x80\x75\x20\xf5\xbc\x43\xd9\x5f\x2c\xb9\xd7\x2d\x31\xd6\xed\xf0\xf2\xa0\xfe\x3f\x00\x00\xff\xff\xfd\xe3\x0b\xe8\x85\x0c\x00\x00") func manifests00ClusterRoleYamlBytes() ([]byte, error) { return bindataRead( @@ -417,8 +417,8 @@ func manifests00ClusterRoleYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/00-cluster-role.yaml", size: 3181, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xbc, 0x19, 0xbd, 0x6e, 0x68, 0x84, 0x61, 0xc8, 0x1a, 0x23, 0xe9, 0x91, 0xfb, 0x1f, 0xb2, 0xd9, 0xc2, 0x59, 0x46, 0xc7, 0x74, 0x9b, 0x46, 0x32, 0xe2, 0x96, 0x94, 0xe3, 0xe1, 0xc5, 0x91, 0x53}} + info := bindataFileInfo{name: "manifests/00-cluster-role.yaml", size: 3205, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x94, 0x52, 0x55, 0xc2, 0x19, 0xba, 0x6f, 0x98, 0x8, 0x9a, 0x9, 0x75, 0x6a, 0x4c, 0x58, 0x77, 0xb9, 0x61, 0xb6, 0x27, 0xe, 0xc3, 0x2c, 0x29, 0xe1, 0x27, 0xd2, 0x1d, 0x2b, 0x7f, 0x16, 0x13}} return a, nil } diff --git a/pkg/operator/controller/crl/crl_configmap.go b/pkg/operator/controller/crl/crl_configmap.go index 9c74166269..0a00fc9078 100644 --- a/pkg/operator/controller/crl/crl_configmap.go +++ b/pkg/operator/controller/crl/crl_configmap.go @@ -1,19 +1,8 @@ package crl import ( - "bytes" "context" - "crypto/x509" - "crypto/x509/pkix" - "encoding/asn1" - "encoding/hex" - "encoding/pem" "fmt" - "io/ioutil" - "net/http" - "reflect" - "strings" - "time" operatorv1 "github.com/openshift/api/operator/v1" "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" @@ -21,18 +10,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kerrors "k8s.io/apimachinery/pkg/util/errors" ) -// authorityKeyIdentifier is a certificate's authority key identifier. -type authorityKeyIdentifier struct { - KeyIdentifier []byte `asn1:"optional,tag:0"` -} - -// authorityKeyIdentifierOID is the ASN.1 object identifier for the authority -// key identifier extension. -var authorityKeyIdentifierOID = asn1.ObjectIdentifier{2, 5, 29, 35} - // ensureCRLConfigmap ensures the client CA certificate revocation list // configmap exists for a given ingresscontroller if the ingresscontroller // specifies a client CA certificate bundle in which any certificates specify @@ -44,32 +23,9 @@ func (r *reconciler) ensureCRLConfigmap(ctx context.Context, ic *operatorv1.Ingr return false, nil, ctx, err } - var oldCRLs map[string]*pkix.CertificateList - if haveCM { - if data, ok := current.Data["crl.pem"]; ok { - if crls, err := buildCRLMap([]byte(data)); err != nil { - log.Error(err, "failed to parse current client CA configmap", "namespace", current.Namespace, "name", current.Name) - } else { - oldCRLs = crls - } - } - - } - - var clientCAData []byte - if haveClientCA { - clientCABundleFilename := "ca-bundle.pem" - if data, ok := clientCAConfigmap.Data[clientCABundleFilename]; !ok { - return haveCM, current, ctx, fmt.Errorf("client CA configmap %s/%s is missing %q", clientCAConfigmap.Namespace, clientCAConfigmap.Name, clientCABundleFilename) - } else { - clientCAData = []byte(data) - } - } - - wantCM, desired, ctx, err := desiredCRLConfigMap(ctx, ic, ownerRef, clientCAData, oldCRLs) - if err != nil { - return false, nil, ctx, fmt.Errorf("failed to build configmap: %w", err) - } + // The CRL management code has been moved into the router, so the CRL configmap is no longer necessary. + // TODO: Remove this whole controller after 4.14 + wantCM := false switch { case !wantCM && !haveCM: @@ -83,191 +39,8 @@ func (r *reconciler) ensureCRLConfigmap(ctx context.Context, ic *operatorv1.Ingr log.Info("deleted configmap", "namespace", current.Namespace, "name", current.Name) } return false, nil, ctx, nil - case wantCM && !haveCM: - if err := r.client.Create(ctx, desired); err != nil { - return false, nil, ctx, fmt.Errorf("failed to create configmap: %w", err) - } - log.Info("created configmap", "namespace", desired.Namespace, "name", desired.Name) - exists, current, err := r.currentCRLConfigMap(ctx, ic) - return exists, current, ctx, err - case wantCM && haveCM: - if updated, err := r.updateCRLConfigMap(ctx, current, desired); err != nil { - return true, current, ctx, fmt.Errorf("failed to update configmap: %w", err) - } else if updated { - log.Info("updated configmap", "namespace", desired.Namespace, "name", desired.Name) - exists, current, err := r.currentCRLConfigMap(ctx, ic) - return exists, current, ctx, err - } - } - - return true, current, ctx, nil -} - -// buildCRLMap builds a map of key identifier to certificate list using the -// provided PEM-encoded certificate revocation list. -func buildCRLMap(crlData []byte) (map[string]*pkix.CertificateList, error) { - crlForKeyId := make(map[string]*pkix.CertificateList) - for len(crlData) > 0 { - block, data := pem.Decode(crlData) - if block == nil { - break - } - crl, err := x509.ParseCRL(block.Bytes) - if err != nil { - return crlForKeyId, err - } - for _, ext := range crl.TBSCertList.Extensions { - if ext.Id.Equal(authorityKeyIdentifierOID) { - var authKeyId authorityKeyIdentifier - if _, err := asn1.Unmarshal(ext.Value, &authKeyId); err != nil { - return crlForKeyId, err - } - subjectKeyId := hex.EncodeToString(authKeyId.KeyIdentifier) - crlForKeyId[subjectKeyId] = crl - } - } - crlData = data - } - return crlForKeyId, nil -} - -// desiredCRLConfigMap returns the desired CRL configmap. Returns a Boolean -// indicating whether a configmap is desired, the configmap if one is desired, -// the context (containing the next CRL update time as "nextCRLUpdate"), and an -// error if one occurred -func desiredCRLConfigMap(ctx context.Context, ic *operatorv1.IngressController, ownerRef metav1.OwnerReference, clientCAData []byte, crls map[string]*pkix.CertificateList) (bool, *corev1.ConfigMap, context.Context, error) { - if len(ic.Spec.ClientTLS.ClientCertificatePolicy) == 0 || len(ic.Spec.ClientTLS.ClientCA.Name) == 0 { - return false, nil, ctx, nil - } - - if crls == nil { - crls = make(map[string]*pkix.CertificateList) - } - - var subjectKeyIds []string - var nextCRLUpdate time.Time - now := time.Now() - for len(clientCAData) > 0 { - block, data := pem.Decode(clientCAData) - if block == nil { - break - } - clientCAData = data - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return false, nil, ctx, fmt.Errorf("client CA configmap has an invalid certificate: %w", err) - } - subjectKeyId := hex.EncodeToString(cert.SubjectKeyId) - if len(cert.CRLDistributionPoints) == 0 { - continue - } - if crl, ok := crls[subjectKeyId]; ok { - if crl.HasExpired(now) { - log.Info("certificate revocation list has expired", "subject key identifier", subjectKeyId) - } else { - subjectKeyIds = append(subjectKeyIds, subjectKeyId) - if (nextCRLUpdate.IsZero() || crl.TBSCertList.NextUpdate.Before(nextCRLUpdate)) && crl.TBSCertList.NextUpdate.After(now) { - nextCRLUpdate = crl.TBSCertList.NextUpdate - } - continue - } - } - log.Info("retrieving certificate revocation list", "subject key identifier", subjectKeyId) - if crl, err := getCRL(cert.CRLDistributionPoints); err != nil { - // Creating or updating the configmap with incomplete - // data would compromise security by potentially - // permitting revoked certificates. - return false, nil, ctx, fmt.Errorf("failed to get certificate revocation list for certificate key %s: %w", subjectKeyId, err) - } else { - crls[subjectKeyId] = crl - subjectKeyIds = append(subjectKeyIds, subjectKeyId) - log.Info("new certificate revocation list", "subject key identifier", subjectKeyId, "next update", crl.TBSCertList.NextUpdate.String()) - if (nextCRLUpdate.IsZero() || crl.TBSCertList.NextUpdate.Before(nextCRLUpdate)) && crl.TBSCertList.NextUpdate.After(now) { - nextCRLUpdate = crl.TBSCertList.NextUpdate - } - } - } - - if len(subjectKeyIds) == 0 { - return false, nil, ctx, nil - } - - buf := &bytes.Buffer{} - for _, subjectKeyId := range subjectKeyIds { - asn1Data, err := asn1.Marshal(*crls[subjectKeyId]) - if err != nil { - return false, nil, ctx, fmt.Errorf("failed to encode ASN.1 for CRL for certificate key %s: %w", subjectKeyId, err) - } - block := &pem.Block{ - Type: "X509 CRL", - Bytes: asn1Data, - } - if err := pem.Encode(buf, block); err != nil { - return false, nil, ctx, fmt.Errorf("failed to encode PEM for CRL for certificate key %s: %w", subjectKeyId, err) - } - } - crlData := buf.String() - - crlConfigmapName := controller.CRLConfigMapName(ic) - crlConfigmap := corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: crlConfigmapName.Name, - Namespace: crlConfigmapName.Namespace, - }, - Data: map[string]string{ - "crl.pem": crlData, - }, - } - crlConfigmap.SetOwnerReferences([]metav1.OwnerReference{ownerRef}) - - return true, &crlConfigmap, context.WithValue(ctx, "nextCRLUpdate", nextCRLUpdate), nil -} - -// getCRL gets a certificate revocation list using the provided distribution -// points and returns the certificate list. -func getCRL(distributionPoints []string) (*pkix.CertificateList, error) { - var errs []error - for _, distributionPoint := range distributionPoints { - // The distribution point is typically a URL with the "http" - // scheme. "https" is generally not used because the - // certificate list is signed, and because using TLS to get the - // certificate list could introduce a circular dependency - // (cannot use TLS without the revocation list, and cannot get - // the revocation list without using TLS). - // - // TODO Support ldap. - switch { - case strings.HasPrefix(distributionPoint, "http:"): - log.Info("retrieving CRL distribution point", "distribution point", distributionPoint) - crl, err := getHTTPCRL(distributionPoint) - if err != nil { - errs = append(errs, fmt.Errorf("error getting %q: %w", distributionPoint, err)) - continue - } - return crl, nil - default: - errs = append(errs, fmt.Errorf("unsupported distribution point type: %s", distributionPoint)) - } - } - return nil, kerrors.NewAggregate(errs) -} - -// getHTTPCRL gets a certificate revocation list using the provided HTTP URL. -func getHTTPCRL(url string) (*pkix.CertificateList, error) { - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("http.Get failed: %w", err) - } - defer resp.Body.Close() - bytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("error reading response: %w", err) - } - crl, err := x509.ParseCRL(bytes) - if err != nil { - return nil, fmt.Errorf("error parsing response: %w", err) } - return crl, nil + return false, nil, ctx, nil } // currentCRLConfigMap returns the current CRL configmap. Returns a Boolean @@ -283,30 +56,3 @@ func (r *reconciler) currentCRLConfigMap(ctx context.Context, ic *operatorv1.Ing } return true, cm, nil } - -// updateCRLConfigMap updates a configmap. Returns a Boolean indicating whether -// the configmap was updated, and an error value. -func (r *reconciler) updateCRLConfigMap(ctx context.Context, current, desired *corev1.ConfigMap) (bool, error) { - if crlConfigmapsEqual(current, desired) { - return false, nil - } - updated := current.DeepCopy() - updated.Data = desired.Data - if err := r.client.Update(ctx, updated); err != nil { - if errors.IsAlreadyExists(err) { - return false, nil - } - return false, err - } - return true, nil -} - -// crlConfigmapsEqual compares two CRL configmaps. Returns true if the -// configmaps should be considered equal for the purpose of determining whether -// an update is necessary, false otherwise -func crlConfigmapsEqual(a, b *corev1.ConfigMap) bool { - if !reflect.DeepEqual(a.Data, b.Data) { - return false - } - return true -} diff --git a/pkg/operator/controller/ingress/controller.go b/pkg/operator/controller/ingress/controller.go index 93b203e403..d545fa12d9 100644 --- a/pkg/operator/controller/ingress/controller.go +++ b/pkg/operator/controller/ingress/controller.go @@ -125,6 +125,10 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) { if err := c.Watch(&source.Kind{Type: &configv1.Ingress{}}, handler.EnqueueRequestsFromMapFunc(reconciler.ingressConfigToIngressController)); err != nil { return nil, err } + // Watch for changes to cluster-wide proxy config. + if err := c.Watch(&source.Kind{Type: &configv1.Proxy{}}, handler.EnqueueRequestsFromMapFunc(reconciler.ingressConfigToIngressController)); err != nil { + return nil, err + } return c, nil } @@ -264,6 +268,12 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( if platformStatus == nil { return reconcile.Result{}, fmt.Errorf("failed to determine infrastructure platform status for ingresscontroller %s/%s: PlatformStatus is nil", ingress.Namespace, ingress.Name) } + clusterProxyConfig := &configv1.Proxy{} + if err := r.client.Get(ctx, types.NamespacedName{Name: "cluster"}, clusterProxyConfig); err != nil { + if !kerrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to get proxy 'cluster': %v", err) + } + } // Admit if necessary. Don't process until admission succeeds. If admission is // successful, immediately re-queue to refresh state. @@ -307,7 +317,7 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( } // The ingresscontroller is safe to process, so ensure it. - if err := r.ensureIngressController(ingress, dnsConfig, infraConfig, platformStatus, ingressConfig, apiConfig, networkConfig); err != nil { + if err := r.ensureIngressController(ingress, dnsConfig, infraConfig, platformStatus, ingressConfig, apiConfig, networkConfig, clusterProxyConfig); err != nil { switch e := err.(type) { case retryable.Error: log.Error(e, "got retryable error; requeueing", "after", e.After()) @@ -950,7 +960,7 @@ func (r *reconciler) ensureIngressDeleted(ingress *operatorv1.IngressController) // given ingresscontroller. Any error values are collected into either a // retryable.Error value, if any of the error values are retryable, or else an // Aggregate error value. -func (r *reconciler) ensureIngressController(ci *operatorv1.IngressController, dnsConfig *configv1.DNS, infraConfig *configv1.Infrastructure, platformStatus *configv1.PlatformStatus, ingressConfig *configv1.Ingress, apiConfig *configv1.APIServer, networkConfig *configv1.Network) error { +func (r *reconciler) ensureIngressController(ci *operatorv1.IngressController, dnsConfig *configv1.DNS, infraConfig *configv1.Infrastructure, platformStatus *configv1.PlatformStatus, ingressConfig *configv1.Ingress, apiConfig *configv1.APIServer, networkConfig *configv1.Network, clusterProxyConfig *configv1.Proxy) error { // Before doing anything at all with the controller, ensure it has a finalizer // so we can clean up later. if !slice.ContainsString(ci.Finalizers, manifests.IngressControllerFinalizer) { @@ -1001,7 +1011,7 @@ func (r *reconciler) ensureIngressController(ci *operatorv1.IngressController, d haveClientCAConfigmap = true } - haveDepl, deployment, err := r.ensureRouterDeployment(ci, infraConfig, ingressConfig, apiConfig, networkConfig, haveClientCAConfigmap, clientCAConfigmap, platformStatus) + haveDepl, deployment, err := r.ensureRouterDeployment(ci, infraConfig, ingressConfig, apiConfig, networkConfig, haveClientCAConfigmap, clientCAConfigmap, platformStatus, clusterProxyConfig) if err != nil { errs = append(errs, fmt.Errorf("failed to ensure deployment: %v", err)) return utilerrors.NewAggregate(errs) diff --git a/pkg/operator/controller/ingress/deployment.go b/pkg/operator/controller/ingress/deployment.go index e4e9bef816..e7c3c25c8f 100644 --- a/pkg/operator/controller/ingress/deployment.go +++ b/pkg/operator/controller/ingress/deployment.go @@ -2,9 +2,7 @@ package ingress import ( "context" - "crypto/x509" "encoding/json" - "encoding/pem" "fmt" "hash" "hash/fnv" @@ -87,7 +85,6 @@ const ( RouterClientAuthPolicy = "ROUTER_MUTUAL_TLS_AUTH" RouterClientAuthCA = "ROUTER_MUTUAL_TLS_AUTH_CA" - RouterClientAuthCRL = "ROUTER_MUTUAL_TLS_AUTH_CRL" RouterClientAuthFilter = "ROUTER_MUTUAL_TLS_AUTH_FILTER" RouterEnableCompression = "ROUTER_ENABLE_COMPRESSION" @@ -107,7 +104,7 @@ const ( // ensureRouterDeployment ensures the router deployment exists for a given // ingresscontroller. -func (r *reconciler) ensureRouterDeployment(ci *operatorv1.IngressController, infraConfig *configv1.Infrastructure, ingressConfig *configv1.Ingress, apiConfig *configv1.APIServer, networkConfig *configv1.Network, haveClientCAConfigmap bool, clientCAConfigmap *corev1.ConfigMap, platformStatus *configv1.PlatformStatus) (bool, *appsv1.Deployment, error) { +func (r *reconciler) ensureRouterDeployment(ci *operatorv1.IngressController, infraConfig *configv1.Infrastructure, ingressConfig *configv1.Ingress, apiConfig *configv1.APIServer, networkConfig *configv1.Network, haveClientCAConfigmap bool, clientCAConfigmap *corev1.ConfigMap, platformStatus *configv1.PlatformStatus, clusterProxyConfig *configv1.Proxy) (bool, *appsv1.Deployment, error) { haveDepl, current, err := r.currentRouterDeployment(ci) if err != nil { return false, nil, err @@ -116,7 +113,7 @@ func (r *reconciler) ensureRouterDeployment(ci *operatorv1.IngressController, in if err != nil { return false, nil, fmt.Errorf("failed to determine if proxy protocol is needed for ingresscontroller %s/%s: %v", ci.Namespace, ci.Name, err) } - desired, err := desiredRouterDeployment(ci, r.config.IngressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, haveClientCAConfigmap, clientCAConfigmap) + desired, err := desiredRouterDeployment(ci, r.config.IngressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, haveClientCAConfigmap, clientCAConfigmap, clusterProxyConfig) if err != nil { return haveDepl, current, fmt.Errorf("failed to build router deployment: %v", err) } @@ -222,7 +219,7 @@ func determineDeploymentReplicas(ic *operatorv1.IngressController, ingressConfig } // desiredRouterDeployment returns the desired router deployment. -func desiredRouterDeployment(ci *operatorv1.IngressController, ingressControllerImage string, ingressConfig *configv1.Ingress, infraConfig *configv1.Infrastructure, apiConfig *configv1.APIServer, networkConfig *configv1.Network, proxyNeeded bool, haveClientCAConfigmap bool, clientCAConfigmap *corev1.ConfigMap) (*appsv1.Deployment, error) { +func desiredRouterDeployment(ci *operatorv1.IngressController, ingressControllerImage string, ingressConfig *configv1.Ingress, infraConfig *configv1.Infrastructure, apiConfig *configv1.APIServer, networkConfig *configv1.Network, proxyNeeded bool, haveClientCAConfigmap bool, clientCAConfigmap *corev1.ConfigMap, clusterProxyConfig *configv1.Proxy) (*appsv1.Deployment, error) { deployment := manifests.RouterDeployment() name := controller.RouterDeploymentName(ci) deployment.Name = name.Name @@ -1009,68 +1006,6 @@ func desiredRouterDeployment(ci *operatorv1.IngressController, ingressController clientAuthCAPath := filepath.Join(clientCAVolumeMount.MountPath, clientCABundleFilename) env = append(env, corev1.EnvVar{Name: RouterClientAuthCA, Value: clientAuthCAPath}) - if haveClientCAConfigmap { - // If any certificates in the client CA bundle - // specify any CRL distribution points, then we - // need to configure a configmap volume. The - // crl controller is responsible for managing - // the configmap. - var clientCAData []byte - if v, ok := clientCAConfigmap.Data[clientCABundleFilename]; !ok { - return nil, fmt.Errorf("client CA configmap %s/%s is missing %q", clientCAConfigmap.Namespace, clientCAConfigmap.Name, clientCABundleFilename) - } else { - clientCAData = []byte(v) - } - var someClientCAHasCRL bool - for len(clientCAData) > 0 { - block, data := pem.Decode(clientCAData) - if block == nil { - break - } - clientCAData = data - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("client CA configmap %s/%s has an invalid certificate: %w", clientCAConfigmap.Namespace, clientCAConfigmap.Name, err) - } - if len(cert.CRLDistributionPoints) != 0 { - someClientCAHasCRL = true - break - } - } - if someClientCAHasCRL { - clientCACRLSecretName := controller.CRLConfigMapName(ci) - clientCACRLVolumeName := "client-ca-crl" - clientCACRLVolumeMountPath := "/etc/pki/tls/client-ca-crl" - clientCACRLFilename := "crl.pem" - clientCACRLVolume := corev1.Volume{ - Name: clientCACRLVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: clientCACRLSecretName.Name, - }, - Items: []corev1.KeyToPath{ - { - Key: clientCACRLFilename, - Path: clientCACRLFilename, - }, - }, - }, - }, - } - clientCACRLVolumeMount := corev1.VolumeMount{ - Name: clientCACRLVolumeName, - MountPath: clientCACRLVolumeMountPath, - ReadOnly: true, - } - volumes = append(volumes, clientCACRLVolume) - routerVolumeMounts = append(routerVolumeMounts, clientCACRLVolumeMount) - - clientAuthCRLPath := filepath.Join(clientCACRLVolumeMount.MountPath, clientCACRLFilename) - env = append(env, corev1.EnvVar{Name: RouterClientAuthCRL, Value: clientAuthCRLPath}) - } - } - if len(ci.Spec.ClientTLS.AllowedSubjectPatterns) != 0 { pattern := "(?:" + strings.Join(ci.Spec.ClientTLS.AllowedSubjectPatterns, "|") + ")" env = append(env, corev1.EnvVar{Name: RouterClientAuthFilter, Value: pattern}) @@ -1090,6 +1025,31 @@ func desiredRouterDeployment(ci *operatorv1.IngressController, ingressController env = append(env, corev1.EnvVar{Name: RouterCompressionMIMETypes, Value: strings.Join(mimes, " ")}) } + // Add cluster-wide proxy environment variables if necessary. + if clusterProxyConfig.Status.HTTPProxy != "" { + env = append(env, + corev1.EnvVar{Name: "HTTP_PROXY", Value: clusterProxyConfig.Status.HTTPProxy}, + corev1.EnvVar{Name: "http_proxy", Value: clusterProxyConfig.Status.HTTPProxy}, + ) + } + if clusterProxyConfig.Status.HTTPSProxy != "" { + env = append(env, + corev1.EnvVar{Name: "HTTPS_PROXY", Value: clusterProxyConfig.Status.HTTPSProxy}, + corev1.EnvVar{Name: "https_proxy", Value: clusterProxyConfig.Status.HTTPSProxy}, + ) + } + if clusterProxyConfig.Status.NoProxy != "" { + env = append(env, + corev1.EnvVar{Name: "NO_PROXY", Value: clusterProxyConfig.Status.NoProxy}, + corev1.EnvVar{Name: "no_proxy", Value: clusterProxyConfig.Status.NoProxy}, + ) + } + + // TODO: The only connections from the router that may need the cluster-wide proxy are those for downloading CRLs, + // which, as of writing this, will always be http. If https becomes necessary, the router will need to mount the + // trusted CA bundle that cluster-network-operator generates. The process for adding that is described here: + // https://docs.openshift.com/container-platform/4.13/operators/admin/olm-configuring-proxy-support.html#olm-inject-custom-ca_olm-configuring-proxy-support + // Add the environment variables to the container deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, env...) diff --git a/pkg/operator/controller/ingress/deployment_test.go b/pkg/operator/controller/ingress/deployment_test.go index 91139edb73..b5e1627e92 100644 --- a/pkg/operator/controller/ingress/deployment_test.go +++ b/pkg/operator/controller/ingress/deployment_test.go @@ -154,7 +154,7 @@ func checkDeploymentEnvironment(t *testing.T, deployment *appsv1.Deployment, exp } func TestTuningOptions(t *testing.T) { - ic, ingressConfig, infraConfig, apiConfig, networkConfig, _ := getRouterDeploymentComponents(t) + ic, ingressConfig, infraConfig, apiConfig, networkConfig, _, clusterProxyConfig := getRouterDeploymentComponents(t) // Set up tuning options ic.Spec.TuningOptions.ClientTimeout = &metav1.Duration{Duration: 45 * time.Second} @@ -166,7 +166,7 @@ func TestTuningOptions(t *testing.T) { ic.Spec.TuningOptions.HealthCheckInterval = &metav1.Duration{Duration: 15 * time.Second} ic.Spec.TuningOptions.ReloadInterval = metav1.Duration{Duration: 30 * time.Second} - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, false, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, false, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -190,9 +190,58 @@ func TestTuningOptions(t *testing.T) { checkDeploymentHasEnvSorted(t, deployment) } +// TestClusterProxy tests that the cluster-wide proxy settings from proxies.config.openshift.io/cluster are included in the desired router deployment. +func TestClusterProxy(t *testing.T) { + ic, ingressConfig, infraConfig, apiConfig, networkConfig, _, clusterProxyConfig := getRouterDeploymentComponents(t) + + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, false, false, nil, clusterProxyConfig) + if err != nil { + t.Fatalf("invalid router Deployment: %v", err) + } + + // Verify that with an empty cluster proxy config, none of the proxy variables are set. + expectedEnv := []envData{ + {"HTTP_PROXY", false, ""}, + {"http_proxy", false, ""}, + {"HTTPS_PROXY", false, ""}, + {"https_proxy", false, ""}, + {"NO_PROXY", false, ""}, + {"no_proxy", false, ""}, + } + + if err := checkDeploymentEnvironment(t, deployment, expectedEnv); err != nil { + t.Errorf("empty configv1.Proxy: %v", err) + } + + // With values set in the cluster proxy config, verify that those values are set in the desired deployment. + clusterProxyConfig.Status.HTTPProxy = "foo" + clusterProxyConfig.Status.HTTPSProxy = "bar" + clusterProxyConfig.Status.NoProxy = "baz" + + deployment, err = desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, false, false, nil, clusterProxyConfig) + if err != nil { + t.Fatalf("invalid router Deployment: %v", err) + } + + expectedEnv = []envData{ + {"HTTP_PROXY", true, clusterProxyConfig.Status.HTTPProxy}, + {"http_proxy", true, clusterProxyConfig.Status.HTTPProxy}, + {"HTTPS_PROXY", true, clusterProxyConfig.Status.HTTPSProxy}, + {"https_proxy", true, clusterProxyConfig.Status.HTTPSProxy}, + {"NO_PROXY", true, clusterProxyConfig.Status.NoProxy}, + {"no_proxy", true, clusterProxyConfig.Status.NoProxy}, + } + + if err := checkDeploymentEnvironment(t, deployment, expectedEnv); err != nil { + t.Error(err) + } + + checkDeploymentHasEnvSorted(t, deployment) +} + // return defaulted IngressController, Ingress config, Infrastructure config, APIServer config, Network config, // and whether proxy is needed -func getRouterDeploymentComponents(t *testing.T) (*operatorv1.IngressController, *configv1.Ingress, *configv1.Infrastructure, *configv1.APIServer, *configv1.Network, bool) { +func getRouterDeploymentComponents(t *testing.T) (*operatorv1.IngressController, *configv1.Ingress, *configv1.Infrastructure, *configv1.APIServer, *configv1.Network, bool, *configv1.Proxy) { t.Helper() var one int32 = 1 @@ -257,13 +306,15 @@ func getRouterDeploymentComponents(t *testing.T) (*operatorv1.IngressController, t.Errorf("failed to determine infrastructure platform status for ingresscontroller %s/%s: %v", ic.Namespace, ic.Name, err) } - return ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded + clusterProxyConfig := &configv1.Proxy{} + + return ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, clusterProxyConfig } func TestDesiredRouterDeployment(t *testing.T) { - ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded := getRouterDeploymentComponents(t) + ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, clusterProxyConfig := getRouterDeploymentComponents(t) - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -340,9 +391,9 @@ func TestDesiredRouterDeployment(t *testing.T) { } func TestDesiredRouterDeploymentSpecTemplate(t *testing.T) { - ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded := getRouterDeploymentComponents(t) + ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, clusterProxyConfig := getRouterDeploymentComponents(t) - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -402,7 +453,7 @@ func TestDesiredRouterDeploymentSpecTemplate(t *testing.T) { } func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) { - ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded := getRouterDeploymentComponents(t) + ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, clusterProxyConfig := getRouterDeploymentComponents(t) ic.Spec.Logging = &operatorv1.IngressControllerLogging{ Access: &operatorv1.AccessLogging{ @@ -470,7 +521,7 @@ func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) { if err != nil { t.Errorf("failed to determine infrastructure platform status for ingresscontroller %s/%s: %v", ic.Namespace, ic.Name, err) } - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -557,7 +608,7 @@ func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) { if err != nil { t.Errorf("failed to determine infrastructure platform status for ingresscontroller %s/%s: %v", ic.Namespace, ic.Name, err) } - deployment, err = desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil) + deployment, err = desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -592,7 +643,7 @@ func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) { } func TestDesiredRouterDeploymentVariety(t *testing.T) { - ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded := getRouterDeploymentComponents(t) + ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, clusterProxyConfig := getRouterDeploymentComponents(t) secretName := fmt.Sprintf("secret-%v", time.Now().UnixNano()) ic.Spec.DefaultCertificate = &corev1.LocalObjectReference{ @@ -672,7 +723,7 @@ func TestDesiredRouterDeploymentVariety(t *testing.T) { if err != nil { t.Errorf("failed to determine infrastructure platform status for ingresscontroller %s/%s: %v", ic.Namespace, ic.Name, err) } - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -780,13 +831,13 @@ func TestDesiredRouterDeploymentVariety(t *testing.T) { // subfields for spec.endpointPublishingStrategy.hostNetwork. // See . func TestDesiredRouterDeploymentHostNetworkNil(t *testing.T) { - ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded := getRouterDeploymentComponents(t) + ic, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, clusterProxyConfig := getRouterDeploymentComponents(t) ic.Status.EndpointPublishingStrategy.Type = operatorv1.HostNetworkStrategyType proxyNeeded, err := IsProxyProtocolNeeded(ic, infraConfig.Status.PlatformStatus) if err != nil { t.Fatal(err) } - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, proxyNeeded, false, nil, clusterProxyConfig) if err != nil { t.Fatal(err) } @@ -805,7 +856,7 @@ func TestDesiredRouterDeploymentHostNetworkNil(t *testing.T) { } func TestDesiredRouterDeploymentSingleReplica(t *testing.T) { - ic, ingressConfig, _, apiConfig, networkConfig, _ := getRouterDeploymentComponents(t) + ic, ingressConfig, _, apiConfig, networkConfig, _, clusterProxyConfig := getRouterDeploymentComponents(t) infraConfig := &configv1.Infrastructure{ Status: configv1.InfrastructureStatus{ @@ -817,7 +868,7 @@ func TestDesiredRouterDeploymentSingleReplica(t *testing.T) { }, } - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, false, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, ingressConfig, infraConfig, apiConfig, networkConfig, false, false, nil, clusterProxyConfig) if err != nil { t.Fatalf("invalid router Deployment: %v", err) } @@ -2019,7 +2070,7 @@ func TestDesiredRouterDeploymentDefaultPlacement(t *testing.T) { // This value does not matter in the context of this test, just use a dummy value dummyProxyNeeded := true - deployment, err := desiredRouterDeployment(ic, ingressControllerImage, tc.ingressConfig, tc.infraConfig, apiConfig, networkConfig, dummyProxyNeeded, false, nil) + deployment, err := desiredRouterDeployment(ic, ingressControllerImage, tc.ingressConfig, tc.infraConfig, apiConfig, networkConfig, dummyProxyNeeded, false, nil, &configv1.Proxy{}) if err != nil { t.Error(err) } diff --git a/test/e2e/all_test.go b/test/e2e/all_test.go index 2f70de3d7e..823e4faf79 100644 --- a/test/e2e/all_test.go +++ b/test/e2e/all_test.go @@ -22,6 +22,8 @@ func TestAll(t *testing.T) { t.Run("parallel", func(t *testing.T) { t.Run("TestAWSELBConnectionIdleTimeout", TestAWSELBConnectionIdleTimeout) t.Run("TestClientTLS", TestClientTLS) + t.Run("TestMTLSWithCRLs", TestMTLSWithCRLs) + t.Run("TestCRLUpdate", TestCRLUpdate) t.Run("TestContainerLogging", TestContainerLogging) t.Run("TestCustomErrorpages", TestCustomErrorpages) t.Run("TestCustomIngressClass", TestCustomIngressClass) diff --git a/test/e2e/client_tls_test.go b/test/e2e/client_tls_test.go index 8b85b4264c..0e4317fe88 100644 --- a/test/e2e/client_tls_test.go +++ b/test/e2e/client_tls_test.go @@ -13,6 +13,8 @@ import ( "encoding/pem" "fmt" "math/big" + "path/filepath" + "strconv" "strings" "testing" "time" @@ -20,22 +22,19 @@ import ( configv1 "github.com/openshift/api/config/v1" operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" - "sigs.k8s.io/controller-runtime/pkg/client/config" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apiserver/pkg/storage/names" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/tools/remotecommand" ) // TestClientTLS generates a CA certificate, uses it to sign some client @@ -164,11 +163,11 @@ func TestClientTLS(t *testing.T) { Namespace: "openshift-ingress", }, Data: map[string]string{ - "valid-matching.crt": encodeCert(validMatchingCert), + "valid-matching.pem": encodeCert(validMatchingCert), "valid-matching.key": encodeKey(validMatchingKey), - "valid-mismatching.crt": encodeCert(validMismatchingCert), + "valid-mismatching.pem": encodeCert(validMismatchingCert), "valid-mismatching.key": encodeKey(validMismatchingKey), - "invalid-matching.crt": encodeCert(invalidMatchingCert), + "invalid-matching.pem": encodeCert(invalidMatchingCert), "invalid-matching.key": encodeKey(invalidMatchingKey), }, } @@ -237,73 +236,6 @@ func TestClientTLS(t *testing.T) { t.Fatalf("timed out waiting for pod %q to become ready: %v", clientPodName, err) } - // We need a client-go client in order to execute commands in the client - // pod. - kubeConfig, err := config.GetConfig() - if err != nil { - t.Fatalf("failed to get kube config: %v", err) - } - cl, err := kubernetes.NewForConfig(kubeConfig) - if err != nil { - t.Fatalf("failed to create kube client: %v", err) - } - - // curl execs a Curl command in the test client pod and returns an error - // value. The Curl command uses the specified certificate from the - // client certificates configmap, sends a request for the canary route - // via the router's internal service, and returns an error if the Curl - // command fails or the HTTP response status code indicates an error. - curl := func(cert string) error { - req := cl.CoreV1().RESTClient().Post().Resource("pods"). - Namespace(clientPod.Namespace).Name(clientPod.Name). - SubResource("exec"). - Param("container", clientPod.Spec.Containers[0].Name) - cmd := []string{ - "/bin/curl", "-k", "-v", - "-w", "%{http_code}", - "--retry", "10", "--retry-delay", "1", - } - if len(cert) != 0 { - cmd = append(cmd, - "--cert", fmt.Sprintf("/tmp/tls/%s.crt", cert), - "--key", fmt.Sprintf("/tmp/tls/%s.key", cert), - ) - } - cmd = append(cmd, "--resolve", - fmt.Sprintf("%s:443:%s", route.Spec.Host, - service.Spec.ClusterIP), - fmt.Sprintf("https://%s", route.Spec.Host), - ) - req.VersionedParams(&corev1.PodExecOptions{ - Container: "curl", - Command: cmd, - Stdout: true, - Stderr: true, - }, scheme.ParameterCodec) - exec, err := remotecommand.NewSPDYExecutor(kubeConfig, "POST", req.URL()) - if err != nil { - return err - } - var stdout, stderr bytes.Buffer - err = exec.Stream(remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, - }) - stdoutStr := stdout.String() - t.Logf("command: %s\nstdout:\n%s\n\nstderr:\n%s\n", - strings.Join(cmd, " "), stdoutStr, stderr.String()) - if err != nil { - return err - } - httpStatusCode := stdoutStr[len(stdoutStr)-3:] - switch string(httpStatusCode[0]) { - case "0", "4", "5": - return fmt.Errorf("got HTTP %s status code", httpStatusCode) - default: - return nil - } - } - optionalPolicyTestCases := []struct { description string cert string @@ -328,7 +260,7 @@ func TestClientTLS(t *testing.T) { expectAllowed: false, }} for _, tc := range optionalPolicyTestCases { - err := curl(tc.cert) + _, err := curlGetStatusCode(t, clientPod, tc.cert, route.Spec.Host, service.Spec.ClusterIP, true) if err == nil && !tc.expectAllowed { t.Errorf("%q: expected error, got success", tc.description) } @@ -349,8 +281,8 @@ func TestClientTLS(t *testing.T) { if err := waitForDeploymentEnvVar(t, kclient, deployment, 1*time.Minute, "ROUTER_MUTUAL_TLS_AUTH", "required"); err != nil { t.Fatalf("expected updated deployment to have ROUTER_MUTUAL_TLS_AUTH=required: %v", err) } - if err := waitForDeploymentComplete(t, kclient, deployment, 3*time.Minute); err != nil { - t.Fatalf("failed to observe expected conditions: %v", err) + if err := waitForDeploymentCompleteWithOldPodTermination(t, kclient, deploymentName, 3*time.Minute); err != nil { + t.Fatalf("timed out waiting for old router generation to be cleaned up: %v", err) } requiredPolicyTestCases := []struct { @@ -374,7 +306,7 @@ func TestClientTLS(t *testing.T) { expectAllowed: false, }} for _, tc := range requiredPolicyTestCases { - err := curl(tc.cert) + _, err := curlGetStatusCode(t, clientPod, tc.cert, route.Spec.Host, service.Spec.ClusterIP, true) if err == nil && !tc.expectAllowed { t.Errorf("%q: expected error, got success", tc.description) } @@ -384,6 +316,1146 @@ func TestClientTLS(t *testing.T) { } } +// TestMTLSWithCRLsCerts includes all the certificates needed for a particular test case of TestMTLSWithCRLs. +type TestMTLSWithCRLsCerts struct { + // CABundle is the complete PEM-encoded list of certificates that will be used both by HAProxy for client + // validation, and by openshift-router for CRL distribution points. + CABundle []string + // CRLs is a map of the PEM-encoded CRLs, indexed by the filename that will be used in the CRL host pod's configmap. + CRLs map[string]string + // ClientCerts contains maps of the client certificates used to verify that mTLS is working as intended. + ClientCerts struct { + // Accepted is a map containing the client keys and certificates that should be able to connect to backends + // successfully, indexed by a unique name. + Accepted map[string]KeyCert + // Rejected is a map containing the client keys and certificates that should NOT be able to connect to backends, + // indexed by a unique name. + Rejected map[string]KeyCert + } +} + +// TestMTLSWithCRLs verifies that mTLS works when the client auth chain includes certificate revocation lists (CRLs). +func TestMTLSWithCRLs(t *testing.T) { + t.Parallel() + namespaceName := names.SimpleNameGenerator.GenerateName("mtls-with-crls") + crlHostName := types.NamespacedName{ + Name: "crl-host", + Namespace: namespaceName, + } + // When generating certificates, the CRL distribution points need to be specified by URL + crlHostServiceName := "crl-host-service" + crlHostURL := crlHostServiceName + "." + crlHostName.Namespace + ".svc" + testCases := []struct { + // Name is the name of the test case. + Name string + // CreateCerts generates the certificates for the test case. Certificates and CRLs must not have expired at the + // time of the run, so they must be generated at runtime. + CreateCerts func() TestMTLSWithCRLsCerts + }{ + { + // This test case has CA certificates including a CRL distribution point (CDP) for the CRL that they + // generate and sign. This is the default way to distribute CRLs according to RFC-5280 + // + // CA Bundle: + // - Intermediate CA + // - Includes CRL distribution point for intermediate-ca.crl + // - Signed by Root CA. + // - Root CA + // - Includes CRL distribution point for root-ca.crl + // - Self signed. + // + // Client Certificates: + // - signed-by-root + // - Signed by Root CA. + // - Should successfully connect. + // - signed-by-intermediate + // - Signed by Intermediate CA. + // - Should successfully connect. + // - revoked-by-root + // - Signed by Root CA. + // - Has been revoked. + // - Should be rejected due to revocation. + // - revoked-by-intermediate + // - Signed by Intermediate CA + // - Has been revoked + // - Should be rejected due to revocation. + // - self-signed + // - Self signed + // - Should be rejected because it's not signed by any trusted CA. + Name: "certificate-distributes-its-own-crl", + CreateCerts: func() TestMTLSWithCRLsCerts { + rootCDP := "http://" + crlHostURL + "/root/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate/intermediate.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &rootCA) + + signedByRoot := MustCreateTLSKeyCert("client signed by root", time.Now(), time.Now().Add(24*time.Hour), false, nil, &rootCA) + signedByIntermediate := MustCreateTLSKeyCert("client signed by intermediate", time.Now(), time.Now().Add(24*time.Hour), false, nil, &intermediateCA) + revokedByRoot := MustCreateTLSKeyCert("client revoked by root", time.Now(), time.Now().Add(24*time.Hour), false, nil, &rootCA) + revokedByIntermediate := MustCreateTLSKeyCert("client revoked by intermediate", time.Now(), time.Now().Add(24*time.Hour), false, nil, &intermediateCA) + selfSigned := MustCreateTLSKeyCert("self signed cert", time.Now(), time.Now().Add(24*time.Hour), false, nil, nil) + + _, rootCRLPem := MustCreateCRL(nil, rootCA, time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedByRoot)) + _, intermediateCRLPem := MustCreateCRL(nil, intermediateCA, time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedByIntermediate)) + + return TestMTLSWithCRLsCerts{ + CABundle: []string{ + intermediateCA.CertPem, + rootCA.CertPem, + }, + CRLs: map[string]string{ + "root": rootCRLPem, + "intermediate": intermediateCRLPem, + }, + ClientCerts: struct { + Accepted map[string]KeyCert + Rejected map[string]KeyCert + }{ + Accepted: map[string]KeyCert{ + "signed-by-root": signedByRoot, + "signed-by-intermediate": signedByIntermediate, + }, + Rejected: map[string]KeyCert{ + "revoked-by-root": revokedByRoot, + "revoked-by-intermediate": revokedByIntermediate, + "self-signed": selfSigned, + }, + }, + } + }, + }, + { + // This test case has certificates including the CRL distribution point of their signer (i.e. intermediate + // CA is signed by root CA, and includes the URL for root's CRL). In this case, neither of the certificates + // in the CA bundle include the intermediate CRL, so connections that rely on it will be rejected. + // + // CA Bundle: + // - Intermediate CA + // - Includes CRL distribution point for root-ca.crl + // - Signed by Root CA. + // - Root CA + // - No CRL distribution point. + // - Self signed. + // + // Note that intermediate-ca.crl is not present in the CA bundle. + // + // Client Certificates: + // - signed-by-root + // - Includes CRL distribution point for root-ca.crl + // - Signed by Root CA. + // - Should successfully connect. + // - signed-by-intermediate + // - Includes CRL distribution point for intermediate-ca.crl + // - Signed by Intermediate CA. + // - Should be rejected because HAProxy doesn't have intermediate-ca.crl (SSL error "unknown ca"). + // - revoked-by-root + // - Includes CRL distribution point for root-ca.crl + // - Signed by Root CA. + // - Has been revoked. + // - Should be rejected due to revocation. + // - revoked-by-intermediate + // - Includes CRL distribution point for intermediate-ca.crl + // - Signed by Intermediate CA + // - Has been revoked + // - Should be rejected because HAProxy doesn't have intermediate-ca.crl (SSL error "unknown ca"). + // - self-signed + // - Self signed + // - Should be rejected because it's not signed by any trusted CA (SSL error "unknown ca"). + Name: "certificate-distributes-its-signers-crl", + CreateCerts: func() TestMTLSWithCRLsCerts { + rootCDP := "http://" + crlHostURL + "/root/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate/intermediate.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, nil, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + + signedByRoot := MustCreateTLSKeyCert("client signed by root", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + signedByIntermediate := MustCreateTLSKeyCert("client signed by intermediate", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &intermediateCA) + revokedByRoot := MustCreateTLSKeyCert("client revoked by root", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + revokedByIntermediate := MustCreateTLSKeyCert("client revoked by intermediate", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &intermediateCA) + selfSigned := MustCreateTLSKeyCert("self signed cert", time.Now(), time.Now().Add(24*time.Hour), false, nil, nil) + + _, rootCRLPem := MustCreateCRL(nil, rootCA, time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedByRoot)) + _, intermediateCRLPem := MustCreateCRL(nil, intermediateCA, time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedByIntermediate)) + + return TestMTLSWithCRLsCerts{ + CABundle: []string{ + intermediateCA.CertPem, + rootCA.CertPem, + }, + CRLs: map[string]string{ + "root": rootCRLPem, + "intermediate": intermediateCRLPem, + }, + ClientCerts: struct { + Accepted map[string]KeyCert + Rejected map[string]KeyCert + }{ + Accepted: map[string]KeyCert{ + "signed-by-root": signedByRoot, + }, + Rejected: map[string]KeyCert{ + "signed-by-intermediate": signedByIntermediate, + "revoked-by-root": revokedByRoot, + "revoked-by-intermediate": revokedByIntermediate, + "self-signed": selfSigned, + }, + }, + } + }, + }, + { + // This test case has certificates including the CRL distribution point of their signer. In this case, a + // leaf (client) certificate is included in the CA bundle so that openshift-router is aware of the + // intermediate CRL's distribution point, so certificates signed by intermediate will work. + // TODO: update this test case when RFE-3605 or a similar fix is implemented + // + // CA Bundle: + // - revoked-by-intermediate + // - Includes CRL distribution point for intermediate-ca.crl + // - Signed by Intermediate CA + // - Intermediate CA + // - Includes CRL distribution point for root-ca.crl + // - Signed by Root CA. + // - Root CA + // - No CRL distribution point. + // - Self signed. + // + // Including revoked-by-intermediate in the CA bundle is the "workaround" in the test name. It makes sure + // intermediate-ca.crl is listed in the CA bundle, so openshift-router knows to download it. + // + // Client Certificates: + // - signed-by-root + // - Includes CRL distribution point for root-ca.crl + // - Signed by Root CA. + // - Should successfully connect. + // - signed-by-intermediate + // - Includes CRL distribution point for intermediate-ca.crl + // - Signed by Intermediate CA. + // - Should successfully connect. + // - revoked-by-root + // - Includes CRL distribution point for root-ca.crl + // - Signed by Root CA. + // - Has been revoked. + // - Should be rejected due to revocation. + // - revoked-by-intermediate + // - Includes CRL distribution point for intermediate-ca.crl + // - Signed by Intermediate CA + // - Has been revoked + // - Should be rejected due to revocation. + // - self-signed + // - Self signed + // - Should be rejected because it's not signed by any trusted CA (SSL error "unknown ca"). + Name: "certificate-distributes-its-signers-crl-with-workaround", + CreateCerts: func() TestMTLSWithCRLsCerts { + rootCDP := "http://" + crlHostURL + "/root/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate/intermediate.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, nil, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + + signedByRoot := MustCreateTLSKeyCert("client signed by root", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + signedByIntermediate := MustCreateTLSKeyCert("client signed by intermediate", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &intermediateCA) + revokedByRoot := MustCreateTLSKeyCert("client revoked by root", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + revokedByIntermediate := MustCreateTLSKeyCert("client revoked by intermediate", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &intermediateCA) + selfSigned := MustCreateTLSKeyCert("self signed cert", time.Now(), time.Now().Add(24*time.Hour), false, nil, nil) + + _, rootCRLPem := MustCreateCRL(nil, rootCA, time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedByRoot)) + _, intermediateCRLPem := MustCreateCRL(nil, intermediateCA, time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedByIntermediate)) + + return TestMTLSWithCRLsCerts{ + CABundle: []string{ + revokedByIntermediate.CertPem, + intermediateCA.CertPem, + rootCA.CertPem, + }, + CRLs: map[string]string{ + "root": rootCRLPem, + "intermediate": intermediateCRLPem, + }, + ClientCerts: struct { + Accepted map[string]KeyCert + Rejected map[string]KeyCert + }{ + Accepted: map[string]KeyCert{ + "signed-by-root": signedByRoot, + "signed-by-intermediate": signedByIntermediate, + }, + Rejected: map[string]KeyCert{ + "revoked-by-root": revokedByRoot, + "revoked-by-intermediate": revokedByIntermediate, + "self-signed": selfSigned, + }, + }, + } + }, + }, + { + // large-crl verifies that CRLs larger than 1MB can be used. This tests the fix for OCPBUGS-6661 + Name: "large-crl", + CreateCerts: func() TestMTLSWithCRLsCerts { + maxDummyRevokedSerialNumber := 25000 + rootCDP := "http://" + crlHostURL + "/root/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate/intermediate.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &rootCA) + + signedByRoot := MustCreateTLSKeyCert("client signed by root", time.Now(), time.Now().Add(24*time.Hour), false, nil, &rootCA) + signedByIntermediate := MustCreateTLSKeyCert("client signed by intermediate", time.Now(), time.Now().Add(24*time.Hour), false, nil, &intermediateCA) + revokedByRoot := MustCreateTLSKeyCert("client revoked by root", time.Now(), time.Now().Add(24*time.Hour), false, nil, &rootCA) + revokedByIntermediate := MustCreateTLSKeyCert("client revoked by intermediate", time.Now(), time.Now().Add(24*time.Hour), false, nil, &intermediateCA) + selfSigned := MustCreateTLSKeyCert("self signed cert", time.Now(), time.Now().Add(24*time.Hour), false, nil, nil) + + // Generate a set of CRL files that are larger than 1MB by revoking a large number of certificates. The + // revocation list only includes the serial number of the certificate and the time of revocation, so we + // don't actually need to generate real certificates, just some serial numbers. We can also repeat the + // same serial numbers in each CRL, cutting down on the number we need to generate by half + revokedCerts := []pkix.RevokedCertificate{} + for i := int64(1); i <= int64(maxDummyRevokedSerialNumber); i++ { + serialNumber := big.NewInt(i) + // It's highly unlikely that any of the certs we explicitly generated have serial numbers less than + // maxDummyRevokedSerialNumber since they're 20-byte UUIDs, but is possible. In the unlikely event + // that there's some overlap, don't include those serial numbers in the initial list of "revoked" + // certs. + switch { + case signedByRoot.Cert.SerialNumber.Cmp(serialNumber) == 0: + continue + case revokedByRoot.Cert.SerialNumber.Cmp(serialNumber) == 0: + continue + case signedByIntermediate.Cert.SerialNumber.Cmp(serialNumber) == 0: + continue + case revokedByIntermediate.Cert.SerialNumber.Cmp(serialNumber) == 0: + continue + } + revokedCerts = append(revokedCerts, pkix.RevokedCertificate{ + SerialNumber: serialNumber, + RevocationTime: time.Now(), + }) + } + + rootRevokedCerts := make([]pkix.RevokedCertificate, len(revokedCerts)) + copy(rootRevokedCerts, revokedCerts) + rootRevokedCerts = append(rootRevokedCerts, pkix.RevokedCertificate{ + SerialNumber: revokedByRoot.Cert.SerialNumber, + RevocationTime: time.Now(), + }) + _, rootCRLPem := MustCreateCRL(nil, rootCA, time.Now(), time.Now().Add(1*time.Hour), rootRevokedCerts) + + intermediateRevokedCerts := make([]pkix.RevokedCertificate, len(revokedCerts)) + copy(intermediateRevokedCerts, revokedCerts) + intermediateRevokedCerts = append(intermediateRevokedCerts, pkix.RevokedCertificate{ + SerialNumber: revokedByIntermediate.Cert.SerialNumber, + RevocationTime: time.Now(), + }) + _, intermediateCRLPem := MustCreateCRL(nil, intermediateCA, time.Now(), time.Now().Add(1*time.Hour), intermediateRevokedCerts) + t.Logf("Root CRL Size: %dKB\nIntermediate CRL Size: %dKB\nTotal Size: %dKB", len(rootCRLPem)/1024, len(intermediateCRLPem)/1024, (len(rootCRLPem)+len(intermediateCRLPem))/1024) + + return TestMTLSWithCRLsCerts{ + CABundle: []string{ + intermediateCA.CertPem, + rootCA.CertPem, + }, + CRLs: map[string]string{ + "root": rootCRLPem, + "intermediate": intermediateCRLPem, + }, + ClientCerts: struct { + Accepted map[string]KeyCert + Rejected map[string]KeyCert + }{ + Accepted: map[string]KeyCert{ + "signed-by-root": signedByRoot, + "signed-by-intermediate": signedByIntermediate, + }, + Rejected: map[string]KeyCert{ + "revoked-by-root": revokedByRoot, + "revoked-by-intermediate": revokedByIntermediate, + "self-signed": selfSigned, + }, + }, + } + }, + }, + { + // multiple-intermediate-ca tests that more than 2 CAs can be used. Each CA lists its own CRL's distribution point. + Name: "multiple-intermediate-ca", + CreateCerts: func() TestMTLSWithCRLsCerts { + CANames := []string{ + "root", + "foo", + "bar", + "baz", + "quux", + } + caCerts := map[string]KeyCert{} + acceptedClientCerts := map[string]KeyCert{} + rejectedClientCerts := map[string]KeyCert{} + crls := map[string]string{} + caBundle := []string{} + + for i, name := range CANames { + crlDistributionPoint := "http://" + crlHostURL + "/" + name + "/" + name + ".crl" + caCert := KeyCert{} + if i == 0 { + // i = 0 is the root certificate, so it's self signed. + caCert = MustCreateTLSKeyCert(name, time.Now(), time.Now().Add(24*time.Hour), true, []string{crlDistributionPoint}, nil) + } else { + // Non-root certificates are signed by the previous CA in the list. + signer := caCerts[CANames[i-1]] + caCert = MustCreateTLSKeyCert(name, time.Now(), time.Now().Add(24*time.Hour), true, []string{crlDistributionPoint}, &signer) + } + caCerts[name] = caCert + caBundle = append(caBundle, caCerts[name].CertPem) + + // For each CA, generate 1 cert that will be accepted, and 1 that will be revoked (and therefore rejected). + acceptedCert := MustCreateTLSKeyCert("client signed by "+name, time.Now(), time.Now().Add(24*time.Hour), false, nil, &caCert) + revokedCert := MustCreateTLSKeyCert("client revoked by "+name, time.Now(), time.Now().Add(24*time.Hour), false, nil, &caCert) + _, crls[name] = MustCreateCRL(nil, caCerts[name], time.Now(), time.Now().Add(1*time.Hour), RevokeCertificates(time.Now(), revokedCert)) + acceptedClientCerts["signed-by-"+name] = acceptedCert + rejectedClientCerts["revoked-by-"+name] = revokedCert + } + + // In addition to the certificates for each CA, include a self-signed certificate to make sure it's rejected. + rejectedClientCerts["self-signed"] = MustCreateTLSKeyCert("self signed cert", time.Now(), time.Now().Add(24*time.Hour), false, nil, nil) + + return TestMTLSWithCRLsCerts{ + CABundle: caBundle, + CRLs: crls, + ClientCerts: struct { + Accepted map[string]KeyCert + Rejected map[string]KeyCert + }{ + Accepted: acceptedClientCerts, + Rejected: rejectedClientCerts, + }, + } + }, + }, + } + + namespace := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + if err := kclient.Create(context.TODO(), &namespace); err != nil { + t.Fatalf("Failed to create namespace %q: %v", namespace.Name, err) + } + defer assertDeletedWaitForCleanup(t, kclient, &namespace) + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + tcCerts := tc.CreateCerts() + // Get the URL path of one of the CRLs to use in the CRL host pod's readiness probe. + readinessProbePath := "" + for crlName := range tcCerts.CRLs { + readinessProbePath = fmt.Sprintf("/%s/%s.crl", crlName, crlName) + break + } + // Create a pod which will host the CRLs. + crlHostPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: crlHostName.Name, + Namespace: namespace.Name, + Labels: map[string]string{"app": crlHostName.Name}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "httpd", + Image: "quay.io/centos7/httpd-24-centos7", + Ports: []corev1.ContainerPort{{ + ContainerPort: 8080, + Name: "http-svc", + }}, + SecurityContext: generateUnprivilegedSecurityContext(), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: readinessProbePath, + Port: intstr.IntOrString{ + Type: intstr.String, + StrVal: "http-svc", + }, + }, + }, + }, + }}, + }, + } + for name, crl := range tcCerts.CRLs { + crlConfigMapName := name + "-crl" + crlConfigMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: crlConfigMapName, + Namespace: namespace.Name, + }, + Data: map[string]string{ + name + ".crl": crl, + }, + } + crlHostPod.Spec.Volumes = append(crlHostPod.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: crlConfigMap.Name, + }, + }, + }, + }) + crlHostPod.Spec.Containers[0].VolumeMounts = append(crlHostPod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: name, + MountPath: filepath.Join("/var/www/html", name), + ReadOnly: true, + }) + + if err := kclient.Create(context.TODO(), &crlConfigMap); err != nil { + t.Fatalf("Failed to create configmap %q: %v", crlConfigMap.Name, err) + } + defer assertDeleted(t, kclient, &crlConfigMap) + } + + if err := kclient.Create(context.TODO(), &crlHostPod); err != nil { + t.Fatalf("Failed to create pod %q: %v", crlHostPod.Name, err) + } + // the crlHostPod is one of the first resources to be created, and one of the last to be deleted thanks to + // defer stack ordering. calling assertDeletedWaitForCleanup here makes sure that the test case doesn't + // finish until it's fully cleaned up, so when the next test case creates its own version of crlHostPod, it + // won't be clashing. As of this writing, the other resources are normally cleaned up before the next test + // case comes through and creates a new one, but if that stops being true in the future, their assertDeleted + // calls may need to be replaced by the slower assertDeletedWaitForCleanup option. + defer assertDeletedWaitForCleanup(t, kclient, &crlHostPod) + crlHostService := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: crlHostServiceName, + Namespace: namespace.Name, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": crlHostName.Name}, + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 80, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("http-svc"), + }}, + }, + } + if err := kclient.Create(context.TODO(), &crlHostService); err != nil { + t.Fatalf("Failed to create service %q: %v", crlHostService.Name, err) + } + defer assertDeleted(t, kclient, &crlHostService) + // Wait for CRL host to be ready. + err := wait.PollImmediate(2*time.Second, 3*time.Minute, func() (bool, error) { + if err := kclient.Get(context.TODO(), crlHostName, &crlHostPod); err != nil { + t.Logf("error getting pod %s/%s: %v", crlHostName.Namespace, crlHostName.Name, err) + return false, nil + } + for _, condition := range crlHostPod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) + // Create CA cert bundle + clientCAConfigmapName := "client-ca-cm-" + namespace.Name + clientCAConfigmap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientCAConfigmapName, + Namespace: "openshift-config", + }, + Data: map[string]string{ + "ca-bundle.pem": strings.Join(tcCerts.CABundle, "\n"), + }, + } + if err := kclient.Create(context.TODO(), &clientCAConfigmap); err != nil { + t.Fatalf("Failed to create CA cert configmap: %v", err) + } + defer assertDeleted(t, kclient, &clientCAConfigmap) + icName := types.NamespacedName{ + Name: "mtls-with-crls", + Namespace: operatorNamespace, + } + icDomain := icName.Name + "." + dnsConfig.Spec.BaseDomain + ic := newPrivateController(icName, icDomain) + ic.Spec.ClientTLS = operatorv1.ClientTLS{ + ClientCA: configv1.ConfigMapNameReference{ + Name: clientCAConfigmapName, + }, + ClientCertificatePolicy: operatorv1.ClientCertificatePolicyRequired, + } + if err := kclient.Create(context.TODO(), ic); err != nil { + t.Fatalf("failed to create ingresscontroller %s: %v", icName, err) + } + defer assertIngressControllerDeleted(t, kclient, ic) + + if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil { + t.Fatalf("failed to observe expected conditions: %v", err) + } + + // The client pod will need the client certificates we generated, so create a configmap with all the client + // certificates and keys, and mount that to the client pod. + clientCertsConfigmap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "client-certificates", + Namespace: namespace.Name, + }, + Data: map[string]string{}, + } + for name, keyCert := range tcCerts.ClientCerts.Accepted { + clientCertsConfigmap.Data[name+".key"] = encodeKey(keyCert.Key) + clientCertsConfigmap.Data[name+".pem"] = keyCert.CertFullChain + } + for name, keyCert := range tcCerts.ClientCerts.Rejected { + clientCertsConfigmap.Data[name+".key"] = encodeKey(keyCert.Key) + clientCertsConfigmap.Data[name+".pem"] = keyCert.CertFullChain + } + if err := kclient.Create(context.TODO(), &clientCertsConfigmap); err != nil { + t.Fatalf("failed to create configmap %q: %v", clientCertsConfigmap.Name, err) + } + defer assertDeleted(t, kclient, &clientCertsConfigmap) + + // Use the router image for the exec pod since it has curl. + routerDeployment := &appsv1.Deployment{} + routerDeploymentName := controller.RouterDeploymentName(ic) + if err := kclient.Get(context.TODO(), routerDeploymentName, routerDeployment); err != nil { + t.Fatalf("failed to get routerDeployment %q: %v", routerDeploymentName, err) + } + + podName := "mtls-with-crls-client" + image := routerDeployment.Spec.Template.Spec.Containers[0].Image + clientPod := buildExecPod(podName, namespace.Name, image) + clientPod.Spec.Volumes = []corev1.Volume{{ + Name: "client-certificates", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: clientCertsConfigmap.Name, + }, + }, + }, + }} + clientPod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{ + Name: "client-certificates", + MountPath: "/tmp/tls/", + ReadOnly: true, + }} + clientPodName := types.NamespacedName{ + Name: clientPod.Name, + Namespace: clientPod.Namespace, + } + if err := kclient.Create(context.TODO(), clientPod); err != nil { + t.Fatalf("failed to create pod %q: %v", clientPodName, err) + } + defer assertDeleted(t, kclient, clientPod) + + err = wait.PollImmediate(2*time.Second, 3*time.Minute, func() (bool, error) { + if err := kclient.Get(context.TODO(), clientPodName, clientPod); err != nil { + t.Logf("failed to get client pod %q: %v", clientPodName, err) + return false, nil + } + for _, cond := range clientPod.Status.Conditions { + if cond.Type == corev1.PodReady { + return cond.Status == corev1.ConditionTrue, nil + } + } + return false, nil + }) + if err != nil { + t.Fatalf("timed out waiting for pod %q to become ready: %v", clientPodName, err) + } + + // Wait until the CRLs are downloaded + podList := &corev1.PodList{} + labels := map[string]string{ + controller.ControllerDeploymentLabel: icName.Name, + } + if err := kclient.List(context.TODO(), podList, client.InNamespace("openshift-ingress"), client.MatchingLabels(labels)); err != nil { + t.Logf("failed to list pods for ingress controllers %s: %v", ic.Name, err) + } + if len(podList.Items) == 0 { + t.Fatalf("no router pods found for ingresscontroller %s: %v", ic.Name, err) + } + routerPod := podList.Items[0] + err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) { + // Get the current CRL file from the router container + cmd := []string{"cat", "/var/lib/haproxy/mtls/latest/crls.pem"} + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + if err := podExec(t, routerPod, &stdout, &stderr, cmd); err != nil { + t.Logf("exec %q failed. error: %v\nstdout:\n%s\nstderr:\n%s", cmd, err, stdout.String(), stderr.String()) + return false, err + } + // Parse the first CRL. If CRLs haven't been downloaded yet, it will be the placeholder CRL. + block, _ := pem.Decode(stdout.Bytes()) + crl, err := x509.ParseRevocationList(block.Bytes) + if err != nil { + return false, fmt.Errorf("invalid CRL: %v", err) + } + return crl.Issuer.CommonName != "Placeholder CA", nil + }) + + // Get the canary route to use as the target for curl. + route := &routev1.Route{} + routeName := controller.CanaryRouteName() + err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) { + if err := kclient.Get(context.TODO(), routeName, route); err != nil { + t.Logf("failed to get route %q: %v", routeName, err) + return false, nil + } + return true, nil + }) + if err != nil { + t.Fatalf("failed to observe route %q: %v", routeName, err) + } + + // If the canary route is used, normally the default ingress controller will handle the request, but by + // using curl's --resolve flag, we can send an HTTP request intended for the canary pod directly to our + // ingress controller instead. In order to do that, we need the ingress controller's service IP. + service := &corev1.Service{} + serviceName := controller.InternalIngressControllerServiceName(ic) + if err := kclient.Get(context.TODO(), serviceName, service); err != nil { + t.Fatalf("failed to get service %q: %v", serviceName, err) + } + + for certName := range tcCerts.ClientCerts.Accepted { + if _, err := curlGetStatusCode(t, clientPod, certName, route.Spec.Host, service.Spec.ClusterIP, false); err != nil { + t.Errorf("Failed to curl route with cert %q: %v", certName, err) + } + } + for certName := range tcCerts.ClientCerts.Rejected { + if httpStatusCode, err := curlGetStatusCode(t, clientPod, certName, route.Spec.Host, service.Spec.ClusterIP, false); err != nil { + if httpStatusCode == 0 { + // TLS/SSL verification failures result in a 0 http status code (no connection is made to the backend, so no http status code is returned). + continue + } + t.Errorf("Unexpected error from curl for cert %q: %v", certName, err) + } else { + t.Errorf("Expected curl route with cert %q to fail but succeeded", certName) + } + } + }) + } +} + +// TestCRLUpdate verifies that CRLs are updated when they expire +func TestCRLUpdate(t *testing.T) { + t.Parallel() + testName := names.SimpleNameGenerator.GenerateName("crl-update") + crlHostName := types.NamespacedName{ + Name: "crl-host", + Namespace: testName, + } + // When generating certificates, the CRL distribution points need to be specified by URL + crlHostServiceName := "crl-host-service" + crlHostURL := crlHostServiceName + "." + crlHostName.Namespace + ".svc" + namespace := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testName, + }, + } + if err := kclient.Create(context.TODO(), &namespace); err != nil { + t.Fatalf("Failed to create namespace %q: %v", namespace.Name, err) + } + defer assertDeleted(t, kclient, &namespace) + testCases := []struct { + // Test case name + Name string + // Function to generate the necessary certificates. These will be put into the ingresscontroller's client CA + // bundle, and will be used to generate CRLs during the test. + CreateCerts func() map[string]KeyCert + // The names of the CAs whose CRLs are expected to be downloaded by the router pod. Only the certs with the + // corresponding names will be used to generate CRLs + ExpectedCRLs []string + }{ + { + Name: "certificate-distributes-its-own-crl", + CreateCerts: func() map[string]KeyCert { + rootCDP := "http://" + crlHostURL + "/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &rootCA) + return map[string]KeyCert{ + "root": rootCA, + "intermediate": intermediateCA, + } + }, + ExpectedCRLs: []string{ + "root", + "intermediate", + }, + }, + { + Name: "certificate-distributes-its-signers-crl", + CreateCerts: func() map[string]KeyCert { + rootCDP := "http://" + crlHostURL + "/root.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{}, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + return map[string]KeyCert{ + "root": rootCA, + "intermediate": intermediateCA, + } + }, + ExpectedCRLs: []string{ + "root", + }, + }, + { + Name: "certificate-distributes-its-signers-crl-with-workaround", + CreateCerts: func() map[string]KeyCert { + rootCDP := "http://" + crlHostURL + "/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{}, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + client := MustCreateTLSKeyCert("workaround client cert", time.Now(), time.Now().Add(24*time.Hour), false, []string{intermediateCDP}, &intermediateCA) + return map[string]KeyCert{ + "root": rootCA, + "intermediate": intermediateCA, + "client": client, + } + }, + ExpectedCRLs: []string{ + "root", + "intermediate", + }, + }, + { + Name: "many-CAs-with-signers-crl-workaround", + CreateCerts: func() map[string]KeyCert { + rootCDP := "http://" + crlHostURL + "/root.crl" + intermediateCDP := "http://" + crlHostURL + "/intermediate.crl" + fooCDP := "http://" + crlHostURL + "/foo.crl" + barCDP := "http://" + crlHostURL + "/bar.crl" + + rootCA := MustCreateTLSKeyCert("testing root CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{}, nil) + intermediateCA := MustCreateTLSKeyCert("testing intermediate CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{rootCDP}, &rootCA) + fooCA := MustCreateTLSKeyCert("testing foo CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{intermediateCDP}, &intermediateCA) + barCA := MustCreateTLSKeyCert("testing bar CA", time.Now(), time.Now().Add(24*time.Hour), true, []string{fooCDP}, &fooCA) + client := MustCreateTLSKeyCert("workaround client cert", time.Now(), time.Now().Add(24*time.Hour), false, []string{barCDP}, &barCA) + return map[string]KeyCert{ + "root": rootCA, + "intermediate": intermediateCA, + "foo": fooCA, + "bar": barCA, + "client": client, + } + }, + ExpectedCRLs: []string{ + "root", + "intermediate", + "foo", + "bar", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + caCerts := tc.CreateCerts() + // Generate CRLs. Offset the expiration times by 1 minute each so that we can verify that only the correct CRLs get updated at each expiration + currentCRLs := map[string]*x509.RevocationList{} + crlPems := map[string]string{} + caBundle := []string{} + validTime := 3 * time.Minute + //expirations := []time.Time{} + for _, caName := range tc.ExpectedCRLs { + currentCRLs[caName], crlPems[caName+".crl"] = MustCreateCRL(nil, caCerts[caName], time.Now(), time.Now().Add(validTime), nil) + validTime += time.Minute + } + for _, caCert := range caCerts { + caBundle = append(caBundle, caCert.CertPem) + } + // Create a pod which will host the CRLs. + crlConfigMapName := crlHostName.Name + "-configmap" + crlConfigMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: crlConfigMapName, + Namespace: namespace.Name, + }, + Data: crlPems, + } + if err := kclient.Create(context.TODO(), &crlConfigMap); err != nil { + t.Fatalf("Failed to create configmap %q: %v", crlConfigMap.Name, err) + } + defer assertDeleted(t, kclient, &crlConfigMap) + crlHostPod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: crlHostName.Name, + Namespace: namespace.Name, + Labels: map[string]string{"app": crlHostName.Name}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "httpd", + Image: "quay.io/centos7/httpd-24-centos7", + Ports: []corev1.ContainerPort{{ + ContainerPort: 8080, + Name: "http-svc", + }}, + VolumeMounts: []corev1.VolumeMount{{ + Name: "data", + MountPath: "/var/www/html", + ReadOnly: true, + }}, + SecurityContext: generateUnprivilegedSecurityContext(), + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: fmt.Sprintf("/%s.crl", tc.ExpectedCRLs[0]), + Port: intstr.IntOrString{ + Type: intstr.String, + StrVal: "http-svc", + }, + }, + }, + }, + }}, + Volumes: []corev1.Volume{{ + Name: "data", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: crlConfigMap.Name, + }, + }, + }, + }}, + }, + } + + if err := kclient.Create(context.TODO(), &crlHostPod); err != nil { + t.Fatalf("Failed to create pod %q: %v", crlHostPod.Name, err) + } + // the crlHostPod is one of the first resources to be created, and one of the last to be deleted thanks to + // defer stack ordering. calling assertDeletedWaitForCleanup here makes sure that the test case doesn't + // finish until it's fully cleaned up, so when the next test case creates its own version of crlHostPod, it + // won't be clashing. As of this writing, the other resources are normally cleaned up before the next test + // case comes through and creates a new one, but if that stops being true in the future, their assertDeleted + // calls may need to be replaced by the slower assertDeletedWaitForCleanup option. + defer assertDeletedWaitForCleanup(t, kclient, &crlHostPod) + crlHostService := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: crlHostServiceName, + Namespace: namespace.Name, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": crlHostName.Name}, + Ports: []corev1.ServicePort{{ + Name: "http", + Port: 80, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("http-svc"), + }}, + }, + } + if err := kclient.Create(context.TODO(), &crlHostService); err != nil { + t.Fatalf("Failed to create service %q: %v", crlHostService.Name, err) + } + defer assertDeleted(t, kclient, &crlHostService) + // Wait for CRL host to be ready. + err := wait.PollImmediate(2*time.Second, 3*time.Minute, func() (bool, error) { + if err := kclient.Get(context.TODO(), crlHostName, &crlHostPod); err != nil { + t.Logf("error getting pod %s/%s: %v", crlHostName.Namespace, crlHostName.Name, err) + return false, nil + } + for _, condition := range crlHostPod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) + // Create CA cert bundle. + clientCAConfigmapName := "client-ca-cm-" + namespace.Name + clientCAConfigmap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientCAConfigmapName, + Namespace: "openshift-config", + }, + Data: map[string]string{ + "ca-bundle.pem": strings.Join(caBundle, "\n"), + }, + } + if err := kclient.Create(context.TODO(), &clientCAConfigmap); err != nil { + t.Fatalf("Failed to create CA cert configmap: %v", err) + } + defer assertDeleted(t, kclient, &clientCAConfigmap) + icName := types.NamespacedName{ + Name: testName, + Namespace: operatorNamespace, + } + icDomain := icName.Name + "." + dnsConfig.Spec.BaseDomain + ic := newPrivateController(icName, icDomain) + ic.Spec.ClientTLS = operatorv1.ClientTLS{ + ClientCA: configv1.ConfigMapNameReference{ + Name: clientCAConfigmapName, + }, + ClientCertificatePolicy: operatorv1.ClientCertificatePolicyRequired, + } + if err := kclient.Create(context.TODO(), ic); err != nil { + t.Fatalf("failed to create ingresscontroller %s/%s: %v", icName.Namespace, icName.Name, err) + } + defer assertIngressControllerDeleted(t, kclient, ic) + + if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil { + t.Fatalf("failed to observe expected conditions: %v", err) + } + + deploymentName := controller.RouterDeploymentName(ic) + deployment, err := getDeployment(t, kclient, deploymentName, 1*time.Minute) + if err != nil { + t.Fatalf("failed to get deployment %s/%s: %v", deploymentName.Namespace, deploymentName.Name, err) + } + routerPodList, err := getPods(t, kclient, deployment) + if err != nil { + t.Fatalf("failed to get pods in deployment %s/%s: %v", deploymentName.Namespace, deploymentName.Name, err) + } else if len(routerPodList.Items) == 0 { + t.Fatalf("no pods found in deployment %s/%s", deploymentName.Namespace, deploymentName.Name) + } + routerPod := routerPodList.Items[0] + + // Verify that the router pod has downloaded all the correct CRLs. + err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { + return verifyCRLs(t, &routerPod, currentCRLs) + }) + if err != nil { + t.Fatalf("Failed initial CRL check: %v", err) + } + // Once initial CRLs are downloaded, generate new CRLs, and update the crl-host configmap. These new CRLs + // should be downloaded as the old ones expire. + newCRLs := map[string]*x509.RevocationList{} + for _, caName := range tc.ExpectedCRLs { + newCRLs[caName], crlPems[caName+".crl"] = MustCreateCRL(currentCRLs[caName], caCerts[caName], time.Now(), time.Now().Add(1*time.Hour), nil) + } + crlConfigMap.Data = crlPems + if err := kclient.Update(context.TODO(), &crlConfigMap); err != nil { + t.Fatalf("Failed to update crl-host configmap: %v", err) + } + // Wait until each update CRL is updated, and verify that the updated CRL file matches the expected CRLs + for updates := 0; updates < len(currentCRLs); updates++ { + // Find the CRL that will expire first, and wait until its expiration. + nextExpiration := time.Time{} + expiringCRLCAName := "" + for caName, CRL := range currentCRLs { + if nextExpiration.IsZero() || CRL.NextUpdate.Before(nextExpiration) { + nextExpiration = CRL.NextUpdate + // keep track of the index of the next crl to expire + expiringCRLCAName = caName + } + } + t.Logf("Waiting until %s for CRL expiration", nextExpiration.Format(time.Stamp)) + // Replace the expiring CRL with its updated version in currentCRLs, and when the expiration time + // occurs, verify that the CRL file in the router pod is updated. + currentCRLs[expiringCRLCAName] = newCRLs[expiringCRLCAName] + // Wait for expiration + <-time.After(time.Until(nextExpiration)) + // Verify correct CRLs are present + if err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { + return verifyCRLs(t, &routerPod, currentCRLs) + }); err != nil { + if err != nil { + t.Fatalf("Failed waiting for %s CRL to be updated: %v", expiringCRLCAName, err) + } + } + } + }) + } +} + +func verifyCRLs(t *testing.T, pod *corev1.Pod, expectedCRLs map[string]*x509.RevocationList) (bool, error) { + t.Helper() + activeCRLs, err := getActiveCRLs(t, pod) + if err != nil { + return false, err + } + if len(activeCRLs) == 0 { + // 0 CRLs probably means the router hasn't completed startup yet. Retry. + return false, nil + } + if len(activeCRLs) == 1 && activeCRLs[0].Issuer.CommonName == "Placeholder CA" { + // Placeholder CA CRL means the actual CRLs haven't been downloaded yet. + return false, nil + } + if len(activeCRLs) != len(expectedCRLs) { + return false, fmt.Errorf("incorrect number of CRLs found in pod %s/%s. expected %d, got %d", pod.Namespace, pod.Name, len(expectedCRLs), len(activeCRLs)) + } + matchingCRLs := 0 + for _, expectedCRL := range expectedCRLs { + for _, foundCRL := range activeCRLs { + if foundCRL.Issuer.String() == expectedCRL.Issuer.String() { + if foundCRL.Number.Cmp(expectedCRL.Number) == 0 { + // Name and sequence number match, so the CRLs should be equivalent. + matchingCRLs++ + break + } else { + // Name matches but version doesn't. Wait for it to potentially be updated + t.Logf("Found %s but version does not match. expected %d, got %d", expectedCRL.Issuer.String(), expectedCRL.Number, foundCRL.Number) + return false, nil + } + } + } + } + if len(expectedCRLs) != matchingCRLs { + t.Errorf("Expected:") + for _, crl := range expectedCRLs { + t.Errorf("%q version %d", crl.Issuer.String(), crl.Number) + } + t.Errorf("Found:") + for _, crl := range activeCRLs { + t.Errorf("%q version %d", crl.Issuer.String(), crl.Number) + } + return false, fmt.Errorf("found %d CRLs, but only %d matched", len(activeCRLs), matchingCRLs) + } + return true, nil +} + +func getPods(t *testing.T, cl client.Client, deployment *appsv1.Deployment) (*corev1.PodList, error) { + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("deployment %s has invalid spec.selector: %w", deployment.Name, err) + } + podList := &corev1.PodList{} + if err := cl.List(context.TODO(), podList, client.MatchingLabelsSelector{Selector: selector}); err != nil { + t.Logf("failed to list pods for deployment %q: %v", deployment.Name, err) + return nil, err + } + return podList, nil +} + +func getActiveCRLs(t *testing.T, clientPod *corev1.Pod) ([]*x509.RevocationList, error) { + t.Helper() + cmd := []string{ + "cat", + "/var/lib/haproxy/mtls/latest/crls.pem", + } + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + err := podExec(t, *clientPod, &stdout, &stderr, cmd) + if err != nil { + t.Logf("stdout:\n%s", stdout.String()) + t.Logf("stderr:\n%s", stderr.String()) + return nil, err + } + crls := []*x509.RevocationList{} + crlData := []byte(stdout.String()) + for len(crlData) > 0 { + block, data := pem.Decode(crlData) + if block == nil { + break + } + crl, err := x509.ParseRevocationList(block.Bytes) + if err != nil { + return nil, err + } + crls = append(crls, crl) + crlData = data + } + return crls, nil +} + // generateClientCA generates and returns a CA certificate and key. func generateClientCA() (*x509.Certificate, *rsa.PrivateKey, error) { key, err := rsa.GenerateKey(rand.Reader, 2048) @@ -473,3 +1545,61 @@ func encodeKey(key *rsa.PrivateKey) string { Bytes: x509.MarshalPKCS1PrivateKey(key), })) } + +// curlGetStatusCode execs a Curl command in the test client pod and returns an error value. The Curl command uses the +// specified certificate from the client certificates configmap, sends a request for the canary route via the router's +// internal service. Returns the HTTP status code returned from curl, and an error either if there is an HTTP error, or +// if there's another error in running the command. If the error was not an HTTP error, the HTTP status code returned +// will be -1. +func curlGetStatusCode(t *testing.T, clientPod *corev1.Pod, certName, endpoint, ingressControllerIP string, verbose bool) (int64, error) { + t.Helper() + cmd := []string{ + "/bin/curl", + "--silent", + // Allow self-signed certs. + "-k", + // Output the http status code (i.e. 200 (OK) or 404 (Not found)) to stdout. + "-w", "%{http_code}", + // Retry on timeouts, 4xx errors, or 500/502/503/504 errors. + "--retry", "10", + // Sleep 1 second between retries. + "--retry-delay", "1", + // Use --resolve to guarantee that the request is sent through this test's ingress controller. + "--resolve", fmt.Sprintf("%s:443:%s", endpoint, ingressControllerIP), + fmt.Sprintf("https://%s", endpoint), + } + if verbose { + cmd = append(cmd, "-v") + } + if len(certName) != 0 { + cmd = append(cmd, + "--cert", fmt.Sprintf("/tmp/tls/%s.pem", certName), + "--key", fmt.Sprintf("/tmp/tls/%s.key", certName), + ) + } + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + curlErr := podExec(t, *clientPod, &stdout, &stderr, cmd) + stdoutStr := stdout.String() + t.Logf("command: %s\nstdout:\n%s\n\nstderr:\n%s\n", + strings.Join(cmd, " "), stdoutStr, stderr.String()) + // Try to parse the http status code even if curl returns an error; it may still be relevant. + httpStatusCode := stdoutStr[len(stdoutStr)-3:] + httpStatusCodeInt, err := strconv.ParseInt(httpStatusCode, 10, 64) + if err != nil { + // If parsing the status code returns an error but curl also returned an error, just send the curl one. + if curlErr != nil { + return -1, curlErr + } + return -1, err + } + if curlErr != nil { + return httpStatusCodeInt, curlErr + } + switch httpStatusCode[0] { + case '0', '4', '5': + return httpStatusCodeInt, fmt.Errorf("got HTTP %s status code", httpStatusCode) + default: + return httpStatusCodeInt, nil + } +} diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go index 72f8937fde..eec3d18f4f 100644 --- a/test/e2e/operator_test.go +++ b/test/e2e/operator_test.go @@ -53,6 +53,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/pointer" "k8s.io/apiserver/pkg/storage/names" @@ -3579,6 +3580,46 @@ func waitForDeploymentEnvVar(t *testing.T, cl client.Client, deployment *appsv1. return err } +// waitForDeploymentCompleteWithOldPodTermination waits for a deployment to +// complete its rollout, then waits for the old generation's pods to finish +// terminating. +func waitForDeploymentCompleteWithOldPodTermination(t *testing.T, cl client.Client, deploymentName types.NamespacedName, timeout time.Duration) error { + t.Helper() + deployment := &appsv1.Deployment{} + if err := cl.Get(context.TODO(), deploymentName, deployment); err != nil { + return fmt.Errorf("failed to get deployment %s: %w", deploymentName.Name, err) + } + + if err := waitForDeploymentComplete(t, cl, deployment, timeout); err != nil { + return fmt.Errorf("timed out waiting for deployment %s to complete rollout: %w", deploymentName.Name, err) + } + + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return fmt.Errorf("deployment %s has invalid spec.selector: %w", deploymentName.Name, err) + } + + // If spec.replicas is null, the default value is 1, per the API spec. + expectedReplicas := int(pointer.Int32Deref(deployment.Spec.Replicas, 1)) + + return wait.PollImmediate(2*time.Second, timeout, func() (bool, error) { + podList := &corev1.PodList{} + if err := cl.List(context.TODO(), podList, client.MatchingLabelsSelector{Selector: selector}); err != nil { + t.Logf("failed to list pods for deployment %q: %v", deploymentName.Name, err) + return false, nil + } + readyPods := 0 + for _, pod := range podList.Items { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + readyPods++ + } + } + } + return readyPods == expectedReplicas, nil + }) +} + func clusterOperatorConditionMap(conditions ...configv1.ClusterOperatorStatusCondition) map[string]string { conds := map[string]string{} for _, cond := range conditions { diff --git a/test/e2e/util_certgen.go b/test/e2e/util_certgen.go new file mode 100644 index 0000000000..bd54eb0d05 --- /dev/null +++ b/test/e2e/util_certgen.go @@ -0,0 +1,190 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "fmt" + "math/big" + "strings" + "time" +) + +// KeyCert bundles a certificate with its associated private key. +type KeyCert struct { + // The private key + Key *rsa.PrivateKey + // The certificate in logical form + Cert *x509.Certificate + // The signed certificate in binary form + CertPem string + // The entire certificate chain in binary form + CertFullChain string +} + +// CreateTLSKeyCert creates a key and certificate with CN commonName, valid between notBefore and notAfter, and with CRL +// Distribution Points crlDistributionPoints (if any). If isCA is true, the certificate is marked as being a CA +// certificate. If issuer is non-nil, the pem-encoded certificate is signed by issuer, otherwise it is self signed. +// +// Returns a KeyCert containing the key and certificate in logical form, as well as a pem-encoded version of the +// certificate. +func CreateTLSKeyCert(commonName string, notBefore, notAfter time.Time, isCA bool, crlDistributionPoints []string, issuer *KeyCert) (KeyCert, error) { + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return KeyCert{}, fmt.Errorf("failed to generate serial number: %w", err) + } + + privKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return KeyCert{}, fmt.Errorf("failed to generate key: %w", err) + } + + subjectKeyId, err := generateSubjectKeyID(privKey) + if err != nil { + return KeyCert{}, fmt.Errorf("failed to generate subject key ID: %w", err) + } + + certificate := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Openshift E2E Testing"}, + OrganizationalUnit: []string{"Engineering"}, + CommonName: commonName, + }, + NotBefore: notBefore, + NotAfter: notAfter, + IsCA: isCA, + BasicConstraintsValid: true, + CRLDistributionPoints: crlDistributionPoints, + SubjectKeyId: subjectKeyId, + } + + if isCA { + certificate.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature + } else { + certificate.KeyUsage = x509.KeyUsageDigitalSignature + } + + // If no issuer is specified, self-sign + if issuer == nil { + issuer = &KeyCert{ + Key: privKey, + Cert: certificate, + } + } + + certBytes, err := x509.CreateCertificate(rand.Reader, certificate, issuer.Cert, &privKey.PublicKey, issuer.Key) + if err != nil { + return KeyCert{}, fmt.Errorf("failed to create certificate %s: %w", commonName, err) + } + + pemBuffer := new(bytes.Buffer) + if err := pem.Encode(pemBuffer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }); err != nil { + return KeyCert{}, fmt.Errorf("failed to pem encode certificate %s: %w", commonName, err) + } + certPem := pemBuffer.String() + certFullChain := strings.Join([]string{certPem, issuer.CertFullChain}, "") + + return KeyCert{Key: privKey, Cert: certificate, CertPem: certPem, CertFullChain: certFullChain}, nil +} + +// MustCreateTLSKeyCert calls CreateTLSKeyCert, but instead of returning an error, it panics if an error occurs +func MustCreateTLSKeyCert(commonName string, notBefore, notAfter time.Time, isCA bool, crlDistributionPoints []string, issuer *KeyCert) KeyCert { + keyCert, err := CreateTLSKeyCert(commonName, notBefore, notAfter, isCA, crlDistributionPoints, issuer) + if err != nil { + panic(err) + } + return keyCert +} + +// CreateCRL generates a pem-encoded CRL for issuer, valid between thisUpdate and nextUpdate, that lists revokedCerts as +// revoked. Returns the logical form of the CRL, as well as a pem-encoded version. +func CreateCRL(existingRevocationList *x509.RevocationList, issuer KeyCert, thisUpdate, nextUpdate time.Time, revokedCerts []pkix.RevokedCertificate) (*x509.RevocationList, string, error) { + revocationList := x509.RevocationList{} + if existingRevocationList == nil { + revocationList = x509.RevocationList{ + Issuer: issuer.Cert.Subject, + Number: big.NewInt(1), + } + } else { + revocationList = *existingRevocationList + // revocationList.Number is a pointer, so create a copy in order to avoid overwriting the number in + // existingRevocationList + number := *revocationList.Number + revocationList.Number = &number + revocationList.Number.Add(revocationList.Number, big.NewInt(1)) + } + revocationList.ThisUpdate = thisUpdate + revocationList.NextUpdate = nextUpdate + revocationList.RevokedCertificates = revokedCerts + + crlBytes, err := x509.CreateRevocationList(rand.Reader, &revocationList, issuer.Cert, issuer.Key) + if err != nil { + return nil, "", fmt.Errorf("failed to create CRL for issuer %s: %w", issuer.Cert.Subject.CommonName, err) + } + + crlBuffer := new(bytes.Buffer) + if err := pem.Encode(crlBuffer, &pem.Block{ + Type: "X509 CRL", + Bytes: crlBytes, + }); err != nil { + return nil, "", fmt.Errorf("failed to pem encode CRL for issuer %s: %w", issuer.Cert.Subject.CommonName, err) + } + + return &revocationList, crlBuffer.String(), nil +} + +// MustCreateCRL calls CreateCRL, but instead of returning an error, it panics if an error occurs +func MustCreateCRL(revocationList *x509.RevocationList, issuer KeyCert, thisUpdate, nextUpdate time.Time, revokedCerts []pkix.RevokedCertificate) (*x509.RevocationList, string) { + crl, crlPem, err := CreateCRL(revocationList, issuer, thisUpdate, nextUpdate, revokedCerts) + if err != nil { + panic(err) + } + return crl, crlPem +} + +// RevokeCertificates revokes the certificates in keyCerts at revocationTime, and returns the list of revoked +// certificates, which can be appended to an existing list of revoked certificates and passed to CreateCRL(). +func RevokeCertificates(revocationTime time.Time, keyCerts ...KeyCert) []pkix.RevokedCertificate { + revokedCerts := []pkix.RevokedCertificate{} + for _, keyCert := range keyCerts { + revokedCert := pkix.RevokedCertificate{ + RevocationTime: revocationTime, + SerialNumber: keyCert.Cert.SerialNumber, + } + revokedCerts = append(revokedCerts, revokedCert) + } + return revokedCerts +} + +// pkcs1PublicKey reflects the ASN.1 structure of a PKCS #1 public key. +type pkcs1PublicKey struct { + N *big.Int + E int +} + +// generateSubjectKeyID generates a subject key by hashing the ASN.1-encoded public key bit string, as proposed in +// section 4.2.1.2 of RFC-5280. +func generateSubjectKeyID(key *rsa.PrivateKey) ([]byte, error) { + publicKeyBytes, err := asn1.Marshal(pkcs1PublicKey{ + N: key.PublicKey.N, + E: key.PublicKey.E, + }) + if err != nil { + return nil, err + } + subjectKeyId := sha1.Sum(publicKeyBytes) + return subjectKeyId[:], nil +} diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go index 2bd4b0fd9c..145ca0584f 100644 --- a/test/e2e/util_test.go +++ b/test/e2e/util_test.go @@ -79,6 +79,21 @@ func buildEchoPod(name, namespace string) *corev1.Pod { } } +// generateUnprivilegedSecurityContext returns a SecurityContext with the minimum possible privileges that satisfy +// restricted pod security requirements +func generateUnprivilegedSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: pointer.Bool(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: pointer.Bool(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + } +} + func waitForHTTPClientCondition(t *testing.T, httpClient *http.Client, req *http.Request, interval, timeout time.Duration, compareFunc func(*http.Response) bool) error { t.Helper() return wait.PollImmediate(interval, timeout, func() (done bool, err error) { @@ -638,3 +653,38 @@ func verifyInternalIngressController(t *testing.T, name types.NamespacedName, ho t.Fatalf("failed to verify connectivity with workload with address: %s using internal curl client. Curl Pod Logs:\n%s", address, curlPodLogs) } } + +// assertDeleted tries to delete a cluster resource, and causes test failure if the delete fails. +func assertDeleted(t *testing.T, cl client.Client, thing client.Object) { + t.Helper() + if err := cl.Delete(context.TODO(), thing); err != nil { + if errors.IsNotFound(err) { + return + } + t.Fatalf("Failed to delete %s: %v", thing.GetName(), err) + } else { + t.Logf("Deleted %s", thing.GetName()) + } +} + +// assertDeletedWaitForCleanup tries to delete a cluster resource, and waits for it to actually be cleaned up before +// returning. It causes test failure if the delete fails or if the cleanup times out. +func assertDeletedWaitForCleanup(t *testing.T, cl client.Client, thing client.Object) { + t.Helper() + thingName := types.NamespacedName{ + Name: thing.GetName(), + Namespace: thing.GetNamespace(), + } + assertDeleted(t, cl, thing) + if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { + if err := cl.Get(context.TODO(), thingName, thing); err != nil { + if errors.IsNotFound(err) { + return true, nil + } + return false, err + } + return false, nil + }); err != nil { + t.Fatalf("Timed out waiting for %s to be cleaned up: %v", thing.GetName(), err) + } +}