diff --git a/Makefile b/Makefile index caf2e65e0..cdaeb35db 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,11 @@ TRUST_MANAGER_VERSION ?= v0.20.3 # --- Test Versions --- +# OpenShift Service Mesh versions for IstioCSR ServiceMesh e2e tests. +# Keep servicemesh_helpers_test.go ossmDefault* constants in sync when bumping. +E2E_OSM_ISTIO_VERSION ?= v1.24.3 +E2E_OSM_OPERATOR_VERSION ?= 3.2.5 + # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION ?= 1.32.0 @@ -188,7 +193,7 @@ E2E_TIMEOUT ?= 2h # E2E_GINKGO_LABEL_FILTER is ginkgo label query for selecting tests. # See https://onsi.github.io/ginkgo/#spec-labels # The default is to run tests on the AWS platform. -E2E_GINKGO_LABEL_FILTER ?= Platform: isSubsetOf {AWS,Generic} && CredentialsMode: isSubsetOf {Mint} +E2E_GINKGO_LABEL_FILTER ?= Platform: isSubsetOf {AWS,Generic} && CredentialsMode: isSubsetOf {Mint} && !Feature:ServiceMesh # ============================================================================ # Default Target @@ -279,6 +284,8 @@ test-apis: $(SETUP_ENVTEST) $(GINKGO) TEST ?= .PHONY: test-e2e test-e2e: test-e2e-wait-for-stable-state ## Run end-to-end tests. + E2E_OSM_ISTIO_VERSION=$(E2E_OSM_ISTIO_VERSION) \ + E2E_OSM_OPERATOR_VERSION=$(E2E_OSM_OPERATOR_VERSION) \ go test -C $(PROJECT_ROOT)/test/e2e \ -timeout $(E2E_TIMEOUT) \ -count 1 -v -p 1 \ @@ -590,4 +597,4 @@ $(OPERATOR_SDK): ## Download operator-sdk locally if necessary. hack/download-tools.sh operator-sdk $(OPERATOR_SDK) $(OPM): ## Download opm locally if necessary. - hack/download-tools.sh opm $(OPM) + hack/download-tools.sh opm $(OPM) \ No newline at end of file diff --git a/test/e2e/certmanager_resource_helpers_test.go b/test/e2e/certmanager_resource_helpers_test.go new file mode 100644 index 000000000..cb43911c3 --- /dev/null +++ b/test/e2e/certmanager_resource_helpers_test.go @@ -0,0 +1,70 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmanagerclientset "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func ensureClusterIssuer(ctx context.Context, client certmanagerclientset.Interface, issuer *certmanagerv1.ClusterIssuer) error { + _, err := client.CertmanagerV1().ClusterIssuers().Create(ctx, issuer, metav1.CreateOptions{}) + if err == nil { + return nil + } + if !apierrors.IsAlreadyExists(err) { + return err + } + + existing, getErr := client.CertmanagerV1().ClusterIssuers().Get(ctx, issuer.Name, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + + issuer.ResourceVersion = existing.ResourceVersion + _, err = client.CertmanagerV1().ClusterIssuers().Update(ctx, issuer, metav1.UpdateOptions{}) + return err +} + +func ensureIssuer(ctx context.Context, client certmanagerclientset.Interface, issuer *certmanagerv1.Issuer) error { + _, err := client.CertmanagerV1().Issuers(issuer.Namespace).Create(ctx, issuer, metav1.CreateOptions{}) + if err == nil { + return nil + } + if !apierrors.IsAlreadyExists(err) { + return err + } + + existing, getErr := client.CertmanagerV1().Issuers(issuer.Namespace).Get(ctx, issuer.Name, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + + issuer.ResourceVersion = existing.ResourceVersion + _, err = client.CertmanagerV1().Issuers(issuer.Namespace).Update(ctx, issuer, metav1.UpdateOptions{}) + return err +} + +func ensureCertificate(ctx context.Context, client certmanagerclientset.Interface, certificate *certmanagerv1.Certificate) error { + _, err := client.CertmanagerV1().Certificates(certificate.Namespace).Create(ctx, certificate, metav1.CreateOptions{}) + if err == nil { + return nil + } + if !apierrors.IsAlreadyExists(err) { + return err + } + + existing, getErr := client.CertmanagerV1().Certificates(certificate.Namespace).Get(ctx, certificate.Name, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + + certificate.ResourceVersion = existing.ResourceVersion + _, err = client.CertmanagerV1().Certificates(certificate.Namespace).Update(ctx, certificate, metav1.UpdateOptions{}) + return err +} diff --git a/test/e2e/config_template.go b/test/e2e/config_template.go index 5d0d91ece..0afc35e95 100644 --- a/test/e2e/config_template.go +++ b/test/e2e/config_template.go @@ -27,6 +27,8 @@ type IstioCSRGRPCurlJobConfig struct { IstioCSRStatus v1alpha1.IstioCSRStatus ClusterID string JobName string + ProtoConfigMapName string + ServiceAccountName string } // ServiceMonitorConfig customizes fields in the ServiceMonitor spec @@ -37,6 +39,46 @@ type ServiceMonitorConfig struct { ComponentName string } +// OSSMv3Config customizes OpenShift Service Mesh v3 install manifests. +type OSSMv3Config struct { + OperatorVersion string + IstioVersion string + ClusterID string + CAAddress string +} + +const ( + istioCSRProfileMinimal = "minimal" + istioCSRProfileOSSM = "ossm" + istioCSROperandManifest = "testdata/istio/istio_csr_template.yaml" +) + +// IstioCSRConfig customizes the IstioCSR operand manifest. +// Profile is "minimal" (default) for isolated IstioCSR tests or "ossm" for Service Mesh smoke. +// IstioNamespace is spec.istioCSRConfig.istio.namespace; for the minimal profile it must match +// the test namespace where the istio-ca Issuer is created. +type IstioCSRConfig struct { + Namespace string + IstioNamespace string + ClusterID string + IstioDataPlaneNamespaceSelector string + Profile string + IssuerName string +} + +func istioCSRConfigForNS(namespace string, overrides IstioCSRConfig) IstioCSRConfig { + if overrides.Namespace == "" { + overrides.Namespace = namespace + } + if overrides.IstioNamespace == "" { + overrides.IstioNamespace = namespace + } + if overrides.Profile == "" { + overrides.Profile = istioCSRProfileMinimal + } + return overrides +} + // replaceWithTemplate puts field values from a template struct func replaceWithTemplate(sourceFileContents string, templatedValues any) ([]byte, error) { tmpl, err := template.New("template").Option("missingkey=error").Parse(sourceFileContents) diff --git a/test/e2e/issuer_acme_dns01_test.go b/test/e2e/issuer_acme_dns01_test.go index 2ae9f5e0e..4bd46f4ba 100644 --- a/test/e2e/issuer_acme_dns01_test.go +++ b/test/e2e/issuer_acme_dns01_test.go @@ -108,8 +108,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { }, }, } - _, err = loader.KubeClient.CoreV1().ConfigMaps("cert-manager").Create(ctx, trustedCA, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + Expect(library.UpsertConfigMap(ctx, loader.KubeClient, trustedCA)).NotTo(HaveOccurred()) DeferCleanup(func(cleanupCtx context.Context) { loader.KubeClient.CoreV1().ConfigMaps("cert-manager").Delete(cleanupCtx, "trusted-ca", metav1.DeleteOptions{}) @@ -272,8 +271,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { secretKey: secretAccessKey, }, } - _, err := loader.KubeClient.CoreV1().Secrets(namespace).Create(ctx, awsSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create secret %s", secretName)) + Expect(library.UpsertSecret(ctx, loader.KubeClient, awsSecret)).NotTo(HaveOccurred(), fmt.Sprintf("failed to create secret %s", secretName)) } // setupAmbientAWSCredentials sets up ambient AWS credentials via CredentialsRequest and subscription patch @@ -343,8 +341,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { secretKey: serviceAccount, }, } - _, err := loader.KubeClient.CoreV1().Secrets(namespace).Create(ctx, gcpSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create secret %s", secretName)) + Expect(library.UpsertSecret(ctx, loader.KubeClient, gcpSecret)).NotTo(HaveOccurred(), fmt.Sprintf("failed to create secret %s", secretName)) } // setupAmbientGCPCredentials sets up ambient GCP credentials via CredentialsRequest and subscription patch @@ -532,8 +529,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { secretKey: clientSecret, }, } - _, err := loader.KubeClient.CoreV1().Secrets(namespace).Create(ctx, azureSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create secret %s", secretName)) + Expect(library.UpsertSecret(ctx, loader.KubeClient, azureSecret)).NotTo(HaveOccurred(), fmt.Sprintf("failed to create secret %s", secretName)) } Context("with AWS Route53", Label("Platform:AWS", "CredentialsMode:Mint"), func() { @@ -978,8 +974,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { "credentials": credContent, }, } - _, err := loader.KubeClient.CoreV1().Secrets("cert-manager").Create(ctx, stsSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create STS credential secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, stsSecret)).NotTo(HaveOccurred(), "failed to create STS credential secret") DeferCleanup(func(ctx context.Context) { By("Deleting manually created STS credential secret") @@ -990,7 +985,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { }) By("patching subscription to inject 'CLOUD_CREDENTIALS_SECRET_NAME' env var") - err = patchSubscriptionWithEnvVars(ctx, loader, map[string]string{ + err := patchSubscriptionWithEnvVars(ctx, loader, map[string]string{ "CLOUD_CREDENTIALS_SECRET_NAME": secretName, }) Expect(err).NotTo(HaveOccurred(), "failed to patch subscription with 'CLOUD_CREDENTIALS_SECRET_NAME'") @@ -1243,8 +1238,7 @@ var _ = Describe("ACME Issuer DNS01 solver", Ordered, func() { "service_account.json": credContent, }, } - _, err = loader.KubeClient.CoreV1().Secrets("cert-manager").Create(ctx, stsSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create GCP STS credentials secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, stsSecret)).NotTo(HaveOccurred(), "failed to create GCP STS credentials secret") DeferCleanup(func(ctx context.Context, namespace, name string) { By("Deleting GCP STS credentials secret") diff --git a/test/e2e/issuer_acme_http01_test.go b/test/e2e/issuer_acme_http01_test.go index 0e1a2e6c3..eac2c54d4 100644 --- a/test/e2e/issuer_acme_http01_test.go +++ b/test/e2e/issuer_acme_http01_test.go @@ -27,6 +27,16 @@ import ( . "github.com/onsi/gomega" ) +// acmeHTTP01OpenShiftIngressClass is required on OpenShift so HTTP-01 challenge Ingresses get Routes. +const acmeHTTP01OpenShiftIngressClass = "openshift-default" + +func acmeHTTP01OpenShiftIngress() *acmev1.ACMEChallengeSolverHTTP01Ingress { + ingressClass := acmeHTTP01OpenShiftIngressClass + return &acmev1.ACMEChallengeSolverHTTP01Ingress{ + IngressClassName: &ingressClass, + } +} + var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered, func() { var ctx context.Context var cancel context.CancelFunc @@ -68,8 +78,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered }, }, } - _, err = loader.KubeClient.CoreV1().ConfigMaps("cert-manager").Create(ctx, trustedCA, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred()) + Expect(library.UpsertConfigMap(ctx, loader.KubeClient, trustedCA)).NotTo(HaveOccurred()) DeferCleanup(func(cleanupCtx context.Context) { loader.KubeClient.CoreV1().ConfigMaps("cert-manager").Delete(cleanupCtx, "trusted-ca", metav1.DeleteOptions{}) @@ -126,7 +135,6 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered BeforeEach(func() { clusterIssuerName := "letsencrypt-http01" - ingressClassName := "openshift-default" secretName = "ingress-http01-secret" By("creating a cluster issuer") @@ -146,9 +154,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered Solvers: []acmev1.ACMEChallengeSolver{ { HTTP01: &acmev1.ACMEChallengeSolverHTTP01{ - Ingress: &acmev1.ACMEChallengeSolverHTTP01Ingress{ - IngressClassName: &ingressClassName, - }, + Ingress: acmeHTTP01OpenShiftIngress(), }, }, }, @@ -172,6 +178,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered By("creating Ingress object") ingressHost = fmt.Sprintf("ahi-%s.%s", randomStr(3), appsDomain) // acronym for "ACME http-01 Ingress" pathType := networkingv1.PathTypePrefix + ingressClassName := acmeHTTP01OpenShiftIngressClass ingress := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "ingress-http01", @@ -311,7 +318,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered Solvers: []acmev1.ACMEChallengeSolver{ { HTTP01: &acmev1.ACMEChallengeSolverHTTP01{ - Ingress: &acmev1.ACMEChallengeSolverHTTP01Ingress{}, + Ingress: acmeHTTP01OpenShiftIngress(), }, }, }, @@ -369,8 +376,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered "client-secret": "dummy-client-secret", }, } - _, err := loader.KubeClient.CoreV1().Secrets("cert-manager").Create(ctx, azureDNSSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create Azure DNS secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, azureDNSSecret)).NotTo(HaveOccurred(), "failed to create Azure DNS secret") DeferCleanup(func(ctx context.Context) { err := loader.KubeClient.CoreV1().Secrets("cert-manager").Delete(ctx, azureDNSSecretName, metav1.DeleteOptions{}) @@ -389,8 +395,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered "secret-access-key": "dummy-secret-key", }, } - _, err = loader.KubeClient.CoreV1().Secrets("cert-manager").Create(ctx, route53Secret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create Route53 secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, route53Secret)).NotTo(HaveOccurred(), "failed to create Route53 secret") DeferCleanup(func(ctx context.Context) { err := loader.KubeClient.CoreV1().Secrets("cert-manager").Delete(ctx, route53SecretName, metav1.DeleteOptions{}) @@ -423,7 +428,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered DNSZones: []string{testDomain}, }, HTTP01: &acmev1.ACMEChallengeSolverHTTP01{ - Ingress: &acmev1.ACMEChallengeSolverHTTP01Ingress{}, + Ingress: acmeHTTP01OpenShiftIngress(), }, }, // Solver 2: DNS-01 (Azure) with specific dnsNames selector @@ -470,7 +475,7 @@ var _ = Describe("ACME Issuer HTTP01 solver", Label("Platform:Generic"), Ordered }, }, } - _, err = certmanagerClient.CertmanagerV1().ClusterIssuers().Create(ctx, clusterIssuer, metav1.CreateOptions{}) + _, err := certmanagerClient.CertmanagerV1().ClusterIssuers().Create(ctx, clusterIssuer, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred(), "failed to create ClusterIssuer") DeferCleanup(func(ctx context.Context) { diff --git a/test/e2e/issuer_vault_test.go b/test/e2e/issuer_vault_test.go index bdc797c8f..194c754a3 100644 --- a/test/e2e/issuer_vault_test.go +++ b/test/e2e/issuer_vault_test.go @@ -16,6 +16,7 @@ import ( certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/openshift/cert-manager-operator/test/library" "github.com/tidwall/gjson" . "github.com/onsi/ginkgo/v2" @@ -101,9 +102,6 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { } BeforeEach(func() { - ctx, cancel = context.WithTimeout(context.Background(), highTimeout) - DeferCleanup(cancel) - By("waiting for operator status to become available") err := VerifyHealthyOperatorConditions(certmanageroperatorclient.OperatorV1alpha1()) Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") @@ -140,6 +138,9 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { By("configuring Vault PKI engine") err = configureVaultPKI(setupCtx, cfg, loader, ns.Name, vaultPodName, vaultRootToken) Expect(err).NotTo(HaveOccurred()) + + ctx, cancel = context.WithTimeout(context.Background(), highTimeout) + DeferCleanup(cancel) }) Context("AppRole authentication", func() { @@ -151,20 +152,19 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { certCommonName := certName + ".cluster.local" By("configuring Vault AppRole authentication") - tokenEnv := fmt.Sprintf("export VAULT_TOKEN=%s", vaultRootToken) - vaultCmd := fmt.Sprintf(`%s && vault auth enable approle && vault write auth/approle/role/%s token_policies="cert-manager" token_ttl=1h token_max_ttl=4h`, tokenEnv, appRoleVaultRoleName) + vaultCmd := vaultShellCmd(fmt.Sprintf(`vault auth enable approle && vault write auth/approle/role/%s token_policies="cert-manager" token_ttl=1h token_max_ttl=4h`, appRoleVaultRoleName)) _, err := execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to configure Vault AppRole authentication") By("retrieving AppRole role ID") - vaultCmd = fmt.Sprintf(`%s && vault read -format=json auth/approle/role/%s/role-id`, tokenEnv, appRoleVaultRoleName) + vaultCmd = vaultShellCmd(fmt.Sprintf(`vault read -format=json auth/approle/role/%s/role-id`, appRoleVaultRoleName)) output, err := execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to retrieve AppRole role ID") vaultRoleID := gjson.Get(output, "data.role_id").String() Expect(vaultRoleID).NotTo(BeEmpty(), "AppRole role ID should not be empty") By("retrieving AppRole secret ID") - vaultCmd = fmt.Sprintf(`%s && vault write -format=json -force auth/approle/role/%s/secret-id`, tokenEnv, appRoleVaultRoleName) + vaultCmd = vaultShellCmd(fmt.Sprintf(`vault write -format=json -force auth/approle/role/%s/secret-id`, appRoleVaultRoleName)) output, err = execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to retrieve AppRole secret ID") vaultSecretID := gjson.Get(output, "data.secret_id").String() @@ -180,8 +180,7 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { "secretId": vaultSecretID, }, } - _, err = loader.KubeClient.CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create AppRole secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, secret)).NotTo(HaveOccurred(), "failed to create AppRole secret") By("creating Vault issuer with AppRole authentication") issuer := createVaultIssuer(issuerName, certmanagerv1.VaultAuth{ @@ -215,8 +214,7 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { certCommonName := certName + ".cluster.local" By("creating Vault token with cert-manager policy") - tokenEnv := fmt.Sprintf("export VAULT_TOKEN=%s", vaultRootToken) - vaultCmd := fmt.Sprintf(`%s && vault token create -format=json -policy=cert-manager -ttl=720h`, tokenEnv) + vaultCmd := vaultShellCmd(`vault token create -format=json -policy=cert-manager -ttl=720h`) output, err := execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to create Vault token") vaultToken := gjson.Get(output, "auth.client_token").String() @@ -232,8 +230,7 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { "token": vaultToken, }, } - _, err = loader.KubeClient.CoreV1().Secrets(ns.Name).Create(ctx, secret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create token secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, secret)).NotTo(HaveOccurred(), "failed to create token secret") By("creating Vault issuer with token authentication") issuer := createVaultIssuer(issuerName, certmanagerv1.VaultAuth{ @@ -283,14 +280,12 @@ var _ = Describe("Vault Issuer", Ordered, Label("Platform:Generic"), func() { }, Type: corev1.SecretTypeServiceAccountToken, } - _, err = loader.KubeClient.CoreV1().Secrets(ns.Name).Create(ctx, tokenSecret, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create service account token secret") + Expect(library.UpsertSecret(ctx, loader.KubeClient, tokenSecret)).NotTo(HaveOccurred(), "failed to create service account token secret") By("configuring Kubernetes auth in Vault") - tokenEnv := fmt.Sprintf("export VAULT_TOKEN=%s", vaultRootToken) - vaultCmd := fmt.Sprintf(`%s && vault auth enable kubernetes && vault write auth/kubernetes/config kubernetes_host="%s" kubernetes_ca_cert=@%s && \ + vaultCmd := vaultShellCmd(fmt.Sprintf(`vault auth enable kubernetes && vault write auth/kubernetes/config kubernetes_host="%s" kubernetes_ca_cert=@%s && \ vault write auth/kubernetes/role/issuer bound_service_account_names=%s bound_service_account_namespaces=%s token_policies=cert-manager ttl=1h`, - tokenEnv, vaultKubernetesHost, vaultServiceAccountCA, serviceAccountName, ns.Name) + vaultKubernetesHost, vaultServiceAccountCA, serviceAccountName, ns.Name)) _, err = execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to configure Kubernetes auth in Vault") @@ -375,10 +370,9 @@ vault write auth/kubernetes/role/issuer bound_service_account_names=%s bound_ser Expect(err).NotTo(HaveOccurred(), "failed to create role binding") By("configuring Kubernetes auth in Vault") - tokenEnv := fmt.Sprintf("export VAULT_TOKEN=%s", vaultRootToken) - vaultCmd := fmt.Sprintf(`%s && vault auth enable kubernetes && vault write auth/kubernetes/config kubernetes_host="%s" kubernetes_ca_cert=@%s && \ + vaultCmd := vaultShellCmd(fmt.Sprintf(`vault auth enable kubernetes && vault write auth/kubernetes/config kubernetes_host="%s" kubernetes_ca_cert=@%s && \ vault write auth/kubernetes/role/issuer bound_service_account_names=%s bound_service_account_namespaces=%s token_policies=cert-manager ttl=1h`, - tokenEnv, vaultKubernetesHost, vaultServiceAccountCA, serviceAccountName, ns.Name) + vaultKubernetesHost, vaultServiceAccountCA, serviceAccountName, ns.Name)) _, err = execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to configure Kubernetes auth in Vault") @@ -468,8 +462,10 @@ vault write auth/kubernetes/role/issuer bound_service_account_names=%s bound_ser Expect(oidcIssuer).NotTo(BeEmpty(), "OIDC issuer URL should not be empty") By("configuring JWT auth in Vault") - tokenEnv := fmt.Sprintf("export VAULT_TOKEN=%s", vaultRootToken) - vaultCmd := fmt.Sprintf(`%s && vault auth enable jwt && vault write auth/jwt/config oidc_discovery_url=%s`, tokenEnv, oidcIssuer) + _, err = execVaultInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "auth", "enable", "jwt") + Expect(err).NotTo(HaveOccurred(), "failed to enable JWT auth in Vault") + + jwtConfigArgs := []string{"write", "auth/jwt/config", "oidc_discovery_url=" + oidcIssuer} // Handle non-STS environments where OIDC issuer is internal URL if strings.Contains(oidcIssuer, "kubernetes.default.svc") { @@ -515,15 +511,15 @@ vault write auth/kubernetes/role/issuer bound_service_account_names=%s bound_ser }) // Add CA certificate for internal OIDC issuer - vaultCmd += " oidc_discovery_ca_pem=@" + vaultServiceAccountCA + jwtConfigArgs = append(jwtConfigArgs, "oidc_discovery_ca_pem=@"+vaultServiceAccountCA) } - _, err = execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) + _, err = execVaultInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, jwtConfigArgs...) Expect(err).NotTo(HaveOccurred(), "failed to configure JWT auth in Vault") By("creating JWT role in Vault") - vaultCmd = fmt.Sprintf(`%s && vault write auth/jwt/role/issuer role_type=jwt bound_audiences="vault://%s/%s" user_claim=sub bound_subject="system:serviceaccount:%s:%s" token_policies=cert-manager ttl=1m`, - tokenEnv, ns.Name, issuerName, ns.Name, serviceAccountName) + vaultCmd := vaultShellCmd(fmt.Sprintf(`vault write auth/jwt/role/issuer role_type=jwt bound_audiences="vault://%s/%s" user_claim=sub bound_subject="system:serviceaccount:%s:%s" token_policies=cert-manager ttl=1m`, + ns.Name, issuerName, ns.Name, serviceAccountName)) _, err = execInPod(ctx, cfg, loader.KubeClient, ns.Name, vaultPodName, "vault", "sh", "-c", vaultCmd) Expect(err).NotTo(HaveOccurred(), "failed to create JWT role in Vault") diff --git a/test/e2e/istio_csr_helpers_test.go b/test/e2e/istio_csr_helpers_test.go new file mode 100644 index 000000000..7771ee101 --- /dev/null +++ b/test/e2e/istio_csr_helpers_test.go @@ -0,0 +1,79 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" + "github.com/openshift/cert-manager-operator/test/library" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" +) + +type LogEntry struct { + CertChain []string `json:"certChain"` +} + +const grpcurlLogExcerptMaxLen = 512 + +func formatGRPCurlLogExcerpt(logData []byte) string { + if len(logData) == 0 { + return "" + } + + excerpt := logData + if len(excerpt) > grpcurlLogExcerptMaxLen { + excerpt = append(append([]byte(nil), excerpt[:grpcurlLogExcerptMaxLen]...), []byte("...")...) + } + return strings.ReplaceAll(string(excerpt), "\n", `\n`) +} + +// parseGRPCurlLogEntry extracts the grpcurl CreateCertificate JSON response from pod logs. +// It accepts compact or multi-line JSON, and falls back to the last valid line when retries +// leave non-JSON noise before or after the response. +func parseGRPCurlLogEntry(logData []byte) (LogEntry, error) { + trimmed := bytes.TrimSpace(logData) + var entry LogEntry + if len(trimmed) > 0 && json.Unmarshal(trimmed, &entry) == nil { + return entry, nil + } + + lines := bytes.Split(trimmed, []byte("\n")) + for i := len(lines) - 1; i >= 0; i-- { + line := bytes.TrimSpace(lines[i]) + if len(line) == 0 { + continue + } + if err := json.Unmarshal(line, &entry); err == nil { + return entry, nil + } + } + + return LogEntry{}, fmt.Errorf( + "no valid grpcurl JSON log entry found in pod logs (excerpt: %s)", + formatGRPCurlLogExcerpt(trimmed), + ) +} + +// waitForIstioCSROperandReady waits for the cert-manager-istio-csr deployment and IstioCSR CR status. +func waitForIstioCSROperandReady(ctx context.Context, clientset *kubernetes.Clientset, loader library.DynamicResourceLoader, namespace string) (v1alpha1.IstioCSRStatus, error) { + if err := pollTillDeploymentAvailable(ctx, clientset, namespace, istioCSRGRPCServiceName); err != nil { + return v1alpha1.IstioCSRStatus{}, err + } + return pollTillIstioCSRAvailable(ctx, loader, namespace, istioCSRResourceName) +} + +// expectIstioCSROperandReady waits for the operand to become ready and fails the spec on error. +func expectIstioCSROperandReady(ctx context.Context, clientset *kubernetes.Clientset, loader library.DynamicResourceLoader, namespace string) v1alpha1.IstioCSRStatus { + By("waiting for IstioCSR operand to become ready") + status, err := waitForIstioCSROperandReady(ctx, clientset, loader, namespace) + Expect(err).NotTo(HaveOccurred()) + return status +} diff --git a/test/e2e/istio_csr_operand_test.go b/test/e2e/istio_csr_operand_test.go new file mode 100644 index 000000000..e30c5b591 --- /dev/null +++ b/test/e2e/istio_csr_operand_test.go @@ -0,0 +1,773 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "io" + "net/url" + "path/filepath" + "strconv" + "strings" + + acmev1 "github.com/cert-manager/cert-manager/pkg/apis/acme/v1" + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" + "github.com/openshift/cert-manager-operator/test/library" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes" +) + +const ( + istioCSRClusterIssuerName = "selfsigned-issuer" + istioCSRCAClusterIssuerName = "istiocsr-ca-clusterissuer" + istioCSRCASecretName = "root-secret" + istioCSRCACertificateName = "my-selfsigned-ca" + istioCSRRejectAnnotation = "operator.openshift.io/istio-csr-reject-multiple-instance" + istioCSRManagedResourceLabel = "app.kubernetes.io/name=cert-manager-istio-csr" + istioCSRResourceName = "default" + istioCSRGRPCServiceName = "cert-manager-istio-csr" + istioCSRGRPCServicePortName = "web" + istioCSRMissingConfigMapKeyMessage = "not found in ConfigMap" + + istioCSRIstiodTLSSecretName = "istiod-tls" + + // Non-routable ACME directory URL for negative-path tests; operator rejects ACME issuers by type only. + istioCSRACMEPlaceholderServer = "https://example.invalid/directory" +) + +type istioCSRBuildConfig struct { + issuerRefKind string + issuerRefName string + serverPort int32 + logLevel int32 + logFormat string + customCAConfigMapName string + customCAConfigMapNamespace string + customCAConfigMapKey string + addServerBlock bool + addCustomCAConfigMap bool +} + +func generateMeshWorkloadCSR(meshNamespace, serviceAccountName string) string { + csrTemplate := &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"OpenShift cert-manager IstioCSR E2E"}, + }, + URIs: []*url.URL{ + { + Scheme: "spiffe", + Host: "cluster.local", + Path: fmt.Sprintf("/ns/%s/sa/%s", meshNamespace, serviceAccountName), + }, + }, + SignatureAlgorithm: x509.SHA256WithRSA, + } + csr, err := library.GenerateCSR(csrTemplate) + Expect(err).NotTo(HaveOccurred()) + return csr +} + +func copySecretToNamespace(ctx context.Context, clientset *kubernetes.Clientset, sourceNS, targetNS, secretName string) error { + source, err := clientset.CoreV1().Secrets(sourceNS).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get secret %s/%s: %w", sourceNS, secretName, err) + } + + copied := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: targetNS, + }, + Data: source.Data, + Type: source.Type, + } + if err := library.UpsertSecret(ctx, clientset, copied); err != nil { + return fmt.Errorf("upsert secret %s/%s: %w", targetNS, secretName, err) + } + return nil +} + +var _ = Describe("Istio-CSR operand coverage [apigroup:operator.openshift.io]", Ordered, Label("Platform:Generic", "Feature:IstioCSR"), func() { + ctx := context.TODO() + var clientset *kubernetes.Clientset + + ensureClusterIssuerReady := func() { + By("ensuring self-signed ClusterIssuer exists") + clusterIssuer := &certmanagerv1.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: istioCSRClusterIssuerName, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + SelfSigned: &certmanagerv1.SelfSignedIssuer{}, + }, + }, + } + Expect(ensureClusterIssuer(ctx, certmanagerClient, clusterIssuer)).NotTo(HaveOccurred()) + + By("waiting for self-signed ClusterIssuer readiness") + Expect(waitForClusterIssuerReadiness(ctx, istioCSRClusterIssuerName)).NotTo(HaveOccurred()) + } + + createIssuerPrerequisites := func(namespace string) { + By("creating a CA certificate in test namespace") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), namespace) + + By("waiting for CA certificate readiness") + Expect(waitForCertificateReadiness(ctx, "my-selfsigned-ca", namespace)).NotTo(HaveOccurred()) + + By("creating Istio CA issuer") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), namespace) + + DeferCleanup(func() { + loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "istio", "istio_ca_issuer.yaml"), namespace) + loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), namespace) + }) + } + + newIstioCSR := func(namespace string, cfg istioCSRBuildConfig) *unstructured.Unstructured { + issuerRefKind := "Issuer" + if cfg.issuerRefKind != "" { + issuerRefKind = cfg.issuerRefKind + } + issuerRefName := "istio-ca" + if cfg.issuerRefName != "" { + issuerRefName = cfg.issuerRefName + } + + istioCSR := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "operator.openshift.io/v1alpha1", + "kind": "IstioCSR", + "metadata": map[string]interface{}{ + "name": istioCSRResourceName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "istioCSRConfig": map[string]interface{}{ + "certManager": map[string]interface{}{ + "issuerRef": map[string]interface{}{ + "name": issuerRefName, + "kind": issuerRefKind, + "group": "cert-manager.io", + }, + }, + "istiodTLSConfig": map[string]interface{}{ + "trustDomain": "cluster.local", + }, + "istio": map[string]interface{}{ + "namespace": namespace, + }, + }, + }, + }, + } + + istioCSRConfig := istioCSR.Object["spec"].(map[string]interface{})["istioCSRConfig"].(map[string]interface{}) + if cfg.logLevel > 0 { + istioCSRConfig["logLevel"] = cfg.logLevel + } + if cfg.logFormat != "" { + istioCSRConfig["logFormat"] = cfg.logFormat + } + if cfg.addServerBlock { + istioCSRConfig["server"] = map[string]interface{}{ + "port": cfg.serverPort, + } + } + if cfg.addCustomCAConfigMap { + istioCSRConfig["certManager"].(map[string]interface{})["istioCACertificate"] = map[string]interface{}{ + "name": cfg.customCAConfigMapName, + "key": cfg.customCAConfigMapKey, + } + if cfg.customCAConfigMapNamespace != "" { + istioCSRConfig["certManager"].(map[string]interface{})["istioCACertificate"].(map[string]interface{})["namespace"] = cfg.customCAConfigMapNamespace + } + } + + return istioCSR + } + + createIstioCSR := func(namespace string, istioCSR *unstructured.Unstructured) { + _, err := loader.DynamicClient.Resource(istiocsrSchema).Namespace(namespace).Create(ctx, istioCSR, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + err := loader.DynamicClient.Resource(istiocsrSchema).Namespace(namespace).Delete(ctx, istioCSR.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } + }) + } + + getGRPCPortFromEndpoint := func(endpoint string) int32 { + lastColon := strings.LastIndex(endpoint, ":") + Expect(lastColon).To(BeNumerically(">", 0)) + portValue := endpoint[lastColon+1:] + port, err := strconv.ParseInt(portValue, 10, 32) + Expect(err).NotTo(HaveOccurred()) + return int32(port) + } + + BeforeAll(func() { + var err error + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + ensureClusterIssuerReady() + }) + + It("should reject IstioCSR with name other than default", Label("ISTIOCSR-001"), func() { + ns, err := loader.CreateTestingNS("istiocsr-invalid-name", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + invalid := newIstioCSR(ns.Name, istioCSRBuildConfig{}) + invalid.SetName("not-default") + + _, err = loader.DynamicClient.Resource(istiocsrSchema).Namespace(ns.Name).Create(ctx, invalid, metav1.CreateOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("metadata.name")) + }) + + It("should reject processing of second IstioCSR instance across namespaces", Label("ISTIOCSR-002"), func() { + firstNS, err := loader.CreateTestingNS("istiocsr-first", true) + Expect(err).NotTo(HaveOccurred()) + secondNS, err := loader.CreateTestingNS("istiocsr-second", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(secondNS.Name, func() bool { return CurrentSpecReport().Failed() }) + loader.DeleteTestingNS(firstNS.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + createIssuerPrerequisites(firstNS.Name) + createIssuerPrerequisites(secondNS.Name) + + createIstioCSR(firstNS.Name, newIstioCSR(firstNS.Name, istioCSRBuildConfig{})) + expectIstioCSROperandReady(ctx, clientset, loader, firstNS.Name) + + createIstioCSR(secondNS.Name, newIstioCSR(secondNS.Name, istioCSRBuildConfig{})) + By("waiting for IstioCSR Ready=False with multiple-instance rejection message") + Expect(waitForIstioCSRConditionMessage(ctx, loader, secondNS.Name, istioCSRResourceName, v1alpha1.Ready, metav1.ConditionFalse, "multiple instances of istiocsr exists", highTimeout, slowPollInterval)).NotTo(HaveOccurred()) + + obj, err := loader.DynamicClient.Resource(istiocsrSchema).Namespace(secondNS.Name).Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(obj.GetAnnotations()).To(HaveKey(istioCSRRejectAnnotation)) + + Consistently(func() bool { + _, err := clientset.AppsV1().Deployments(secondNS.Name).Get(ctx, istioCSRGRPCServiceName, metav1.GetOptions{}) + return apierrors.IsNotFound(err) + }, "30s", "5s").Should(BeTrue()) + }) + + It("should support ClusterIssuer for IstioCSR reconciliation", Label("ISTIOCSR-003"), func() { + ns, err := loader.CreateTestingNS("istiocsr-clusterissuer", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + By("creating CA certificate in test namespace using self-signed ClusterIssuer") + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), ns.Name) + DeferCleanup(func() { + loader.DeleteFromFile(testassets.ReadFile, filepath.Join("testdata", "self_signed", "certificate.yaml"), ns.Name) + }) + Expect(waitForCertificateReadiness(ctx, istioCSRCACertificateName, ns.Name)).NotTo(HaveOccurred()) + + By("copying CA secret to cert-manager namespace for CA ClusterIssuer readiness") + Expect(copySecretToNamespace(ctx, clientset, ns.Name, operandNamespace, istioCSRCASecretName)).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = clientset.CoreV1().Secrets(operandNamespace).Delete(ctx, istioCSRCASecretName, metav1.DeleteOptions{}) + }) + + By("creating CA ClusterIssuer backed by root-secret") + caClusterIssuer := &certmanagerv1.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: istioCSRCAClusterIssuerName, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + CA: &certmanagerv1.CAIssuer{ + SecretName: istioCSRCASecretName, + }, + }, + }, + } + _, err = certmanagerClient.CertmanagerV1().ClusterIssuers().Create(ctx, caClusterIssuer, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = certmanagerClient.CertmanagerV1().ClusterIssuers().Delete(ctx, istioCSRCAClusterIssuerName, metav1.DeleteOptions{}) + }) + Expect(waitForClusterIssuerReadiness(ctx, istioCSRCAClusterIssuerName)).NotTo(HaveOccurred()) + + istioCSR := newIstioCSR(ns.Name, istioCSRBuildConfig{ + issuerRefKind: "ClusterIssuer", + issuerRefName: istioCSRCAClusterIssuerName, + }) + createIstioCSR(ns.Name, istioCSR) + status := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + Expect(status.IstioCSRGRPCEndpoint).NotTo(BeEmpty()) + }) + + It("should report degraded state for unsupported ACME issuer", Label("ISTIOCSR-004"), func() { + ns, err := loader.CreateTestingNS("istiocsr-acme", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + acmeIssuer := &certmanagerv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acme-issuer", + Namespace: ns.Name, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + ACME: &acmev1.ACMEIssuer{ + Email: "istiocsr@example.com", + Server: istioCSRACMEPlaceholderServer, + PrivateKey: certmanagermetav1.SecretKeySelector{ + LocalObjectReference: certmanagermetav1.LocalObjectReference{Name: "acme-private-key"}, + }, + }, + }, + }, + } + _, err = certmanagerClient.CertmanagerV1().Issuers(ns.Name).Create(ctx, acmeIssuer, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + istioCSR := newIstioCSR(ns.Name, istioCSRBuildConfig{ + issuerRefName: "acme-issuer", + }) + createIstioCSR(ns.Name, istioCSR) + By("waiting for IstioCSR Degraded=True with unsupported ACME issuer message") + Expect(waitForIstioCSRConditionMessage(ctx, loader, ns.Name, istioCSRResourceName, v1alpha1.Degraded, metav1.ConditionTrue, "unsupported ACME issuer", highTimeout, slowPollInterval)).NotTo(HaveOccurred()) + }) + + It("should reconcile custom gRPC port to service and status endpoint", Label("ISTIOCSR-005"), func() { + ns, err := loader.CreateTestingNS("istiocsr-port", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + istioCSR := newIstioCSR(ns.Name, istioCSRBuildConfig{ + addServerBlock: true, + serverPort: 7443, + }) + createIstioCSR(ns.Name, istioCSR) + status := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + Expect(status.IstioCSRGRPCEndpoint).To(ContainSubstring(":7443")) + + svc, err := clientset.CoreV1().Services(ns.Name).Get(ctx, istioCSRGRPCServiceName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + var grpcPort int32 + for _, p := range svc.Spec.Ports { + if p.Name == istioCSRGRPCServicePortName { + grpcPort = p.Port + } + } + Expect(grpcPort).To(Equal(int32(7443))) + }) + + It("should reconcile custom log arguments after deployment drift", Label("ISTIOCSR-006"), func() { + ns, err := loader.CreateTestingNS("istiocsr-log", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + istioCSR := newIstioCSR(ns.Name, istioCSRBuildConfig{ + logLevel: 5, + logFormat: "json", + }) + createIstioCSR(ns.Name, istioCSR) + expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + + deployment, err := clientset.AppsV1().Deployments(ns.Name).Get(ctx, istioCSRGRPCServiceName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(deployment.Spec.Template.Spec.Containers).NotTo(BeEmpty()) + Expect(deployment.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--log-level=5")) + Expect(deployment.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--log-format=json")) + + deployment.Spec.Template.Spec.Containers[0].Args = []string{"--log-level=1", "--log-format=text"} + _, err = clientset.AppsV1().Deployments(ns.Name).Update(ctx, deployment, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + current, err := clientset.AppsV1().Deployments(ns.Name).Get(ctx, istioCSRGRPCServiceName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(current.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--log-level=5")) + g.Expect(current.Spec.Template.Spec.Containers[0].Args).To(ContainElement("--log-format=json")) + }, highTimeout, slowPollInterval).Should(Succeed()) + }) + + It("should recreate ServiceAccount when deleted", Label("ISTIOCSR-017"), func() { + ns, err := loader.CreateTestingNS("istiocsr-sa", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + createIstioCSR(ns.Name, newIstioCSR(ns.Name, istioCSRBuildConfig{})) + status := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + Expect(clientset.CoreV1().ServiceAccounts(ns.Name).Delete(ctx, status.ServiceAccount, metav1.DeleteOptions{})).NotTo(HaveOccurred()) + Expect(pollTillServiceAccountAvailable(ctx, clientset, ns.Name, status.ServiceAccount)).NotTo(HaveOccurred()) + }) + + It("should reconcile gRPC service drift", Label("ISTIOCSR-018"), func() { + ns, err := loader.CreateTestingNS("istiocsr-service", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + createIstioCSR(ns.Name, newIstioCSR(ns.Name, istioCSRBuildConfig{})) + status := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + expectedPort := getGRPCPortFromEndpoint(status.IstioCSRGRPCEndpoint) + + service, err := clientset.CoreV1().Services(ns.Name).Get(ctx, istioCSRGRPCServiceName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + for i := range service.Spec.Ports { + if service.Spec.Ports[i].Name == istioCSRGRPCServicePortName { + service.Spec.Ports[i].Port = 9443 + } + } + _, err = clientset.CoreV1().Services(ns.Name).Update(ctx, service, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + current, err := clientset.CoreV1().Services(ns.Name).Get(ctx, istioCSRGRPCServiceName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + var grpcPort int32 + for _, p := range current.Spec.Ports { + if p.Name == istioCSRGRPCServicePortName { + grpcPort = p.Port + } + } + g.Expect(grpcPort).To(Equal(expectedPort)) + }, highTimeout, slowPollInterval).Should(Succeed()) + }) + + It("should recreate deleted network policy", Label("ISTIOCSR-019"), func() { + ns, err := loader.CreateTestingNS("istiocsr-netpol", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + createIstioCSR(ns.Name, newIstioCSR(ns.Name, istioCSRBuildConfig{})) + expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + + networkPolicies, err := clientset.NetworkingV1().NetworkPolicies(ns.Name).List(ctx, metav1.ListOptions{ + LabelSelector: istioCSRManagedResourceLabel, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(networkPolicies.Items).NotTo(BeEmpty()) + + targetPolicy := networkPolicies.Items[0].Name + Expect(clientset.NetworkingV1().NetworkPolicies(ns.Name).Delete(ctx, targetPolicy, metav1.DeleteOptions{})).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + _, err := clientset.NetworkingV1().NetworkPolicies(ns.Name).Get(ctx, targetPolicy, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + }, highTimeout, slowPollInterval).Should(Succeed()) + }) + + It("should publish complete IstioCSR status contract", Label("ISTIOCSR-020"), func() { + ns, err := loader.CreateTestingNS("istiocsr-status", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + createIstioCSR(ns.Name, newIstioCSR(ns.Name, istioCSRBuildConfig{})) + status := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) + Expect(status.IstioCSRImage).NotTo(BeEmpty(), "status.istioCSRImage should be populated") + Expect(status.IstioCSRGRPCEndpoint).NotTo(BeEmpty(), "status.istioCSRGRPCEndpoint should be populated") + Expect(status.ServiceAccount).NotTo(BeEmpty(), "status.serviceAccount should be populated") + Expect(status.ClusterRole).NotTo(BeEmpty(), "status.clusterRole should be populated") + Expect(status.ClusterRoleBinding).NotTo(BeEmpty(), "status.clusterRoleBinding should be populated") + + Expect(status.IstioCSRGRPCEndpoint).To(ContainSubstring(fmt.Sprintf(".%s.svc:", ns.Name))) + Expect(getGRPCPortFromEndpoint(status.IstioCSRGRPCEndpoint)).To(BeNumerically(">", 0)) + + _, err = clientset.RbacV1().ClusterRoles().Get(ctx, status.ClusterRole, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = clientset.RbacV1().ClusterRoleBindings().Get(ctx, status.ClusterRoleBinding, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + _, err = clientset.CoreV1().ServiceAccounts(ns.Name).Get(ctx, status.ServiceAccount, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should report degraded when referenced CA ConfigMap key is missing", Label("ISTIOCSR-023"), func() { + ns, err := loader.CreateTestingNS("istiocsr-missing-key", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "missing-key-ca", + Namespace: ns.Name, + }, + Data: map[string]string{ + "other-key.pem": "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----", + }, + } + Expect(library.UpsertConfigMap(ctx, clientset, cm)).NotTo(HaveOccurred()) + + istioCSR := newIstioCSR(ns.Name, istioCSRBuildConfig{ + addCustomCAConfigMap: true, + customCAConfigMapName: cm.Name, + customCAConfigMapNamespace: "", + customCAConfigMapKey: "ca-cert.pem", + }) + createIstioCSR(ns.Name, istioCSR) + By("waiting for IstioCSR Degraded=True with missing ConfigMap key message") + Expect(waitForIstioCSRConditionMessage(ctx, loader, ns.Name, istioCSRResourceName, v1alpha1.Degraded, metav1.ConditionTrue, istioCSRMissingConfigMapKeyMessage, highTimeout, slowPollInterval)).NotTo(HaveOccurred()) + }) + + It("should report degraded when referenced CA ConfigMap namespace does not exist", Label("ISTIOCSR-027"), func() { + ns, err := loader.CreateTestingNS("istiocsr-missing-ns", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(ns.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + createIssuerPrerequisites(ns.Name) + + istioCSR := newIstioCSR(ns.Name, istioCSRBuildConfig{ + addCustomCAConfigMap: true, + customCAConfigMapName: "external-ca", + customCAConfigMapNamespace: "non-existent-istiocsr-ca-ns", + customCAConfigMapKey: "ca.crt", + }) + createIstioCSR(ns.Name, istioCSR) + By("waiting for IstioCSR Degraded=True with missing CA ConfigMap namespace message") + Expect(waitForIstioCSRConditionMessage(ctx, loader, ns.Name, istioCSRResourceName, v1alpha1.Degraded, metav1.ConditionTrue, "failed to fetch CA certificate ConfigMap", highTimeout, slowPollInterval)).NotTo(HaveOccurred()) + }) + + Context("OpenShift Service Mesh smoke", Label("Feature:IstioCSR-ServiceMesh"), Ordered, func() { + var ( + istioCPNamespace string + clusterID string + meshMemberNS *corev1.Namespace + meshPeerNS *corev1.Namespace + nonMemberNS *corev1.Namespace + istioCSRStatus v1alpha1.IstioCSRStatus + ) + + BeforeAll(func() { + clusterID = deriveClusterID(cfg) + + DeferCleanup(func() { + _ = cleanupOSSMIstioCSROperand(ctx, loader) + }) + + By("creating istio-csr issuer chain for OSSM v3 smoke") + err := ensureOSSMIssuerChain(ctx, clientset, certmanagerClient) + if err != nil { + Skip(fmt.Sprintf("istio-csr issuer prerequisites not available: %v", err)) + } + + By("creating IstioCSR operand in istio-csr namespace") + err = ensureOSSMIstioCSROperand(ctx, loader, clusterID) + Expect(err).NotTo(HaveOccurred()) + istioCSRStatus = expectIstioCSROperandReady(ctx, clientset, loader, ossmIstioCSRNamespace) + Expect(istioCSRStatus.IstioCSRGRPCEndpoint).NotTo(BeEmpty()) + Expect(istioCSRStatus.ServiceAccount).NotTo(BeEmpty()) + + cpNamespace, err := ensureServiceMeshForSmoke(ctx, cfg, loader, clientset, istioCSRStatus.IstioCSRGRPCEndpoint, clusterID) + if err != nil { + Skip(fmt.Sprintf("OpenShift Service Mesh v3 not available: %v", err)) + } + istioCPNamespace = cpNamespace + + meshMemberNS, err = loader.CreateTestingNS("osm-apps-1", true) + Expect(err).NotTo(HaveOccurred()) + Expect(labelNamespaceForIstioInjection(ctx, clientset, meshMemberNS.Name)).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(meshMemberNS.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + meshPeerNS, err = loader.CreateTestingNS("osm-apps-2", true) + Expect(err).NotTo(HaveOccurred()) + Expect(labelNamespaceForIstioInjection(ctx, clientset, meshPeerNS.Name)).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(meshPeerNS.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + + nonMemberNS, err = loader.CreateTestingNS("osm-non-member", true) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + loader.DeleteTestingNS(nonMemberNS.Name, func() bool { return CurrentSpecReport().Failed() }) + }) + }) + + It("should create istio-ca-root-cert in istio-injection=enabled namespace", Label("OSM-SMOKE-TC-001"), func() { + By("waiting for root CA ConfigMap in istio-injection=enabled member namespace") + err := pollTillConfigMapAvailable(ctx, clientset, meshMemberNS.Name, "istio-ca-root-cert") + Expect(err).NotTo(HaveOccurred()) + + cm, err := clientset.CoreV1().ConfigMaps(meshMemberNS.Name).Get(ctx, "istio-ca-root-cert", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(HaveKey("root-cert.pem")) + Expect(cm.Data["root-cert.pem"]).NotTo(BeEmpty()) + + By("verifying root CA ConfigMap is not created in a non-injected namespace") + err = pollTillConfigMapRemains(ctx, clientset, nonMemberNS.Name, "istio-ca-root-cert", lowTimeout) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return cert-chain for mesh workload SPIFFE identity via gRPC [Skipped:Disconnected]", Label("OSM-SMOKE-TC-002"), func() { + const ( + grpcAppName = "grpcurl-istio-csr-osm" + meshWorkloadSA = "mesh-workload" + ) + + By("creating mesh workload service account in injected namespace") + _, err := clientset.CoreV1().ServiceAccounts(meshMemberNS.Name).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: meshWorkloadSA, + Namespace: meshMemberNS.Name, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = clientset.CoreV1().ServiceAccounts(meshMemberNS.Name).Delete(ctx, meshWorkloadSA, metav1.DeleteOptions{}) + }) + + By("preparing grpcurl job in injected namespace with matching mesh workload SPIFFE URI") + protoBytes, err := testassets.ReadFile("testdata/ca.proto") + Expect(err).NotTo(HaveOccurred()) + protoCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "proto-cm-osm", + Namespace: meshMemberNS.Name, + }, + Data: map[string]string{ + "ca.proto": string(protoBytes), + }, + } + Expect(library.UpsertConfigMap(ctx, clientset, protoCM)).NotTo(HaveOccurred()) + DeferCleanup(func() { + _ = clientset.CoreV1().ConfigMaps(meshMemberNS.Name).Delete(ctx, protoCM.Name, metav1.DeleteOptions{}) + }) + + Eventually(func(g Gomega) { + _, err := clientset.CoreV1().Secrets(istioCPNamespace).Get(ctx, istioCSRIstiodTLSSecretName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + }, highTimeout, slowPollInterval).Should(Succeed()) + + Expect(copySecretToNamespace(ctx, clientset, istioCPNamespace, meshMemberNS.Name, istioCSRIstiodTLSSecretName)).NotTo(HaveOccurred()) + + err = pollTillServiceAccountAvailable(ctx, clientset, meshMemberNS.Name, meshWorkloadSA) + Expect(err).NotTo(HaveOccurred()) + + csr := generateMeshWorkloadCSR(meshMemberNS.Name, meshWorkloadSA) + + loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( + IstioCSRGRPCurlJobConfig{ + CertificateSigningRequest: csr, + IstioCSRStatus: istioCSRStatus, + JobName: grpcAppName, + ProtoConfigMapName: protoCM.Name, + ServiceAccountName: meshWorkloadSA, + }, + ), filepath.Join("testdata", "istio", "grpcurl_job.yaml"), meshMemberNS.Name) + DeferCleanup(func() { + policy := metav1.DeletePropagationBackground + _ = clientset.BatchV1().Jobs(meshMemberNS.Name).Delete(ctx, grpcAppName, metav1.DeleteOptions{PropagationPolicy: &policy}) + }) + + Expect(pollTillJobCompleted(ctx, clientset, meshMemberNS.Name, grpcAppName)).NotTo(HaveOccurred()) + + pods, err := clientset.CoreV1().Pods(meshMemberNS.Name).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", grpcAppName), + }) + Expect(err).NotTo(HaveOccurred()) + + var succeededPodName string + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodSucceeded { + succeededPodName = pod.Name + } + } + Expect(succeededPodName).NotTo(BeEmpty()) + + logStream, err := clientset.CoreV1().Pods(meshMemberNS.Name).GetLogs(succeededPodName, &corev1.PodLogOptions{}).Stream(ctx) + Expect(err).NotTo(HaveOccurred()) + defer logStream.Close() + + logData, err := io.ReadAll(logStream) + Expect(err).NotTo(HaveOccurred()) + + entry, err := parseGRPCurlLogEntry(logData) + Expect(err).NotTo(HaveOccurred()) + Expect(entry.CertChain).NotTo(BeEmpty()) + + for _, certPEM := range entry.CertChain { + Expect(certPEM).NotTo(BeEmpty()) + } + }) + + It("should create istio-csr CertificateRequests when mesh workloads start", Label("OSM-SMOKE-TC-003"), func() { + By("deploying sample mesh workloads in injected namespaces") + deployMeshSampleWorkloads(ctx, loader, meshMemberNS.Name) + deployMeshSampleWorkloads(ctx, loader, meshPeerNS.Name) + + By("waiting for injected sleep and httpbin pods to become ready") + Expect(waitForInjectedDeploymentReady(ctx, clientset, meshMemberNS.Name, "sleep")).NotTo(HaveOccurred()) + Expect(waitForInjectedDeploymentReady(ctx, clientset, meshMemberNS.Name, "httpbin")).NotTo(HaveOccurred()) + Expect(waitForInjectedDeploymentReady(ctx, clientset, meshPeerNS.Name, "sleep")).NotTo(HaveOccurred()) + Expect(waitForInjectedDeploymentReady(ctx, clientset, meshPeerNS.Name, "httpbin")).NotTo(HaveOccurred()) + + By("waiting for istio-csr CertificateRequests in istio-system") + Expect(waitForCertificateRequestsFromIstioCSR(ctx, istioCPNamespace, 1)).NotTo(HaveOccurred()) + }) + + It("should allow cross-namespace mesh traffic between injected namespaces", Label("OSM-SMOKE-TC-004"), func() { + By("ensuring sample workloads are present in both injected namespaces") + if _, err := clientset.AppsV1().Deployments(meshMemberNS.Name).Get(ctx, "sleep", metav1.GetOptions{}); err != nil { + deployMeshSampleWorkloads(ctx, loader, meshMemberNS.Name) + } + if _, err := clientset.AppsV1().Deployments(meshPeerNS.Name).Get(ctx, "httpbin", metav1.GetOptions{}); err != nil { + deployMeshSampleWorkloads(ctx, loader, meshPeerNS.Name) + } + Expect(waitForInjectedDeploymentReady(ctx, clientset, meshMemberNS.Name, "sleep")).NotTo(HaveOccurred()) + Expect(waitForInjectedDeploymentReady(ctx, clientset, meshPeerNS.Name, "httpbin")).NotTo(HaveOccurred()) + + By("curling peer httpbin service from sleep pod across namespaces") + Eventually(func(g Gomega) { + sleepPod, err := getRunningPodName(ctx, clientset, meshMemberNS.Name, "app=sleep") + g.Expect(err).NotTo(HaveOccurred()) + + output, err := execInPod(ctx, cfg, clientset, meshMemberNS.Name, sleepPod, "sleep", + "curl", "-sIL", fmt.Sprintf("http://httpbin.%s.svc.cluster.local:8000/status/200", meshPeerNS.Name)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("HTTP/1.1 200")) + g.Expect(output).To(ContainSubstring("server: envoy")) + }, highTimeout, slowPollInterval).Should(Succeed()) + }) + }) + +}) diff --git a/test/e2e/istio_csr_test.go b/test/e2e/istio_csr_test.go index 5fca6ef91..c9aa037c1 100644 --- a/test/e2e/istio_csr_test.go +++ b/test/e2e/istio_csr_test.go @@ -7,7 +7,6 @@ import ( "context" "crypto/x509" "crypto/x509/pkix" - "encoding/json" "fmt" "io" "log" @@ -19,27 +18,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" "github.com/openshift/cert-manager-operator/test/library" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -type LogEntry struct { - CertChain []string `json:"certChain"` -} - -type IstioCSRConfig struct { - ClusterID string - IstioDataPlaneNamespaceSelector string -} - var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioCSR"), func() { ctx := context.TODO() var clientset *kubernetes.Clientset - generateCSR := func() string { + generateCSR := func(namespace string) string { csrTemplate := &x509.CertificateRequest{ Subject: pkix.Name{ Organization: []string{"My Organization"}, @@ -49,7 +38,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC Province: []string{"California"}, }, URIs: []*url.URL{ - {Scheme: "spiffe", Host: "cluster.local", Path: "/ns/istio-system/sa/cert-manager-istio-csr"}, + {Scheme: "spiffe", Host: "cluster.local", Path: fmt.Sprintf("/ns/%s/sa/cert-manager-istio-csr", namespace)}, }, SignatureAlgorithm: x509.SHA256WithRSA, } @@ -59,28 +48,19 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC return csr } - waitForIstioCSRReady := func(ns *corev1.Namespace) v1alpha1.IstioCSRStatus { - By("poll till cert-manager-istio-csr deployment is available") - err := pollTillDeploymentAvailable(ctx, clientset, ns.Name, "cert-manager-istio-csr") - Expect(err).Should(BeNil()) - - By("poll till istiocsr object is available") - istioCSRStatus, err := pollTillIstioCSRAvailable(ctx, loader, ns.Name, "default") - Expect(err).Should(BeNil()) - - return istioCSRStatus - } - BeforeAll(func() { var err error clientset, err = kubernetes.NewForConfig(cfg) Expect(err).Should(BeNil()) - By("increase operator log verbosity") - err = patchSubscriptionWithEnvVars(ctx, loader, map[string]string{ + By("increase operator log verbosity when operator is OLM-managed") + err = tryPatchSubscriptionWithEnvVars(ctx, loader, map[string]string{ "OPERATOR_LOG_LEVEL": "5", }) Expect(err).NotTo(HaveOccurred()) + + By("removing leftover IstioCSR operands from earlier suites in the same job") + Expect(cleanupAllIstioCSROperands(ctx, loader)).NotTo(HaveOccurred()) }) var ns *corev1.Namespace @@ -91,7 +71,8 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC Expect(err).NotTo(HaveOccurred(), "Operator is expected to be available") By("creating a test namespace") - namespace, err := loader.CreateTestingNS("istio-system", true) + // Use a generated name: the real istio-system namespace may already exist when OSSM smoke tests run. + namespace, err := loader.CreateTestingNS("istio-csr-e2e", false) Expect(err).NotTo(HaveOccurred()) ns = namespace @@ -148,7 +129,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC "ca.proto": protoContent, }, } - _, err = clientset.CoreV1().ConfigMaps(ns.Name).Create(ctx, configMap, metav1.CreateOptions{}) + err = library.UpsertConfigMap(ctx, clientset, configMap) Expect(err).Should(BeNil()) DeferCleanup(func() { clientset.CoreV1().ConfigMaps(ns.Name).Delete(ctx, configMap.Name, metav1.DeleteOptions{}) @@ -161,20 +142,20 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC By("creating istiocsr.operator.openshift.io resource") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{}, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + istioCSRConfigForNS(ns.Name, IstioCSRConfig{}), + ), istioCSROperandManifest, ns.Name) defer loader.DeleteFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{}, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + istioCSRConfigForNS(ns.Name, IstioCSRConfig{}), + ), istioCSROperandManifest, ns.Name) - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) By("poll till the service account is available") err := pollTillServiceAccountAvailable(ctx, clientset, ns.Name, serviceAccountName) Expect(err).Should(BeNil()) By("generate csr request") - csr := generateCSR() + csr := generateCSR(ns.Name) By("creating an grpcurl job") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( @@ -215,8 +196,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC logData, err := io.ReadAll(logs) Expect(err).Should(BeNil()) - var entry LogEntry - err = json.Unmarshal(logData, &entry) + entry, err := parseGRPCurlLogEntry(logData) Expect(err).Should(BeNil()) Expect(entry.CertChain).ShouldNot(BeEmpty()) @@ -232,24 +212,24 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC By("creating istiocsr.operator.openshift.io resource with custom clusterID") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{ + istioCSRConfigForNS(ns.Name, IstioCSRConfig{ ClusterID: clusterName, - }, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + }), + ), istioCSROperandManifest, ns.Name) defer loader.DeleteFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{ + istioCSRConfigForNS(ns.Name, IstioCSRConfig{ ClusterID: clusterName, - }, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + }), + ), istioCSROperandManifest, ns.Name) - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) By("poll till the service account is available") err := pollTillServiceAccountAvailable(ctx, clientset, ns.Name, "cert-manager-istio-csr") Expect(err).Should(BeNil()) By("generate csr request") - csr := generateCSR() + csr := generateCSR(ns.Name) By("creating grpcurl job with matching clusterID") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( @@ -291,8 +271,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC logData, err := io.ReadAll(logs) Expect(err).Should(BeNil()) - var entry LogEntry - err = json.Unmarshal(logData, &entry) + entry, err := parseGRPCurlLogEntry(logData) Expect(err).Should(BeNil()) Expect(entry.CertChain).ShouldNot(BeEmpty()) }) @@ -302,24 +281,24 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC By("creating istiocsr.operator.openshift.io resource with custom clusterID") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{ + istioCSRConfigForNS(ns.Name, IstioCSRConfig{ ClusterID: clusterName, - }, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + }), + ), istioCSROperandManifest, ns.Name) defer loader.DeleteFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{ + istioCSRConfigForNS(ns.Name, IstioCSRConfig{ ClusterID: clusterName, - }, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + }), + ), istioCSROperandManifest, ns.Name) - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) By("poll till the service account is available") err := pollTillServiceAccountAvailable(ctx, clientset, ns.Name, "cert-manager-istio-csr") Expect(err).Should(BeNil()) By("generate csr request") - csr := generateCSR() + csr := generateCSR(ns.Name) By("creating grpcurl job with wrong clusterID") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( @@ -405,18 +384,18 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC It("should only create istio-ca-root-cert ConfigMap in namespaces matching the selector", func() { By("creating istiocsr.operator.openshift.io resource with istioDataPlaneNamespaceSelector") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{ + istioCSRConfigForNS(ns.Name, IstioCSRConfig{ IstioDataPlaneNamespaceSelector: "cert-manager.io/test-ca-injection=enabled", - }, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + }), + ), istioCSROperandManifest, ns.Name) defer loader.DeleteFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{ + istioCSRConfigForNS(ns.Name, IstioCSRConfig{ IstioDataPlaneNamespaceSelector: "cert-manager.io/test-ca-injection=enabled", - }, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + }), + ), istioCSROperandManifest, ns.Name) By("waiting for IstioCSR to be ready and deployment to be created") - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) log.Printf("IstioCSR status: %+v", istioCSRStatus) By("verifying ConfigMap creation based on namespace selector") @@ -464,14 +443,14 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC It("should create istio-ca-root-cert ConfigMap in all namespaces when istioDataPlaneNamespaceSelector is not set", func() { By("creating istiocsr.operator.openshift.io resource without istioDataPlaneNamespaceSelector") loader.CreateFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{}, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + istioCSRConfigForNS(ns.Name, IstioCSRConfig{}), + ), istioCSROperandManifest, ns.Name) defer loader.DeleteFromFile(AssetFunc(testassets.ReadFile).WithTemplateValues( - IstioCSRConfig{}, - ), filepath.Join("testdata", "istio", "istio_csr_template.yaml"), ns.Name) + istioCSRConfigForNS(ns.Name, IstioCSRConfig{}), + ), istioCSROperandManifest, ns.Name) By("waiting for IstioCSR to be ready and deployment to be created") - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) log.Printf("IstioCSR status: %+v", istioCSRStatus) By("waiting for istio-ca-root-cert ConfigMap to be created in all test namespaces") @@ -506,6 +485,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC ) type istioCSRTemplateData struct { + IstioNamespace string CustomNamespace string ConfigMapName string ConfigMapKey string @@ -630,11 +610,12 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }), }, } - _, err := clientset.CoreV1().ConfigMaps(caCertConfigMap.Namespace).Create(ctx, caCertConfigMap, metav1.CreateOptions{}) + err := library.UpsertConfigMap(ctx, clientset, caCertConfigMap) Expect(err).ShouldNot(HaveOccurred()) By("Creating IstioCSR resource") templateData := istioCSRTemplateData{ + IstioNamespace: ns.Name, CustomNamespace: "", // Empty string for same namespace ConfigMapName: configMapRefName, ConfigMapKey: configMapRefKey, @@ -645,7 +626,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }) By("waiting for IstioCSR to be ready and deployment to be created") - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) log.Printf("IstioCSR status: %+v", istioCSRStatus) // Verify that the source ConfigMap data is copied to the operator-managed ConfigMap @@ -669,11 +650,12 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }), }, } - _, err := clientset.CoreV1().ConfigMaps(nonCACertConfigMap.Namespace).Create(ctx, nonCACertConfigMap, metav1.CreateOptions{}) + err := library.UpsertConfigMap(ctx, clientset, nonCACertConfigMap) Expect(err).ShouldNot(HaveOccurred()) By("Creating IstioCSR resource") templateData := istioCSRTemplateData{ + IstioNamespace: ns.Name, CustomNamespace: "", // Empty string for same namespace ConfigMapName: configMapRefName, ConfigMapKey: configMapRefKey, @@ -712,11 +694,12 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }), }, } - _, err = clientset.CoreV1().ConfigMaps(caCertConfigMap.Namespace).Create(ctx, caCertConfigMap, metav1.CreateOptions{}) + err = library.UpsertConfigMap(ctx, clientset, caCertConfigMap) Expect(err).ShouldNot(HaveOccurred()) By("Creating IstioCSR resource with custom namespace reference") templateData := istioCSRTemplateData{ + IstioNamespace: ns.Name, CustomNamespace: customNamespace.Name, ConfigMapName: configMapRefName, ConfigMapKey: configMapRefKey, @@ -727,7 +710,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }) By("waiting for IstioCSR to be ready and deployment to be created") - istioCSRStatus := waitForIstioCSRReady(ns) + istioCSRStatus := expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) log.Printf("IstioCSR status: %+v", istioCSRStatus) // Verify that the source ConfigMap data is copied from custom namespace to the operator-managed ConfigMap @@ -754,11 +737,12 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC configMapRefKey: initialCert, }, } - _, err := clientset.CoreV1().ConfigMaps(caCertConfigMap.Namespace).Create(ctx, caCertConfigMap, metav1.CreateOptions{}) + err := library.UpsertConfigMap(ctx, clientset, caCertConfigMap) Expect(err).ShouldNot(HaveOccurred()) By("Creating IstioCSR resource") templateData := istioCSRTemplateData{ + IstioNamespace: ns.Name, CustomNamespace: "", ConfigMapName: configMapRefName, ConfigMapKey: configMapRefKey, @@ -769,7 +753,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }) By("Waiting for IstioCSR to be ready") - _ = waitForIstioCSRReady(ns) + _ = expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) By("Verifying initial ConfigMap is copied") Eventually(func() bool { @@ -818,11 +802,12 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC configMapRefKey: initialCert, }, } - _, err := clientset.CoreV1().ConfigMaps(caCertConfigMap.Namespace).Create(ctx, caCertConfigMap, metav1.CreateOptions{}) + err := library.UpsertConfigMap(ctx, clientset, caCertConfigMap) Expect(err).ShouldNot(HaveOccurred()) By("Creating IstioCSR resource") templateData := istioCSRTemplateData{ + IstioNamespace: ns.Name, CustomNamespace: "", ConfigMapName: configMapRefName, ConfigMapKey: configMapRefKey, @@ -833,7 +818,7 @@ var _ = Describe("Istio-CSR", Ordered, Label("Platform:Generic", "Feature:IstioC }) By("Waiting for IstioCSR to be ready") - _ = waitForIstioCSRReady(ns) + _ = expectIstioCSROperandReady(ctx, clientset, loader, ns.Name) By("Verifying initial ConfigMap is copied and mounted") Eventually(func() bool { diff --git a/test/e2e/servicemesh_helpers_test.go b/test/e2e/servicemesh_helpers_test.go new file mode 100644 index 000000000..42000bbfe --- /dev/null +++ b/test/e2e/servicemesh_helpers_test.go @@ -0,0 +1,651 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmanagermetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + certmanagerclientset "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned" + "github.com/openshift/cert-manager-operator/test/library" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + yamlserializer "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + ossmIstioSystemNamespace = "istio-system" + ossmIstioCSRNamespace = "istio-csr" + ossmIstioCNINamespace = "istio-cni" + ossmIstioInjectionLabel = "istio-injection" + ossmIstioInjectionEnabled = "enabled" + ossmDataPlaneSelector = "istio-injection=enabled" + // Defaults match Makefile E2E_OSM_* vars; override via env or make test-e2e. + ossmDefaultIstioVersion = "v1.24.3" + ossmDefaultOperatorVersion = "3.2.5" + ossmIstiodWaitTimeout = 15 * time.Minute + ossmIssuerSelfSignedName = "istio-csr-selfsigned-issuer" + ossmRootCACertName = "istio-csr-root-ca" + ossmRootCASecretName = "istio-csr-root-ca" + ossmClusterIssuerName = "istio-csr-cluster-issuer" + ossmIstioSystemCACertName = "istio-csr-ca" + ossmIstioSystemCASecretName = "istio-csr-ca" + ossmIstioSystemIssuerName = "istio-csr-issuer" + ossmClusterExtensionName = "servicemeshoperator3" +) + +var ( + clusterExtensionGVR = schema.GroupVersionResource{ + Group: "olm.operatorframework.io", + Version: "v1", + Resource: "clusterextensions", + } + istioCNIGVR = schema.GroupVersionResource{ + Group: "sailoperator.io", + Version: "v1", + Resource: "istiocnis", + } + istioGVR = schema.GroupVersionResource{ + Group: "sailoperator.io", + Version: "v1", + Resource: "istios", + } +) + +// deriveClusterID returns the Istio multi-cluster ID derived from the API server URL. +// Example: https://api.bhb.gcp.devcluster.openshift.com:6443 -> api-bhb-gcp-devcluster-openshift-com:6443 +func deriveClusterID(cfg *rest.Config) string { + host := strings.TrimPrefix(strings.TrimPrefix(cfg.Host, "https://"), "http://") + + hostPart, port, found := strings.Cut(host, ":") + if !found { + port = "6443" + } + return strings.ReplaceAll(hostPart, ".", "-") + ":" + port +} + +// discoverIstiodControlPlaneNamespace returns the namespace of a ready istiod deployment, if any. +func discoverIstiodControlPlaneNamespace(ctx context.Context, clientset *kubernetes.Clientset) (string, bool, error) { + deployments, err := clientset.AppsV1().Deployments("").List(ctx, metav1.ListOptions{ + LabelSelector: "app=istiod", + }) + if err != nil { + return "", false, err + } + for _, deployment := range deployments.Items { + if deployment.Status.ReadyReplicas > 0 { + return deployment.Namespace, true, nil + } + } + + allDeployments, err := clientset.AppsV1().Deployments("").List(ctx, metav1.ListOptions{}) + if err != nil { + return "", false, err + } + for _, deployment := range allDeployments.Items { + if deployment.Name != "istiod" && !strings.HasPrefix(deployment.Name, "istiod-") { + continue + } + if deployment.Status.ReadyReplicas > 0 { + return deployment.Namespace, true, nil + } + } + return "", false, nil +} + +func waitForIstiodControlPlaneNamespace(ctx context.Context, clientset *kubernetes.Clientset, timeout time.Duration) (string, error) { + var controlPlaneNamespace string + err := wait.PollUntilContextTimeout(ctx, fastPollInterval, timeout, true, func(context.Context) (bool, error) { + namespace, found, err := discoverIstiodControlPlaneNamespace(ctx, clientset) + if err != nil { + return false, err + } + if !found { + return false, nil + } + controlPlaneNamespace = namespace + return true, nil + }) + if err != nil { + return "", fmt.Errorf("istiod control plane not available after %s: %w", timeout, err) + } + return controlPlaneNamespace, nil +} + +func sailOperatorAPIAvailable(ctx context.Context, loader library.DynamicResourceLoader) bool { + _, err := loader.DynamicClient.Resource(istioGVR).List(ctx, metav1.ListOptions{Limit: 1}) + return err == nil +} + +func clusterExtensionAPIAvailable(ctx context.Context, loader library.DynamicResourceLoader) bool { + _, err := loader.DynamicClient.Resource(clusterExtensionGVR).List(ctx, metav1.ListOptions{Limit: 1}) + return err == nil +} + +func ossmInstallEnabled() bool { + return os.Getenv("E2E_INSTALL_SERVICE_MESH") != "false" +} + +func ossmIstioVersion() string { + if v := os.Getenv("E2E_OSM_ISTIO_VERSION"); v != "" { + return v + } + return ossmDefaultIstioVersion +} + +func ossmOperatorVersion() string { + if v := os.Getenv("E2E_OSM_OPERATOR_VERSION"); v != "" { + return v + } + return ossmDefaultOperatorVersion +} + +func ensureNamespace(ctx context.Context, clientset *kubernetes.Clientset, name string) error { + _, err := clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + }, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +func applyManifestsFromFile(ctx context.Context, loader library.DynamicResourceLoader, clientset *kubernetes.Clientset, assetFn func(string) ([]byte, error), filename string) error { + content, err := assetFn(filename) + if err != nil { + return fmt.Errorf("read manifest %s: %w", filename, err) + } + + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(content), 4096) + yamlDec := yamlserializer.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + + for { + var raw runtime.RawExtension + if decodeErr := decoder.Decode(&raw); decodeErr != nil { + if decodeErr.Error() == "EOF" { + break + } + return fmt.Errorf("decode manifest %s: %w", filename, decodeErr) + } + if len(raw.Raw) == 0 { + continue + } + + unstructuredObj := &unstructured.Unstructured{} + _, gvk, err := yamlDec.Decode(raw.Raw, nil, unstructuredObj) + if err != nil { + return fmt.Errorf("decode object in %s: %w", filename, err) + } + + switch gvk.GroupVersion().String() + "/" + gvk.Kind { + case "v1/Namespace": + if err := ensureNamespace(ctx, clientset, unstructuredObj.GetName()); err != nil { + return err + } + case "v1/ServiceAccount": + sa := &corev1.ServiceAccount{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, sa); err != nil { + return err + } + _, err := clientset.CoreV1().ServiceAccounts(sa.Namespace).Create(ctx, sa, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + case "rbac.authorization.k8s.io/v1/ClusterRoleBinding": + _, err := loader.DynamicClient.Resource(schema.GroupVersionResource{ + Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings", + }).Create(ctx, unstructuredObj, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + default: + gvr, gvrErr := gvrForGVK(gvk.GroupVersion().String(), gvk.Kind) + if gvrErr != nil { + return gvrErr + } + var dri dynamic.ResourceInterface + if unstructuredObj.GetNamespace() != "" { + dri = loader.DynamicClient.Resource(gvr).Namespace(unstructuredObj.GetNamespace()) + } else { + dri = loader.DynamicClient.Resource(gvr) + } + _, err = dri.Create(ctx, unstructuredObj, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + } + } + return nil +} + +func gvrForGVK(gv, kind string) (schema.GroupVersionResource, error) { + switch gv + "/" + kind { + case "olm.operatorframework.io/v1/ClusterExtension": + return clusterExtensionGVR, nil + case "sailoperator.io/v1/IstioCNI": + return istioCNIGVR, nil + case "sailoperator.io/v1/Istio": + return istioGVR, nil + default: + return schema.GroupVersionResource{}, fmt.Errorf("unsupported GVK %s/%s", gv, kind) + } +} + +func clusterExtensionConditionTrue(obj *unstructured.Unstructured, conditionType string) bool { + conditions, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err != nil || !found { + return false + } + for _, item := range conditions { + cond, ok := item.(map[string]interface{}) + if !ok { + continue + } + condType, _ := cond["type"].(string) + condStatus, _ := cond["status"].(string) + if condType == conditionType && strings.EqualFold(condStatus, "True") { + return true + } + } + return false +} + +func waitForClusterExtensionReady(ctx context.Context, loader library.DynamicResourceLoader) error { + return wait.PollUntilContextTimeout(ctx, slowPollInterval, ossmIstiodWaitTimeout, true, func(context.Context) (bool, error) { + ce, err := loader.DynamicClient.Resource(clusterExtensionGVR).Get(ctx, ossmClusterExtensionName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + if clusterExtensionConditionTrue(ce, "Installed") || clusterExtensionConditionTrue(ce, "Ready") { + return true, nil + } + + phase, found, err := unstructured.NestedString(ce.Object, "status", "phase") + if err == nil && found && strings.EqualFold(phase, "Ready") { + return true, nil + } + + // Fallback: operator installed when sailoperator APIs become available. + return sailOperatorAPIAvailable(ctx, loader), nil + }) +} + +func waitForSailIstioReady(ctx context.Context, loader library.DynamicResourceLoader) error { + return wait.PollUntilContextTimeout(ctx, slowPollInterval, ossmIstiodWaitTimeout, true, func(context.Context) (bool, error) { + istioCR, err := loader.DynamicClient.Resource(istioGVR).Get(ctx, "default", metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + status, found, err := unstructured.NestedString(istioCR.Object, "status", "state") + if err != nil || !found { + return false, nil + } + return strings.EqualFold(status, "Healthy"), nil + }) +} + +func waitForSailIstioCNIReady(ctx context.Context, loader library.DynamicResourceLoader) error { + return wait.PollUntilContextTimeout(ctx, slowPollInterval, ossmIstiodWaitTimeout, true, func(context.Context) (bool, error) { + cniCR, err := loader.DynamicClient.Resource(istioCNIGVR).Get(ctx, "default", metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + if ready, found, err := unstructured.NestedBool(cniCR.Object, "status", "ready"); err == nil && found && ready { + return true, nil + } + state, found, err := unstructured.NestedString(cniCR.Object, "status", "state") + if err != nil || !found { + return false, nil + } + return strings.EqualFold(state, "Healthy"), nil + }) +} + +func installOSSMv3(ctx context.Context, cfg *rest.Config, loader library.DynamicResourceLoader, clientset *kubernetes.Clientset, caAddress, clusterID string) error { + templateValues := OSSMv3Config{ + OperatorVersion: ossmOperatorVersion(), + IstioVersion: ossmIstioVersion(), + ClusterID: clusterID, + CAAddress: caAddress, + } + assetFn := AssetFunc(testassets.ReadFile).WithTemplateValues(templateValues) + + By("installing OSSM v3 ClusterExtension") + if err := applyManifestsFromFile(ctx, loader, clientset, assetFn, filepath.Join("testdata", "servicemesh", "v3", "cluster-extension.yaml")); err != nil { + return fmt.Errorf("apply cluster extension: %w", err) + } + Expect(waitForClusterExtensionReady(ctx, loader)).NotTo(HaveOccurred(), "ClusterExtension should become ready before installing Istio operands") + + By("installing OSSM v3 IstioCNI") + if err := applyManifestsFromFile(ctx, loader, clientset, assetFn, filepath.Join("testdata", "servicemesh", "v3", "istio-cni.yaml")); err != nil { + return fmt.Errorf("apply istio cni: %w", err) + } + Expect(waitForSailIstioCNIReady(ctx, loader)).NotTo(HaveOccurred(), "IstioCNI should become ready") + + By("installing OSSM v3 Istio control plane wired to istio-csr") + if err := applyManifestsFromFile(ctx, loader, clientset, assetFn, filepath.Join("testdata", "servicemesh", "v3", "istio.yaml")); err != nil { + return fmt.Errorf("apply istio: %w", err) + } + Expect(waitForSailIstioReady(ctx, loader)).NotTo(HaveOccurred(), "Istio CR should become Healthy") + + _, err := waitForIstiodControlPlaneNamespace(ctx, clientset, ossmIstiodWaitTimeout) + return err +} + +func ensureOSSMIssuerChain(ctx context.Context, clientset *kubernetes.Clientset, certClient *certmanagerclientset.Clientset) error { + By("creating self-signed issuer for istio-csr root CA in cert-manager namespace") + selfSignedIssuer := &certmanagerv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: ossmIssuerSelfSignedName, + Namespace: operandNamespace, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + SelfSigned: &certmanagerv1.SelfSignedIssuer{}, + }, + }, + } + if err := ensureIssuer(ctx, certClient, selfSignedIssuer); err != nil { + return err + } + + By("creating istio-csr root CA certificate in cert-manager namespace") + rootCA := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: ossmRootCACertName, + Namespace: operandNamespace, + }, + Spec: certmanagerv1.CertificateSpec{ + CommonName: ossmRootCACertName, + SecretName: ossmRootCASecretName, + IsCA: true, + Duration: &metav1.Duration{Duration: 3 * time.Hour}, + IssuerRef: certmanagermetav1.IssuerReference{ + Name: ossmIssuerSelfSignedName, + Kind: "Issuer", + Group: "cert-manager.io", + }, + }, + } + if err := ensureCertificate(ctx, certClient, rootCA); err != nil { + return err + } + if err := waitForCertificateReadiness(ctx, ossmRootCACertName, operandNamespace); err != nil { + return err + } + + By("creating istio-csr cluster issuer") + clusterIssuer := &certmanagerv1.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: ossmClusterIssuerName}, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + CA: &certmanagerv1.CAIssuer{ + SecretName: ossmRootCASecretName, + }, + }, + }, + } + if err := ensureClusterIssuer(ctx, certClient, clusterIssuer); err != nil { + return err + } + if err := waitForClusterIssuerReadiness(ctx, ossmClusterIssuerName); err != nil { + return err + } + + if err := ensureNamespace(ctx, clientset, ossmIstioSystemNamespace); err != nil { + return err + } + + By("creating istio-system CA certificate") + istioCA := &certmanagerv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: ossmIstioSystemCACertName, + Namespace: ossmIstioSystemNamespace, + }, + Spec: certmanagerv1.CertificateSpec{ + CommonName: ossmIstioSystemCACertName, + SecretName: ossmIstioSystemCASecretName, + IsCA: true, + Duration: &metav1.Duration{Duration: 2 * time.Hour}, + IssuerRef: certmanagermetav1.IssuerReference{ + Name: ossmClusterIssuerName, + Kind: "ClusterIssuer", + Group: "cert-manager.io", + }, + }, + } + if err := ensureCertificate(ctx, certClient, istioCA); err != nil { + return err + } + if err := waitForCertificateReadiness(ctx, ossmIstioSystemCACertName, ossmIstioSystemNamespace); err != nil { + return err + } + + By("creating istio-system issuer for istio-csr operand") + istioIssuer := &certmanagerv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: ossmIstioSystemIssuerName, + Namespace: ossmIstioSystemNamespace, + }, + Spec: certmanagerv1.IssuerSpec{ + IssuerConfig: certmanagerv1.IssuerConfig{ + CA: &certmanagerv1.CAIssuer{ + SecretName: ossmIstioSystemCASecretName, + }, + }, + }, + } + if err := ensureIssuer(ctx, certClient, istioIssuer); err != nil { + return err + } + return waitForIssuerReadiness(ctx, ossmIstioSystemIssuerName, ossmIstioSystemNamespace) +} + +func cleanupOSSMIstioCSROperand(ctx context.Context, loader library.DynamicResourceLoader) error { + By("cleaning up OSSM IstioCSR operand in istio-csr namespace") + client := loader.DynamicClient.Resource(istiocsrSchema).Namespace(ossmIstioCSRNamespace) + err := client.Delete(ctx, istioCSRResourceName, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + return wait.PollUntilContextTimeout(ctx, slowPollInterval, lowTimeout, true, func(context.Context) (bool, error) { + _, err := client.Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + }) +} + +func ensureOSSMIstioCSROperand(ctx context.Context, loader library.DynamicResourceLoader, clusterID string) error { + clientset, ok := loader.KubeClient.(*kubernetes.Clientset) + if !ok { + return fmt.Errorf("kubernetes client is not a *kubernetes.Clientset") + } + if err := ensureNamespace(ctx, clientset, ossmIstioCSRNamespace); err != nil { + return err + } + + By("creating IstioCSR operand for OSSM v3 smoke") + loader.CreateFromFile( + AssetFunc(testassets.ReadFile).WithTemplateValues(IstioCSRConfig{ + Namespace: ossmIstioCSRNamespace, + ClusterID: clusterID, + Profile: istioCSRProfileOSSM, + }), + istioCSROperandManifest, + ossmIstioCSRNamespace, + ) + return nil +} + +// ensureServiceMeshForSmoke discovers or installs OSSM v3 and returns the istiod namespace. +func ensureServiceMeshForSmoke(ctx context.Context, cfg *rest.Config, loader library.DynamicResourceLoader, clientset *kubernetes.Clientset, caAddress, clusterID string) (string, error) { + if namespace, found, err := discoverIstiodControlPlaneNamespace(ctx, clientset); err != nil { + return "", err + } else if found { + By(fmt.Sprintf("reusing existing istiod control plane in namespace %s", namespace)) + if sailOperatorAPIAvailable(ctx, loader) { + _, getErr := loader.DynamicClient.Resource(istioGVR).Get(ctx, "default", metav1.GetOptions{}) + if getErr == nil { + if err := waitForSailIstioReady(ctx, loader); err != nil { + return "", fmt.Errorf("existing Istio CR is not Healthy: %w", err) + } + } else if !apierrors.IsNotFound(getErr) { + return "", getErr + } + } + if _, err := waitForIstiodControlPlaneNamespace(ctx, clientset, lowTimeout); err != nil { + return "", fmt.Errorf("existing istiod is not ready: %w", err) + } + return namespace, nil + } + + if !ossmInstallEnabled() { + return "", fmt.Errorf("istiod control plane not found and E2E_INSTALL_SERVICE_MESH=false") + } + if !clusterExtensionAPIAvailable(ctx, loader) { + return "", fmt.Errorf("ClusterExtension API (olm.operatorframework.io/v1) is not available on this cluster") + } + + By("installing OpenShift Service Mesh v3 for multi-operand smoke tests") + if err := installOSSMv3(ctx, cfg, loader, clientset, caAddress, clusterID); err != nil { + return "", err + } + return ossmIstioSystemNamespace, nil +} + +func labelNamespaceForIstioInjection(ctx context.Context, clientset *kubernetes.Clientset, namespace string) error { + return wait.PollUntilContextTimeout(ctx, fastPollInterval, lowTimeout, true, func(context.Context) (bool, error) { + ns, err := clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + return false, err + } + if ns.Labels == nil { + ns.Labels = map[string]string{} + } + ns.Labels[ossmIstioInjectionLabel] = ossmIstioInjectionEnabled + _, err = clientset.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) + if err != nil { + return false, nil + } + return true, nil + }) +} + +func deployMeshSampleWorkloads(ctx context.Context, loader library.DynamicResourceLoader, namespace string) { + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "servicemesh", "httpbin.yaml"), namespace) + loader.CreateFromFile(testassets.ReadFile, filepath.Join("testdata", "servicemesh", "sleep.yaml"), namespace) +} + +func waitForInjectedDeploymentReady(ctx context.Context, clientset *kubernetes.Clientset, namespace, deploymentName string) error { + return wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { + deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + if deployment.Status.ReadyReplicas < 1 { + return false, nil + } + + pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", deploymentName), + }) + if err != nil { + return false, err + } + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning { + continue + } + if len(pod.Spec.Containers) < 2 { + return false, nil + } + appReady := false + sidecarReady := false + for _, cs := range pod.Status.ContainerStatuses { + if !cs.Ready { + continue + } + if cs.Name == "istio-proxy" { + sidecarReady = true + continue + } + appReady = true + } + if appReady && sidecarReady { + return true, nil + } + } + return false, nil + }) +} + +func waitForCertificateRequestsFromIstioCSR(ctx context.Context, namespace string, minCount int) error { + return wait.PollUntilContextTimeout(ctx, slowPollInterval, highTimeout, true, func(context.Context) (bool, error) { + crs, err := certmanagerClient.CertmanagerV1().CertificateRequests(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + count := 0 + for _, cr := range crs.Items { + if strings.HasPrefix(cr.Name, "istio-csr-") { + count++ + } + } + return count >= minCount, nil + }) +} + +func getRunningPodName(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelSelector string) (string, error) { + pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return "", err + } + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + return pod.Name, nil + } + } + return "", fmt.Errorf("no running pod found for selector %s in namespace %s", labelSelector, namespace) +} + diff --git a/test/e2e/testdata/istio/grpcurl_job.yaml b/test/e2e/testdata/istio/grpcurl_job.yaml index c629dd03f..de7e59b1a 100644 --- a/test/e2e/testdata/istio/grpcurl_job.yaml +++ b/test/e2e/testdata/istio/grpcurl_job.yaml @@ -1,15 +1,16 @@ apiVersion: batch/v1 kind: Job metadata: - name: grpcurl-istio-csr + name: {{if .JobName}}{{.JobName}}{{else}}grpcurl-istio-csr{{end}} spec: backoffLimit: 10 completions: 1 template: metadata: labels: - app: grpcurl-istio-csr - name: grpcurl-istio-csr + app: {{if .JobName}}{{.JobName}}{{else}}grpcurl-istio-csr{{end}} + sidecar.istio.io/inject: "false" + name: {{if .JobName}}{{.JobName}}{{else}}grpcurl-istio-csr{{end}} spec: automountServiceAccountToken: false containers: @@ -38,7 +39,7 @@ spec: - mountPath: /var/run/secrets/istio-ca name: sa-token restartPolicy: OnFailure - serviceAccountName: '{{.IstioCSRStatus.ServiceAccount}}' + serviceAccountName: '{{if .ServiceAccountName}}{{.ServiceAccountName}}{{else}}{{.IstioCSRStatus.ServiceAccount}}{{end}}' volumes: - name: sa-token projected: @@ -52,5 +53,5 @@ spec: secret: secretName: istiod-tls - configMap: - name: proto-cm + name: {{if .ProtoConfigMapName}}{{.ProtoConfigMapName}}{{else}}proto-cm{{end}} name: proto diff --git a/test/e2e/testdata/istio/grpcurl_job_with_cluster_id.yaml b/test/e2e/testdata/istio/grpcurl_job_with_cluster_id.yaml index eed0820ff..e1752b8f3 100644 --- a/test/e2e/testdata/istio/grpcurl_job_with_cluster_id.yaml +++ b/test/e2e/testdata/istio/grpcurl_job_with_cluster_id.yaml @@ -9,6 +9,7 @@ spec: metadata: labels: app: {{.JobName}} + sidecar.istio.io/inject: "false" name: {{.JobName}} spec: automountServiceAccountToken: false diff --git a/test/e2e/testdata/istio/istio_csr_template.yaml b/test/e2e/testdata/istio/istio_csr_template.yaml index 1c61cd093..ab1574fe4 100644 --- a/test/e2e/testdata/istio/istio_csr_template.yaml +++ b/test/e2e/testdata/istio/istio_csr_template.yaml @@ -2,22 +2,51 @@ apiVersion: operator.openshift.io/v1alpha1 kind: IstioCSR metadata: name: default - namespace: istio-system + namespace: {{.Namespace}} spec: +{{- if eq .Profile "ossm"}} + controllerConfig: + labels: + env: istio-test +{{- end}} istioCSRConfig: certManager: issuerRef: - name: istio-ca - kind: Issuer group: cert-manager.io + kind: Issuer + name: {{if .IssuerName}}{{.IssuerName}}{{else if eq .Profile "ossm"}}istio-csr-issuer{{else}}istio-ca{{end}} + istio: + namespace: {{if eq .Profile "ossm"}}istio-system{{else}}{{.IstioNamespace}}{{end}} +{{- if eq .Profile "ossm"}} + revisions: + - default +{{- end}} +{{- if eq .Profile "ossm"}} + istioDataPlaneNamespaceSelector: istio-injection=enabled +{{- else if .IstioDataPlaneNamespaceSelector}} + istioDataPlaneNamespaceSelector: "{{.IstioDataPlaneNamespaceSelector}}" +{{- end}} istiodTLSConfig: +{{- if eq .Profile "ossm"}} + certificateDNSNames: + - istiod-default.istio-system.svc + certificateDuration: 1h0m0s + certificateRenewBefore: 30m0s + commonName: istiod.istio-system.svc + maxCertificateDuration: 1h0m0s + privateKeyAlgorithm: RSA + privateKeySize: 4096 trustDomain: cluster.local - istio: - namespace: istio-system -{{- if .ClusterID}} +{{- else}} + trustDomain: cluster.local +{{- end}} +{{- if eq .Profile "ossm"}} + logFormat: text + logLevel: 1 + server: + clusterID: {{.ClusterID}} + port: 443 +{{- else if .ClusterID}} server: clusterID: {{.ClusterID}} {{- end}} -{{- if .IstioDataPlaneNamespaceSelector}} - istioDataPlaneNamespaceSelector: "{{.IstioDataPlaneNamespaceSelector}}" -{{- end}} \ No newline at end of file diff --git a/test/e2e/testdata/istio/istiocsr_with_ca_certificate_template.yaml b/test/e2e/testdata/istio/istiocsr_with_ca_certificate_template.yaml index 5fb74efa6..d455f39a2 100644 --- a/test/e2e/testdata/istio/istiocsr_with_ca_certificate_template.yaml +++ b/test/e2e/testdata/istio/istiocsr_with_ca_certificate_template.yaml @@ -19,4 +19,4 @@ spec: istiodTLSConfig: trustDomain: cluster.local istio: - namespace: istio-system + namespace: {{.IstioNamespace}} diff --git a/test/e2e/testdata/servicemesh/httpbin.yaml b/test/e2e/testdata/servicemesh/httpbin.yaml new file mode 100644 index 000000000..cd5c56a82 --- /dev/null +++ b/test/e2e/testdata/servicemesh/httpbin.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: httpbin +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: httpbin +spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + template: + metadata: + labels: + app: httpbin + version: v1 + spec: + serviceAccountName: httpbin + containers: + - name: httpbin + image: docker.io/kennethreitz/httpbin + command: + - gunicorn + args: + - -b + - 0.0.0.0:8080 + - httpbin:app + ports: + - containerPort: 8080 + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: Service +metadata: + name: httpbin +spec: + ports: + - name: http + port: 8000 + targetPort: 8080 + selector: + app: httpbin diff --git a/test/e2e/testdata/servicemesh/sleep.yaml b/test/e2e/testdata/servicemesh/sleep.yaml new file mode 100644 index 000000000..607539e92 --- /dev/null +++ b/test/e2e/testdata/servicemesh/sleep.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sleep +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sleep +spec: + replicas: 1 + selector: + matchLabels: + app: sleep + template: + metadata: + labels: + app: sleep + spec: + serviceAccountName: sleep + containers: + - name: sleep + image: curlimages/curl:8.5.0 + command: ["/bin/sleep", "infinity"] + imagePullPolicy: IfNotPresent diff --git a/test/e2e/testdata/servicemesh/v3/cluster-extension.yaml b/test/e2e/testdata/servicemesh/v3/cluster-extension.yaml new file mode 100644 index 000000000..b1f3919b4 --- /dev/null +++ b/test/e2e/testdata/servicemesh/v3/cluster-extension.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: servicemesh3-installer + namespace: openshift-operators +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: servicemesh3-installer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: servicemesh3-installer + namespace: openshift-operators +--- +apiVersion: olm.operatorframework.io/v1 +kind: ClusterExtension +metadata: + name: servicemeshoperator3 +spec: + namespace: openshift-operators + serviceAccount: + name: servicemesh3-installer + source: + catalog: + packageName: servicemeshoperator3 + selector: + matchLabels: + olm.operatorframework.io/metadata.name: openshift-redhat-operators + upgradeConstraintPolicy: CatalogProvided + version: {{.OperatorVersion}} + sourceType: Catalog diff --git a/test/e2e/testdata/servicemesh/v3/istio-cni.yaml b/test/e2e/testdata/servicemesh/v3/istio-cni.yaml new file mode 100644 index 000000000..c32eb06ac --- /dev/null +++ b/test/e2e/testdata/servicemesh/v3/istio-cni.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: istio-cni +--- +apiVersion: sailoperator.io/v1 +kind: IstioCNI +metadata: + name: default +spec: + namespace: istio-cni + values: + cni: + excludeNamespaces: + - kube-system + version: {{.IstioVersion}} diff --git a/test/e2e/testdata/servicemesh/v3/istio.yaml b/test/e2e/testdata/servicemesh/v3/istio.yaml new file mode 100644 index 000000000..0d0e70965 --- /dev/null +++ b/test/e2e/testdata/servicemesh/v3/istio.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: istio-system +--- +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: default +spec: + namespace: istio-system + updateStrategy: + inactiveRevisionDeletionGracePeriodSeconds: 30 + type: InPlace + values: + global: + caAddress: {{.CAAddress}} + multiCluster: + clusterName: {{.ClusterID}} + pilot: + env: + ENABLE_CA_SERVER: "false" + version: {{.IstioVersion}} diff --git a/test/e2e/trustmanager_bundle_test.go b/test/e2e/trustmanager_bundle_test.go index a1f7566d2..c6dea5840 100644 --- a/test/e2e/trustmanager_bundle_test.go +++ b/test/e2e/trustmanager_bundle_test.go @@ -58,6 +58,7 @@ import ( trustapi "github.com/cert-manager/trust-manager/pkg/apis/trust/v1alpha1" configopenshiftv1 "github.com/openshift/api/config/v1" "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" + "github.com/openshift/cert-manager-operator/test/library" testutils "github.com/openshift/cert-manager-operator/pkg/controller/istiocsr" corev1 "k8s.io/api/core/v1" @@ -796,7 +797,7 @@ var _ = Describe("Bundle", Ordered, Label("Platform:Generic", "Feature:TrustMana return err }, lowTimeout, fastPollInterval).Should(Succeed()) } else { - _, err := k8sClientSet.CoreV1().ConfigMaps(openshiftConfigNS).Create(ctx, &corev1.ConfigMap{ + Expect(library.UpsertConfigMap(ctx, k8sClientSet, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: userCABundleName, Namespace: openshiftConfigNS, @@ -804,8 +805,7 @@ var _ = Describe("Bundle", Ordered, Label("Platform:Generic", "Feature:TrustMana Data: map[string]string{ "ca-bundle.crt": testCACert, }, - }, metav1.CreateOptions{}) - Expect(err).ShouldNot(HaveOccurred()) + })).ShouldNot(HaveOccurred()) } if originalTrustedCAName != userCABundleName { diff --git a/test/e2e/trustmanager_helpers_test.go b/test/e2e/trustmanager_helpers_test.go index 971a44501..c33393686 100644 --- a/test/e2e/trustmanager_helpers_test.go +++ b/test/e2e/trustmanager_helpers_test.go @@ -14,6 +14,7 @@ import ( . "github.com/onsi/gomega" "github.com/openshift/cert-manager-operator/api/operator/v1alpha1" + "github.com/openshift/cert-manager-operator/test/library" operatorclientv1alpha1 "github.com/openshift/cert-manager-operator/pkg/operator/clientset/versioned/typed/operator/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -319,22 +320,20 @@ func createNamespaceWithCleanup(ctx context.Context, prefix string, labels map[s } func createSourceConfigMap(ctx context.Context, namespace, name, key, data string) { - _, err := k8sClientSet.CoreV1().ConfigMaps(namespace).Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: name}, + Expect(library.UpsertConfigMap(ctx, k8sClientSet, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Data: map[string]string{key: data}, - }, metav1.CreateOptions{}) - Expect(err).ShouldNot(HaveOccurred()) + })).ShouldNot(HaveOccurred()) DeferCleanup(func() { _ = k8sClientSet.CoreV1().ConfigMaps(namespace).Delete(ctx, name, metav1.DeleteOptions{}) }) } func createSourceSecret(ctx context.Context, namespace, name, key, data string) { - _, err := k8sClientSet.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: name}, + Expect(library.UpsertSecret(ctx, k8sClientSet, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Data: map[string][]byte{key: []byte(data)}, - }, metav1.CreateOptions{}) - Expect(err).ShouldNot(HaveOccurred()) + })).ShouldNot(HaveOccurred()) DeferCleanup(func() { _ = k8sClientSet.CoreV1().Secrets(namespace).Delete(ctx, name, metav1.DeleteOptions{}) }) diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 2e39a56f3..9f529b283 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -739,6 +739,30 @@ func getSubscriptionEnvVar(ctx context.Context, loader library.DynamicResourceLo return "", nil } +// certManagerOperatorSubscriptionInstalled reports whether the operator was installed via OLM. +func certManagerOperatorSubscriptionInstalled(ctx context.Context, loader library.DynamicResourceLoader) (bool, error) { + subscriptionClient := loader.DynamicClient.Resource(subscriptionSchema).Namespace("cert-manager-operator") + subs, err := subscriptionClient.List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + return len(subs.Items) > 0, nil +} + +// tryPatchSubscriptionWithEnvVars patches subscription env on OLM clusters and is a no-op when +// the operator was installed without a Subscription (e.g. make deploy + make local-run). +func tryPatchSubscriptionWithEnvVars(ctx context.Context, loader library.DynamicResourceLoader, envVars map[string]string) error { + installed, err := certManagerOperatorSubscriptionInstalled(ctx, loader) + if err != nil { + return err + } + if !installed { + fmt.Fprintf(GinkgoWriter, "no OLM Subscription in cert-manager-operator; skipping subscription env patch\n") + return nil + } + return patchSubscriptionWithEnvVars(ctx, loader, envVars) +} + // patchSubscriptionWithEnvVars uses the k8s dynamic client to patch the only Subscription object // in the cert-manager-operator namespace, inject specified env vars into spec.config.env func patchSubscriptionWithEnvVars(ctx context.Context, loader library.DynamicResourceLoader, envVars map[string]string) error { @@ -1080,6 +1104,9 @@ func pollTillJobCompleted(ctx context.Context, clientset *kubernetes.Clientset, job, err := clientset.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } return false, err } @@ -1187,6 +1214,132 @@ func pollTillIstioCSRAvailable(ctx context.Context, loader library.DynamicResour return istioCSRStatus, err } +// waitForIstioCSRConditionMessage polls until IstioCSR status has a condition of the +// given type and status whose message contains messageSubstring. +func waitForIstioCSRConditionMessage(ctx context.Context, loader library.DynamicResourceLoader, namespace, name, conditionType string, conditionStatus metav1.ConditionStatus, messageSubstring string, timeout, interval time.Duration) error { + return wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(pollCtx context.Context) (bool, error) { + obj, err := loader.DynamicClient.Resource(istiocsrSchema).Namespace(namespace).Get(pollCtx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + conditions, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err != nil { + return false, err + } + if !found { + return false, nil + } + + for _, c := range conditions { + cm, ok := c.(map[string]interface{}) + if !ok { + continue + } + t, _ := cm["type"].(string) + s, _ := cm["status"].(string) + if t != conditionType || s != string(conditionStatus) { + continue + } + msg, _ := cm["message"].(string) + if strings.Contains(msg, messageSubstring) { + return true, nil + } + } + return false, nil + }) +} + +// istioCSRDeploymentMissingHint explains why cert-manager-istio-csr may never appear for a new operand. +func istioCSRDeploymentMissingHint(ctx context.Context, operandNamespace string) string { + if loader.DynamicClient == nil { + return "" + } + + istiocsrClient := loader.DynamicClient.Resource(istiocsrSchema).Namespace(operandNamespace) + obj, err := istiocsrClient.Get(ctx, istioCSRResourceName, metav1.GetOptions{}) + if err == nil { + if ann := obj.GetAnnotations(); ann != nil && ann[istioCSRRejectAnnotation] == "true" { + return "IstioCSR was rejected because another istiocsr instance already exists; only one cluster-wide operand is supported" + } + + conditions, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err == nil && found { + for _, c := range conditions { + cm, ok := c.(map[string]interface{}) + if !ok { + continue + } + t, _ := cm["type"].(string) + s, _ := cm["status"].(string) + if t == string(v1alpha1.Degraded) && s == string(metav1.ConditionTrue) { + msg, _ := cm["message"].(string) + if msg != "" { + return fmt.Sprintf("IstioCSR is Degraded: %s", msg) + } + } + } + } + + spec, found, err := unstructured.NestedMap(obj.Object, "spec", "istioCSRConfig", "istio") + if err == nil && found { + istioNS, _ := spec["namespace"].(string) + if istioNS != "" && istioNS != operandNamespace { + return fmt.Sprintf( + "spec.istioCSRConfig.istio.namespace is %q but cert-manager Issuer istio-ca is created in %q; they must match", + istioNS, + operandNamespace, + ) + } + } + } + + list, err := loader.DynamicClient.Resource(istiocsrSchema).List(ctx, metav1.ListOptions{}) + if err != nil { + return "" + } + for _, item := range list.Items { + if item.GetNamespace() == operandNamespace { + continue + } + return fmt.Sprintf( + "another IstioCSR already exists at %s/%s; only one cluster-wide operand is supported", + item.GetNamespace(), + item.GetName(), + ) + } + return "" +} + +// cleanupAllIstioCSROperands removes every IstioCSR CR so grpc tests can create a fresh operand. +func cleanupAllIstioCSROperands(ctx context.Context, loader library.DynamicResourceLoader) error { + client := loader.DynamicClient.Resource(istiocsrSchema) + list, err := client.List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + for _, item := range list.Items { + ns := item.GetNamespace() + name := item.GetName() + if err := client.Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete IstioCSR %s/%s: %w", ns, name, err) + } + } + return wait.PollUntilContextTimeout(ctx, slowPollInterval, lowTimeout, true, func(context.Context) (bool, error) { + list, err := client.List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + if len(list.Items) == 0 { + return true, nil + } + return false, nil + }) +} + // pollTillDeploymentAvailable poll the deployment object and returns non-nil error // once the deployment is available, otherwise should return a time-out error func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clientset, namespace, deploymentName string) error { @@ -1217,6 +1370,9 @@ func pollTillDeploymentAvailable(ctx context.Context, clientSet *kubernetes.Clie deployment, getErr := clientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) if getErr != nil { if apierrors.IsNotFound(getErr) { + if hint := istioCSRDeploymentMissingHint(ctx, namespace); hint != "" { + return fmt.Errorf("timeout waiting for deployment %s/%s: deployment does not exist (%s)", namespace, deploymentName, hint) + } return fmt.Errorf("timeout waiting for deployment %s/%s: deployment does not exist", namespace, deploymentName) } return fmt.Errorf("timeout waiting for deployment %s/%s: failed to get status: %v", namespace, deploymentName, getErr) @@ -1427,6 +1583,8 @@ func resetCertManagerNetworkPolicyState(ctx context.Context, client *certmanoper return nil } +const vaultTokenFile = "/tmp/e2e-vault-token" + // execInPod executes a command in a specific container of a pod and returns the output. func execInPod(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Interface, namespace, podName, containerName string, command ...string) (string, error) { req := kubeClient.CoreV1().RESTClient(). @@ -1459,6 +1617,60 @@ func execInPod(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Inte return stdout.String(), nil } +// execInPodWithStdin executes a command in a pod with optional stdin data. +func execInPodWithStdin(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Interface, namespace, podName, containerName string, stdin []byte, command ...string) (string, error) { + req := kubeClient.CoreV1().RESTClient(). + Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: command, + Stdout: true, + Stderr: true, + Stdin: len(stdin) > 0, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return "", fmt.Errorf("failed to create executor: %w", err) + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + Stdin: bytes.NewReader(stdin), + }) + if err != nil { + return "", fmt.Errorf("failed to execute command: %w, stderr: %s", err, stderr.String()) + } + + return stdout.String(), nil +} + +// setVaultTokenInPod writes the Vault root token to a file in the pod via stdin so the +// token is never embedded in shell command strings that may appear in error logs. +func setVaultTokenInPod(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Interface, namespace, podName, rootToken string) error { + _, err := execInPodWithStdin(ctx, cfg, kubeClient, namespace, podName, "vault", []byte(rootToken), + "sh", "-c", fmt.Sprintf("cat > %s && chmod 600 %s", vaultTokenFile, vaultTokenFile)) + return err +} + +// vaultShellCmd prefixes a shell script with VAULT_TOKEN loaded from the pod-local token file. +func vaultShellCmd(script string) string { + return fmt.Sprintf("export VAULT_TOKEN=$(cat %s) && %s", vaultTokenFile, script) +} + +// execVaultInPod runs vault with the given arguments, loading VAULT_TOKEN from the pod-local token file. +// Arguments are passed to vault via exec rather than interpolated into a shell string. +func execVaultInPod(ctx context.Context, cfg *rest.Config, kubeClient kubernetes.Interface, namespace, podName string, vaultArgs ...string) (string, error) { + cmd := append([]string{"sh", "-c", fmt.Sprintf("export VAULT_TOKEN=$(cat %s) && exec vault \"$@\"", vaultTokenFile), "vault"}, vaultArgs...) + return execInPod(ctx, cfg, kubeClient, namespace, podName, "vault", cmd...) +} + // isPodReady returns true if the pod is running and has the Ready condition set to true. // It returns false for pods that are being deleted. func isPodReady(pod *corev1.Pod) bool { @@ -1682,13 +1894,12 @@ func createCertificateForVaultServer(ctx context.Context, certmanagerClient *cer }, }, } - _, err := certmanagerClient.CertmanagerV1().ClusterIssuers().Create(ctx, clusterIssuer, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { + if err := ensureClusterIssuer(ctx, certmanagerClient, clusterIssuer); err != nil { return fmt.Errorf("failed to create ClusterIssuer: %w", err) } // Wait for ClusterIssuer to become ready - err = wait.PollUntilContextTimeout(ctx, fastPollInterval, lowTimeout, true, + err := wait.PollUntilContextTimeout(ctx, fastPollInterval, lowTimeout, true, func(context.Context) (bool, error) { issuer, err := certmanagerClient.CertmanagerV1().ClusterIssuers().Get(ctx, clusterIssuerName, metav1.GetOptions{}) if err != nil { @@ -1730,8 +1941,7 @@ func createCertificateForVaultServer(ctx context.Context, certmanagerClient *cer }, }, } - _, err = certmanagerClient.CertmanagerV1().Certificates(namespace).Create(ctx, cert, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { + if err := ensureCertificate(ctx, certmanagerClient, cert); err != nil { return fmt.Errorf("failed to create Certificate: %w", err) } @@ -1761,8 +1971,9 @@ func createCertificateForVaultServer(ctx context.Context, certmanagerClient *cer func configureVaultPKI(ctx context.Context, cfg *rest.Config, loader library.DynamicResourceLoader, namespace, vaultPodName, rootToken string) error { kubeClient := loader.KubeClient - // Set VAULT_TOKEN environment variable for subsequent commands - tokenEnv := fmt.Sprintf("export VAULT_TOKEN=%s", rootToken) + if err := setVaultTokenInPod(ctx, cfg, kubeClient, namespace, vaultPodName, rootToken); err != nil { + return fmt.Errorf("failed to set vault token in pod: %w", err) + } // Enable and configure root PKI engine commands := []struct { @@ -1771,32 +1982,32 @@ func configureVaultPKI(ctx context.Context, cfg *rest.Config, loader library.Dyn }{ { "enable PKI secrets engine", - tokenEnv + " && vault secrets enable pki", + "vault secrets enable pki", }, { "tune PKI max lease TTL", - tokenEnv + " && vault secrets tune -max-lease-ttl=8760h pki", + "vault secrets tune -max-lease-ttl=8760h pki", }, { "generate root CA", - tokenEnv + " && vault write pki/root/generate/internal common_name=cluster.local ttl=8760h", + "vault write pki/root/generate/internal common_name=cluster.local ttl=8760h", }, { "configure CA and CRL URLs", - tokenEnv + " && vault write pki/config/urls issuing_certificates=\"https://vault:8200/v1/pki/ca\" crl_distribution_points=\"https://vault:8200/v1/pki/crl\"", + "vault write pki/config/urls issuing_certificates=\"https://vault:8200/v1/pki/ca\" crl_distribution_points=\"https://vault:8200/v1/pki/crl\"", }, { "enable intermediate PKI", - tokenEnv + " && vault secrets enable -path=pki_int pki", + "vault secrets enable -path=pki_int pki", }, { "tune intermediate PKI max lease TTL", - tokenEnv + " && vault secrets tune -max-lease-ttl=4380h pki_int", + "vault secrets tune -max-lease-ttl=4380h pki_int", }, } for _, cmdInfo := range commands { - _, err := execInPod(ctx, cfg, kubeClient, namespace, vaultPodName, "vault", "sh", "-c", cmdInfo.cmd) + _, err := execInPod(ctx, cfg, kubeClient, namespace, vaultPodName, "vault", "sh", "-c", vaultShellCmd(cmdInfo.cmd)) if err != nil { return fmt.Errorf("failed to %s: %w", cmdInfo.description, err) } @@ -1804,7 +2015,7 @@ func configureVaultPKI(ctx context.Context, cfg *rest.Config, loader library.Dyn // Generate intermediate CSR csrOutput, err := execInPod(ctx, cfg, kubeClient, namespace, vaultPodName, "vault", "sh", "-c", - tokenEnv+` && vault write -format=json pki_int/intermediate/generate/internal common_name="cluster.local Intermediate Authority" ttl=4380h`) + vaultShellCmd(`vault write -format=json pki_int/intermediate/generate/internal common_name="cluster.local Intermediate Authority" ttl=4380h`)) if err != nil { return fmt.Errorf("failed to generate intermediate CSR: %w", err) } @@ -1814,9 +2025,9 @@ func configureVaultPKI(ctx context.Context, cfg *rest.Config, loader library.Dyn } // Sign intermediate with root CA - use heredoc to properly handle multi-line CSR - signCmd := tokenEnv + ` && vault write -format=json pki/root/sign-intermediate format=pem_bundle ttl=4380h csr=- < 0 { + secret.Data = make(map[string][]byte, len(secret.StringData)) + for k, v := range secret.StringData { + secret.Data[k] = []byte(v) + } + secret.StringData = nil + } +} + +// UpsertSecret creates the secret or updates it when it already exists so data matches desired. +func UpsertSecret(ctx context.Context, client kubernetes.Interface, secret *corev1.Secret) error { + populateSecretData(secret) + + _, err := client.CoreV1().Secrets(secret.Namespace).Create(ctx, secret.DeepCopy(), metav1.CreateOptions{}) + if err == nil { + return nil + } + if !k8serrors.IsAlreadyExists(err) { + return err + } + + existing, getErr := client.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + + updated := secret.DeepCopy() + updated.ResourceVersion = existing.ResourceVersion + populateSecretData(updated) + _, err = client.CoreV1().Secrets(secret.Namespace).Update(ctx, updated, metav1.UpdateOptions{}) + return err +} + +// UpsertConfigMap creates the ConfigMap or updates it when it already exists so data matches desired. +func UpsertConfigMap(ctx context.Context, client kubernetes.Interface, configMap *corev1.ConfigMap) error { + _, err := client.CoreV1().ConfigMaps(configMap.Namespace).Create(ctx, configMap.DeepCopy(), metav1.CreateOptions{}) + if err == nil { + return nil + } + if !k8serrors.IsAlreadyExists(err) { + return err + } + + existing, getErr := client.CoreV1().ConfigMaps(configMap.Namespace).Get(ctx, configMap.Name, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + + updated := configMap.DeepCopy() + updated.ResourceVersion = existing.ResourceVersion + _, err = client.CoreV1().ConfigMaps(configMap.Namespace).Update(ctx, updated, metav1.UpdateOptions{}) + return err +}