diff --git a/Makefile b/Makefile index ac17e864b9..eb8e66b8c2 100644 --- a/Makefile +++ b/Makefile @@ -130,22 +130,26 @@ kind-cni-calico: kubectl create -f $(CURDIR)/acceptance/framework/environment/cni-kind/custom-resources.yaml @sleep 20 -# Helper target for doing local cni acceptance testing -kind-cni: +kind-delete: kind delete cluster --name dc1 kind delete cluster --name dc2 + kind delete cluster --name dc3 + kind delete cluster --name dc4 + + +# Helper target for doing local cni acceptance testing +kind-cni: kind-delete kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc1 --image $(KIND_NODE_IMAGE) make kind-cni-calico kind create cluster --config=$(CURDIR)/acceptance/framework/environment/cni-kind/kind.config --name dc2 --image $(KIND_NODE_IMAGE) make kind-cni-calico # Helper target for doing local acceptance testing -kind: - kind delete cluster --name dc1 - kind delete cluster --name dc2 +kind: kind-delete kind create cluster --name dc1 --image $(KIND_NODE_IMAGE) kind create cluster --name dc2 --image $(KIND_NODE_IMAGE) - + kind create cluster --name dc3 --image $(KIND_NODE_IMAGE) + kind create cluster --name dc4 --image $(KIND_NODE_IMAGE) # Helper target for loading local dev images (run with `DEV_IMAGE=...` to load non-k8s images) kind-load: diff --git a/acceptance/ci-inputs/kind_acceptance_test_packages.yaml b/acceptance/ci-inputs/kind_acceptance_test_packages.yaml index 8677b83c4e..e0e126bbda 100644 --- a/acceptance/ci-inputs/kind_acceptance_test_packages.yaml +++ b/acceptance/ci-inputs/kind_acceptance_test_packages.yaml @@ -3,7 +3,8 @@ - {runner: 0, test-packages: "partitions"} - {runner: 1, test-packages: "peering"} -- {runner: 2, test-packages: "connect snapshot-agent wan-federation"} -- {runner: 3, test-packages: "cli vault metrics"} -- {runner: 4, test-packages: "api-gateway ingress-gateway sync example consul-dns"} -- {runner: 5, test-packages: "config-entries terminating-gateway basic"} \ No newline at end of file +- {runner: 2, test-packages: "sameness"} +- {runner: 3, test-packages: "connect snapshot-agent wan-federation"} +- {runner: 4, test-packages: "cli vault metrics"} +- {runner: 5, test-packages: "api-gateway ingress-gateway sync example consul-dns"} +- {runner: 6, test-packages: "config-entries terminating-gateway basic"} diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index f52b49ea92..a638732c37 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -96,6 +96,7 @@ type TestConfig struct { VaultServerVersion string NoCleanupOnFailure bool + NoCleanup bool DebugDirectory string UseAKS bool diff --git a/acceptance/framework/connhelper/connect_helper.go b/acceptance/framework/connhelper/connect_helper.go index 9a0d641b26..00413f7686 100644 --- a/acceptance/framework/connhelper/connect_helper.go +++ b/acceptance/framework/connhelper/connect_helper.go @@ -133,22 +133,22 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { // TODO: A base fixture is the wrong place for these files k8s.KubectlApply(t, opts, "../fixtures/bases/openshift/") - helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, func() { k8s.KubectlDelete(t, opts, "../fixtures/bases/openshift/") }) - k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift") + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-openshift") if c.Cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy") + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-tproxy") } else { - k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject") + k8s.DeployKustomize(t, opts, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-openshift-inject") } } else { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if c.Cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, c.Ctx.KubectlOptions(t), c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, c.Cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } } // Check that both static-server and static-client have been injected and @@ -185,7 +185,7 @@ func (c *ConnectHelper) SetupAppNamespace(t *testing.T) { return } require.NoError(t, err) - helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, func() { k8s.RunKubectl(t, opts, "delete", "ns", opts.Namespace) }) @@ -208,7 +208,7 @@ func (c *ConnectHelper) CreateResolverRedirect(t *testing.T) { kustomizeDir := "../fixtures/cases/resolver-redirect-virtualip" k8s.KubectlApplyK(t, opts, kustomizeDir) - helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, c.Cfg.NoCleanupOnFailure, c.Cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, opts, kustomizeDir) }) } diff --git a/acceptance/framework/consul/cli_cluster.go b/acceptance/framework/consul/cli_cluster.go index 9e119af76d..a45bcc665c 100644 --- a/acceptance/framework/consul/cli_cluster.go +++ b/acceptance/framework/consul/cli_cluster.go @@ -45,6 +45,7 @@ type CLICluster struct { kubeConfig string kubeContext string noCleanupOnFailure bool + noCleanup bool debugDirectory string logger terratestLogger.TestLogger cli cli.CLI @@ -109,6 +110,7 @@ func NewCLICluster( kubeConfig: cfg.GetPrimaryKubeEnv().KubeConfig, kubeContext: cfg.GetPrimaryKubeEnv().KubeContext, noCleanupOnFailure: cfg.NoCleanupOnFailure, + noCleanup: cfg.NoCleanup, debugDirectory: cfg.DebugDirectory, logger: logger, cli: *cli, @@ -122,7 +124,7 @@ func (c *CLICluster) Create(t *testing.T) { // Make sure we delete the cluster if we receive an interrupt signal and // register cleanup so that we delete the cluster when test finishes. - helpers.Cleanup(t, c.noCleanupOnFailure, func() { + helpers.Cleanup(t, c.noCleanupOnFailure, c.noCleanup, func() { c.Destroy(t) }) diff --git a/acceptance/framework/consul/helm_cluster.go b/acceptance/framework/consul/helm_cluster.go index aa6fdef7d8..81787ebfc3 100644 --- a/acceptance/framework/consul/helm_cluster.go +++ b/acceptance/framework/consul/helm_cluster.go @@ -52,6 +52,7 @@ type HelmCluster struct { runtimeClient client.Client kubernetesClient kubernetes.Interface noCleanupOnFailure bool + noCleanup bool debugDirectory string logger terratestLogger.TestLogger } @@ -107,6 +108,7 @@ func NewHelmCluster( runtimeClient: ctx.ControllerRuntimeClient(t), kubernetesClient: ctx.KubernetesClient(t), noCleanupOnFailure: cfg.NoCleanupOnFailure, + noCleanup: cfg.NoCleanup, debugDirectory: cfg.DebugDirectory, logger: logger, } @@ -117,7 +119,7 @@ func (h *HelmCluster) Create(t *testing.T) { // Make sure we delete the cluster if we receive an interrupt signal and // register cleanup so that we delete the cluster when test finishes. - helpers.Cleanup(t, h.noCleanupOnFailure, func() { + helpers.Cleanup(t, h.noCleanupOnFailure, h.noCleanup, func() { h.Destroy(t) }) @@ -508,7 +510,7 @@ func configurePodSecurityPolicies(t *testing.T, client kubernetes.Interface, cfg } } - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _ = client.PolicyV1beta1().PodSecurityPolicies().Delete(context.Background(), pspName, metav1.DeleteOptions{}) _ = client.RbacV1().ClusterRoles().Delete(context.Background(), pspName, metav1.DeleteOptions{}) _ = client.RbacV1().RoleBindings(namespace).Delete(context.Background(), pspName, metav1.DeleteOptions{}) @@ -559,7 +561,7 @@ func configureSCCs(t *testing.T, client kubernetes.Interface, cfg *config.TestCo } } - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _ = client.RbacV1().RoleBindings(namespace).Delete(context.Background(), anyuidRoleBinding, metav1.DeleteOptions{}) _ = client.RbacV1().RoleBindings(namespace).Delete(context.Background(), privilegedRoleBinding, metav1.DeleteOptions{}) }) @@ -601,7 +603,7 @@ func CreateK8sSecret(t *testing.T, client kubernetes.Interface, cfg *config.Test } }) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _ = client.CoreV1().Secrets(namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{}) }) } diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index de413bbba5..9f1e3e15aa 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -46,6 +46,7 @@ type TestFlags struct { flagHCPResourceID string flagNoCleanupOnFailure bool + flagNoCleanup bool flagDebugDirectory string @@ -132,6 +133,9 @@ func (t *TestFlags) init() { "If true, the tests will not cleanup Kubernetes resources they create when they finish running."+ "Note this flag must be run with -failfast flag, otherwise subsequent tests will fail.") + flag.BoolVar(&t.flagNoCleanup, "no-cleanup", false, + "If true, the tests will not cleanup Kubernetes resources for Vault test") + flag.StringVar(&t.flagDebugDirectory, "debug-directory", "", "The directory where to write debug information about failed test runs, "+ "such as logs and pod definitions. If not provided, a temporary directory will be created by the tests.") @@ -224,6 +228,7 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { HCPResourceID: t.flagHCPResourceID, NoCleanupOnFailure: t.flagNoCleanupOnFailure, + NoCleanup: t.flagNoCleanup, DebugDirectory: tempDir, UseAKS: t.flagUseAKS, UseEKS: t.flagUseEKS, diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go index 3e6ef039b2..f8b1d2f15d 100644 --- a/acceptance/framework/helpers/helpers.go +++ b/acceptance/framework/helpers/helpers.go @@ -87,7 +87,7 @@ func SetupInterruptHandler(cleanup func()) { // Cleanup will both register a cleanup function with t // and SetupInterruptHandler to make sure resources get cleaned up // if an interrupt signal is caught. -func Cleanup(t *testing.T, noCleanupOnFailure bool, cleanup func()) { +func Cleanup(t *testing.T, noCleanupOnFailure bool, noCleanup bool, cleanup func()) { t.Helper() // Always clean up when an interrupt signal is caught. @@ -97,7 +97,7 @@ func Cleanup(t *testing.T, noCleanupOnFailure bool, cleanup func()) { // We need to wrap the cleanup function because t that is passed in to this function // might not have the information on whether the test has failed yet. wrappedCleanupFunc := func() { - if !(noCleanupOnFailure && t.Failed()) { + if !((noCleanupOnFailure && t.Failed()) || noCleanup) { logger.Logf(t, "cleaning up resources for %s", t.Name()) cleanup() } else { diff --git a/acceptance/framework/k8s/deploy.go b/acceptance/framework/k8s/deploy.go index 6834284c33..828586c8cf 100644 --- a/acceptance/framework/k8s/deploy.go +++ b/acceptance/framework/k8s/deploy.go @@ -21,7 +21,7 @@ import ( // Deploy creates a Kubernetes deployment by applying configuration stored at filepath, // sets up a cleanup function and waits for the deployment to become available. -func Deploy(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, debugDirectory string, filepath string) { +func Deploy(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, noCleanup bool, debugDirectory string, filepath string) { t.Helper() KubectlApply(t, options, filepath) @@ -33,7 +33,7 @@ func Deploy(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, err = yaml.NewYAMLOrJSONDecoder(file, 1024).Decode(&deployment) require.NoError(t, err) - helpers.Cleanup(t, noCleanupOnFailure, func() { + helpers.Cleanup(t, noCleanupOnFailure, noCleanup, func() { // Note: this delete command won't wait for pods to be fully terminated. // This shouldn't cause any test pollution because the underlying // objects are deployments, and so when other tests create these @@ -47,7 +47,7 @@ func Deploy(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, // DeployKustomize creates a Kubernetes deployment by applying the kustomize directory stored at kustomizeDir, // sets up a cleanup function and waits for the deployment to become available. -func DeployKustomize(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, debugDirectory string, kustomizeDir string) { +func DeployKustomize(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailure bool, noCleanup bool, debugDirectory string, kustomizeDir string) { t.Helper() KubectlApplyK(t, options, kustomizeDir) @@ -59,7 +59,7 @@ func DeployKustomize(t *testing.T, options *k8s.KubectlOptions, noCleanupOnFailu err = yaml.NewYAMLOrJSONDecoder(strings.NewReader(output), 1024).Decode(&deployment) require.NoError(t, err) - helpers.Cleanup(t, noCleanupOnFailure, func() { + helpers.Cleanup(t, noCleanupOnFailure, noCleanup, func() { // Note: this delete command won't wait for pods to be fully terminated. // This shouldn't cause any test pollution because the underlying // objects are deployments, and so when other tests create these diff --git a/acceptance/framework/k8s/kubectl.go b/acceptance/framework/k8s/kubectl.go index ea90212e04..acfedbdb3d 100644 --- a/acceptance/framework/k8s/kubectl.go +++ b/acceptance/framework/k8s/kubectl.go @@ -4,6 +4,7 @@ package k8s import ( + "fmt" "strings" "testing" "time" @@ -16,6 +17,10 @@ import ( "github.com/stretchr/testify/require" ) +const ( + kubectlTimeout = "--timeout=120s" +) + // kubeAPIConnectErrs are errors that sometimes occur when talking to the // Kubernetes API related to connection issues. var kubeAPIConnectErrs = []string{ @@ -97,7 +102,7 @@ func KubectlApplyK(t *testing.T, options *k8s.KubectlOptions, kustomizeDir strin // deletes it from the cluster by running 'kubectl delete -f'. // If there's an error deleting the file, fail the test. func KubectlDelete(t *testing.T, options *k8s.KubectlOptions, configPath string) { - _, err := RunKubectlAndGetOutputE(t, options, "delete", "--timeout=60s", "-f", configPath) + _, err := RunKubectlAndGetOutputE(t, options, "delete", kubectlTimeout, "-f", configPath) require.NoError(t, err) } @@ -107,7 +112,13 @@ func KubectlDelete(t *testing.T, options *k8s.KubectlOptions, configPath string) func KubectlDeleteK(t *testing.T, options *k8s.KubectlOptions, kustomizeDir string) { // Ignore not found errors because Kubernetes automatically cleans up the kube secrets that we deployed // referencing the ServiceAccount when it is deleted. - _, err := RunKubectlAndGetOutputE(t, options, "delete", "--timeout=60s", "--ignore-not-found", "-k", kustomizeDir) + _, err := RunKubectlAndGetOutputE(t, options, "delete", kubectlTimeout, "--ignore-not-found", "-k", kustomizeDir) + require.NoError(t, err) +} + +// KubectlScale takes a deployment and scales it to the provided number of replicas. +func KubectlScale(t *testing.T, options *k8s.KubectlOptions, deployment string, replicas int) { + _, err := RunKubectlAndGetOutputE(t, options, "scale", kubectlTimeout, fmt.Sprintf("--replicas=%d", replicas), deployment) require.NoError(t, err) } diff --git a/acceptance/framework/vault/vault_cluster.go b/acceptance/framework/vault/vault_cluster.go index 8dadd2f737..68e41eb238 100644 --- a/acceptance/framework/vault/vault_cluster.go +++ b/acceptance/framework/vault/vault_cluster.go @@ -44,6 +44,7 @@ type VaultCluster struct { kubernetesClient kubernetes.Interface noCleanupOnFailure bool + noCleanup bool debugDirectory string logger terratestLogger.TestLogger } @@ -89,6 +90,7 @@ func NewVaultCluster(t *testing.T, ctx environment.TestContext, cfg *config.Test kubectlOptions: kopts, kubernetesClient: ctx.KubernetesClient(t), noCleanupOnFailure: cfg.NoCleanupOnFailure, + noCleanup: cfg.NoCleanup, debugDirectory: cfg.DebugDirectory, logger: logger, releaseName: releaseName, @@ -224,7 +226,7 @@ func (v *VaultCluster) Create(t *testing.T, ctx environment.TestContext, vaultNa // Make sure we delete the cluster if we receive an interrupt signal and // register cleanup so that we delete the cluster when test finishes. - helpers.Cleanup(t, v.noCleanupOnFailure, func() { + helpers.Cleanup(t, v.noCleanupOnFailure, v.noCleanup, func() { v.Destroy(t) }) @@ -346,7 +348,7 @@ func (v *VaultCluster) createTLSCerts(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { - if !v.noCleanupOnFailure { + if !(v.noCleanupOnFailure || v.noCleanup) { // We're ignoring error here because secret deletion is best-effort. _ = v.kubernetesClient.CoreV1().Secrets(namespace).Delete(context.Background(), certSecretName(v.releaseName), metav1.DeleteOptions{}) _ = v.kubernetesClient.CoreV1().Secrets(namespace).Delete(context.Background(), CASecretName(v.releaseName), metav1.DeleteOptions{}) @@ -419,7 +421,7 @@ func (v *VaultCluster) initAndUnseal(t *testing.T) { rootTokenSecret := fmt.Sprintf("%s-vault-root-token", v.releaseName) v.logger.Logf(t, "saving Vault root token to %q Kubernetes secret", rootTokenSecret) - helpers.Cleanup(t, v.noCleanupOnFailure, func() { + helpers.Cleanup(t, v.noCleanupOnFailure, v.noCleanup, func() { _ = v.kubernetesClient.CoreV1().Secrets(namespace).Delete(context.Background(), rootTokenSecret, metav1.DeleteOptions{}) }) _, err := v.kubernetesClient.CoreV1().Secrets(namespace).Create(context.Background(), &corev1.Secret{ diff --git a/acceptance/tests/api-gateway/api_gateway_external_servers_test.go b/acceptance/tests/api-gateway/api_gateway_external_servers_test.go index 92d98de558..aa0934dc65 100644 --- a/acceptance/tests/api-gateway/api_gateway_external_servers_test.go +++ b/acceptance/tests/api-gateway/api_gateway_external_servers_test.go @@ -60,8 +60,8 @@ func TestAPIGateway_ExternalServers(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") // Override the default proxy config settings for this test consulClient, _ := consulCluster.SetupConsulClient(t, true, serverReleaseName) @@ -79,7 +79,7 @@ func TestAPIGateway_ExternalServers(t *testing.T) { logger.Log(t, "creating api-gateway resources") out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") diff --git a/acceptance/tests/api-gateway/api_gateway_gatewayclassconfig_test.go b/acceptance/tests/api-gateway/api_gateway_gatewayclassconfig_test.go index 444af6af4d..22faf55529 100644 --- a/acceptance/tests/api-gateway/api_gateway_gatewayclassconfig_test.go +++ b/acceptance/tests/api-gateway/api_gateway_gatewayclassconfig_test.go @@ -80,7 +80,7 @@ func TestAPIGateway_GatewayClassConfig(t *testing.T) { logger.Log(t, "creating gateway class config") err = k8sClient.Create(context.Background(), gatewayClassConfig) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all gateway class configs") k8sClient.DeleteAllOf(context.Background(), &v1alpha1.GatewayClassConfig{}) }) @@ -94,7 +94,7 @@ func TestAPIGateway_GatewayClassConfig(t *testing.T) { // Create gateway class referencing gateway-class-config. logger.Log(t, "creating controlled gateway class") createGatewayClass(t, k8sClient, gatewayClassName, gatewayClassControllerName, gatewayParametersRef) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all gateway classes") k8sClient.DeleteAllOf(context.Background(), &gwv1beta1.GatewayClass{}) }) @@ -119,7 +119,7 @@ func TestAPIGateway_GatewayClassConfig(t *testing.T) { logger.Log(t, "creating certificate") err = k8sClient.Create(context.Background(), certificate) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8sClient.Delete(context.Background(), certificate) }) @@ -127,7 +127,7 @@ func TestAPIGateway_GatewayClassConfig(t *testing.T) { gatewayName := "gcctestgateway" + namespace logger.Log(t, "creating controlled gateway") gateway := createGateway(t, k8sClient, gatewayName, namespace, gatewayClassName, certificateName) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all gateways") k8sClient.DeleteAllOf(context.Background(), &gwv1beta1.Gateway{}, client.InNamespace(namespace)) }) diff --git a/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go b/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go index 9443857522..f6f66ed995 100644 --- a/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go +++ b/acceptance/tests/api-gateway/api_gateway_lifecycle_test.go @@ -44,7 +44,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { // create a service to target targetName := "static-server" logger.Log(t, "creating target server") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // create a basic GatewayClassConfig gatewayClassConfigName := "controlled-gateway-class-config" @@ -56,7 +56,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { logger.Log(t, "creating gateway class config") err := k8sClient.Create(context.Background(), gatewayClassConfig) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all gateway class configs") k8sClient.DeleteAllOf(context.Background(), &v1alpha1.GatewayClassConfig{}) }) @@ -72,7 +72,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { logger.Log(t, "creating controlled gateway class one") createGatewayClass(t, k8sClient, controlledGatewayClassOneName, gatewayClassControllerName, gatewayParametersRef) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all gateway classes") k8sClient.DeleteAllOf(context.Background(), &gwv1beta1.GatewayClass{}) }) @@ -105,7 +105,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { logger.Log(t, "creating certificate") err = k8sClient.Create(context.Background(), certificate) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8sClient.Delete(context.Background(), certificate) }) @@ -114,7 +114,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { logger.Log(t, "creating controlled gateway one") controlledGatewayOne := createGateway(t, k8sClient, controlledGatewayOneName, defaultNamespace, controlledGatewayClassOneName, certificateName) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all gateways") k8sClient.DeleteAllOf(context.Background(), &gwv1beta1.Gateway{}, client.InNamespace(defaultNamespace)) }) @@ -132,7 +132,7 @@ func TestAPIGateway_Lifecycle(t *testing.T) { logger.Log(t, "creating route one") routeOne := createRoute(t, k8sClient, routeOneName, defaultNamespace, controlledGatewayOneName, targetName) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { logger.Log(t, "deleting all http routes") k8sClient.DeleteAllOf(context.Background(), &gwv1beta1.HTTPRoute{}, client.InNamespace(defaultNamespace)) }) diff --git a/acceptance/tests/api-gateway/api_gateway_tenancy_test.go b/acceptance/tests/api-gateway/api_gateway_tenancy_test.go index 08a1df4485..f7b0ac6d79 100644 --- a/acceptance/tests/api-gateway/api_gateway_tenancy_test.go +++ b/acceptance/tests/api-gateway/api_gateway_tenancy_test.go @@ -100,7 +100,7 @@ func TestAPIGateway_Tenancy(t *testing.T) { routeNamespace, routeK8SOptions := createNamespace(t, ctx, cfg) logger.Logf(t, "creating target server in %s namespace", serviceNamespace) - k8s.DeployKustomize(t, serviceK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, serviceK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Logf(t, "creating certificate resources in %s namespace", certificateNamespace) applyFixture(t, cfg, certificateK8SOptions, "cases/api-gateways/certificate") @@ -255,7 +255,7 @@ func applyFixture(t *testing.T, cfg *config.TestConfig, k8sOptions *terratestk8s out, err := k8s.RunKubectlAndGetOutputE(t, k8sOptions, "apply", "-k", path.Join("../fixtures", fixture)) require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectlAndGetOutputE(t, k8sOptions, "delete", "-k", path.Join("../fixtures", fixture)) }) } @@ -267,7 +267,7 @@ func createNamespace(t *testing.T, ctx environment.TestContext, cfg *config.Test logger.Logf(t, "creating Kubernetes namespace %s", namespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", namespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", namespace) }) diff --git a/acceptance/tests/api-gateway/api_gateway_test.go b/acceptance/tests/api-gateway/api_gateway_test.go index 5b9c682d1f..721bbf2527 100644 --- a/acceptance/tests/api-gateway/api_gateway_test.go +++ b/acceptance/tests/api-gateway/api_gateway_test.go @@ -7,11 +7,12 @@ import ( "context" "encoding/base64" "fmt" - gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "strconv" "testing" "time" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" @@ -75,7 +76,7 @@ func TestAPIGateway_Basic(t *testing.T) { logger.Log(t, "creating api-gateway resources") out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") @@ -86,7 +87,7 @@ func TestAPIGateway_Basic(t *testing.T) { logger.Log(t, "creating certificate secret") out, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-f", "../fixtures/bases/api-gateway/certificate.yaml") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-f", "../fixtures/bases/api-gateway/certificate.yaml") @@ -98,22 +99,22 @@ func TestAPIGateway_Basic(t *testing.T) { k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "secret", "certificate", "-p", fmt.Sprintf(`{"data":{"tls.crt":"%s","tls.key":"%s"}}`, base64.StdEncoding.EncodeToString(certificate.CertPEM), base64.StdEncoding.EncodeToString(certificate.PrivateKeyPEM)), "--type=merge") logger.Log(t, "creating target http server") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // We use the static-client pod so that we can make calls to the api gateway // via kubectl exec without needing a route into the cluster from the test machine. logger.Log(t, "creating static-client pod") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") logger.Log(t, "patching route to target http server") k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"name":"static-server","port":80}]}]}}`, "--type=merge") logger.Log(t, "creating target tcp server") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server-tcp") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server-tcp") logger.Log(t, "creating tcp-route") k8s.RunKubectl(t, ctx.KubectlOptions(t), "apply", "-f", "../fixtures/cases/api-gateways/tcproute/route.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-f", "../fixtures/cases/api-gateways/tcproute/route.yaml") diff --git a/acceptance/tests/cloud/basic_test.go b/acceptance/tests/cloud/basic_test.go index 8278309ff3..7858450904 100644 --- a/acceptance/tests/cloud/basic_test.go +++ b/acceptance/tests/cloud/basic_test.go @@ -113,7 +113,7 @@ func TestBasicCloud(t *testing.T) { consul.CreateK8sSecret(t, k8sClient, cfg, ns, authUrlSecretName, authUrlSecretKey, authUrlSecretKeyValue) consul.CreateK8sSecret(t, k8sClient, cfg, ns, scadaAddressSecretName, scadaAddressSecretKey, scadaAddressSecretKeyValue) - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/cloud/hcp-mock") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/cloud/hcp-mock") podName, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "get", "pod", "-l", "app=fake-server", "-o", `jsonpath="{.items[0].metadata.name}"`) podName = strings.ReplaceAll(podName, "\"", "") if err != nil { @@ -227,7 +227,7 @@ func TestBasicCloud(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating static-server deployment") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") // time.Sleep(1 * time.Hour) // TODO: add in test assertions here } diff --git a/acceptance/tests/config-entries/config_entries_namespaces_test.go b/acceptance/tests/config-entries/config_entries_namespaces_test.go index 91d0c69df4..a013c99e09 100644 --- a/acceptance/tests/config-entries/config_entries_namespaces_test.go +++ b/acceptance/tests/config-entries/config_entries_namespaces_test.go @@ -102,7 +102,7 @@ func TestControllerNamespaces(t *testing.T) { if err != nil && !strings.Contains(out, "(AlreadyExists)") { require.NoError(t, err) } - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", KubeNS) }) diff --git a/acceptance/tests/config-entries/config_entries_test.go b/acceptance/tests/config-entries/config_entries_test.go index e37e3d6c7f..a36e9baaf5 100644 --- a/acceptance/tests/config-entries/config_entries_test.go +++ b/acceptance/tests/config-entries/config_entries_test.go @@ -86,7 +86,7 @@ func TestController(t *testing.T) { // endpoint fails initially. out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/crds-oss") require.NoError(r, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-k", "../fixtures/bases/crds-oss") diff --git a/acceptance/tests/connect/connect_external_servers_test.go b/acceptance/tests/connect/connect_external_servers_test.go index c95d791773..46f9e14573 100644 --- a/acceptance/tests/connect/connect_external_servers_test.go +++ b/acceptance/tests/connect/connect_external_servers_test.go @@ -75,11 +75,11 @@ func TestConnectInject_ExternalServers(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } // Check that both static-server and static-client have been injected and now have 2 containers. diff --git a/acceptance/tests/connect/connect_inject_namespaces_test.go b/acceptance/tests/connect/connect_inject_namespaces_test.go index 04021ec391..7b7785a44f 100644 --- a/acceptance/tests/connect/connect_inject_namespaces_test.go +++ b/acceptance/tests/connect/connect_inject_namespaces_test.go @@ -101,12 +101,12 @@ func TestConnectInjectNamespaces(t *testing.T) { logger.Logf(t, "creating namespaces %s and %s", staticServerNamespace, StaticClientNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Note: this deletion will take longer in cases when the static-client deployment // hasn't yet fully terminated. k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", StaticClientNamespace) @@ -147,11 +147,11 @@ func TestConnectInjectNamespaces(t *testing.T) { } logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") } // Check that both static-server and static-client have been injected and now have 2 containers. @@ -305,7 +305,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) { logger.Logf(t, "creating namespace %s", StaticClientNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", StaticClientNamespace) }) @@ -315,7 +315,7 @@ func TestConnectInjectNamespaces_CleanupController(t *testing.T) { ConfigPath: ctx.KubectlOptions(t).ConfigPath, Namespace: StaticClientNamespace, } - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") logger.Log(t, "waiting for static-client to be registered with Consul") consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) diff --git a/acceptance/tests/connect/connect_inject_test.go b/acceptance/tests/connect/connect_inject_test.go index 0704b16e54..7c7f17fc0a 100644 --- a/acceptance/tests/connect/connect_inject_test.go +++ b/acceptance/tests/connect/connect_inject_test.go @@ -113,7 +113,7 @@ func TestConnectInject_CleanupKilledPods(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating static-client deployment") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") logger.Log(t, "waiting for static-client to be registered with Consul") consulClient, _ := consulCluster.SetupConsulClient(t, secure) @@ -208,8 +208,8 @@ func TestConnectInject_MultiportServices(t *testing.T) { } logger.Log(t, "creating multiport static-server and static-client deployments") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/multiport-app") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject-multiport") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/multiport-app") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject-multiport") // Check that static-client has been injected and now has 2 containers. podList, err := ctx.KubernetesClient(t).CoreV1().Pods(ctx.KubectlOptions(t).Namespace).List(context.Background(), metav1.ListOptions{ @@ -268,7 +268,7 @@ func TestConnectInject_MultiportServices(t *testing.T) { // pod to static-server. // Deploy static-server. - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // For outbound connections from the multi port pod, only intentions from the first service in the multiport // pod need to be created, since all upstream connections are made through the first service's envoy proxy. diff --git a/acceptance/tests/connect/permissive_mtls_test.go b/acceptance/tests/connect/permissive_mtls_test.go index d3d863ad61..52b5ba513e 100644 --- a/acceptance/tests/connect/permissive_mtls_test.go +++ b/acceptance/tests/connect/permissive_mtls_test.go @@ -59,7 +59,7 @@ func deployNonMeshClient(t *testing.T, ch connhelper.ConnectHelper) { t.Helper() logger.Log(t, "Creating static-client deployment with connect-inject=false") - k8s.DeployKustomize(t, ch.Ctx.KubectlOptions(t), ch.Cfg.NoCleanupOnFailure, ch.Cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, ch.Ctx.KubectlOptions(t), ch.Cfg.NoCleanupOnFailure, ch.Cfg.NoCleanup, ch.Cfg.DebugDirectory, "../fixtures/bases/static-client") requirePodContainers(t, ch, "app=static-client", 1) } @@ -67,7 +67,7 @@ func deployStaticServer(t *testing.T, cfg *config.TestConfig, ch connhelper.Conn t.Helper() logger.Log(t, "Creating static-server deployment with connect-inject=true") - k8s.DeployKustomize(t, ch.Ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ch.Ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, ch.Cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") requirePodContainers(t, ch, "app=static-server", 2) } diff --git a/acceptance/tests/consul-dns/consul_dns_test.go b/acceptance/tests/consul-dns/consul_dns_test.go index c33de7747d..84741175af 100644 --- a/acceptance/tests/consul-dns/consul_dns_test.go +++ b/acceptance/tests/consul-dns/consul_dns_test.go @@ -65,7 +65,7 @@ func TestConsulDNS(t *testing.T) { "run", "-it", dnsPodName, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", fmt.Sprintf("@%s-consul-dns", releaseName), "consul.service.consul", } - helpers.Cleanup(t, suite.Config().NoCleanupOnFailure, func() { + helpers.Cleanup(t, suite.Config().NoCleanupOnFailure, suite.Config().NoCleanup, func() { // Note: this delete command won't wait for pods to be fully terminated. // This shouldn't cause any test pollution because the underlying // objects are deployments, and so when other tests create these diff --git a/acceptance/tests/example/example_test.go b/acceptance/tests/example/example_test.go index b324ac31fe..07b04d6097 100644 --- a/acceptance/tests/example/example_test.go +++ b/acceptance/tests/example/example_test.go @@ -44,7 +44,7 @@ func TestExample(t *testing.T) { k8s.KubectlApply(t, ctx.KubectlOptions(t), "path/to/config") // Clean up any Kubernetes resources you have created - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDelete(t, ctx.KubectlOptions(t), "path/to/config") }) diff --git a/acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml new file mode 100644 index 0000000000..3f9d23c28a --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/default-ns/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - sameness.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml b/acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml new file mode 100644 index 0000000000..0eb7d9e008 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/default-ns/sameness.yaml @@ -0,0 +1,12 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: SamenessGroup +metadata: + name: mine +spec: + members: + - partition: default + - partition: ap1 + - peer: cluster-01-a + - peer: cluster-01-b + - peer: cluster-02-a + - peer: cluster-03-a diff --git a/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml new file mode 100644 index 0000000000..3dc494dd43 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/exportedservices-ap1.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ExportedServices +metadata: + name: ap1 +spec: + services: [] diff --git a/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml new file mode 100644 index 0000000000..1793fa6db7 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/exportedservices-ap1/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - exportedservices-ap1.yaml diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml new file mode 100644 index 0000000000..425b9fe21d --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/intentions.yaml @@ -0,0 +1,12 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: static-server +spec: + destination: + name: static-server + sources: + - name: static-client + namespace: ns1 + samenessGroup: mine + action: allow diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml new file mode 100644 index 0000000000..adfd1c827b --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/kustomization.yaml @@ -0,0 +1,7 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - intentions.yaml + - payment-service-resolver.yaml + - service-defaults.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml new file mode 100644 index 0000000000..b2b6b68c3d --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/payment-service-resolver.yaml @@ -0,0 +1,13 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceResolver +metadata: + name: static-server +spec: + connectTimeout: 15s + failover: + '*': + samenessGroup: mine + policy: + mode: order-by-locality + regions: + - us-west-2 diff --git a/acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml b/acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml new file mode 100644 index 0000000000..f88d143728 --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/override-ns/service-defaults.yaml @@ -0,0 +1,6 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: static-server +spec: + protocol: http \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml b/acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml new file mode 100644 index 0000000000..926e91236d --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/peering/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - mesh.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml b/acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml new file mode 100644 index 0000000000..de84382d3e --- /dev/null +++ b/acceptance/tests/fixtures/bases/sameness/peering/mesh.yaml @@ -0,0 +1,7 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: Mesh +metadata: + name: mesh +spec: + peering: + peerThroughMeshGateways: true diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml new file mode 100644 index 0000000000..2a2f47a332 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/sameness/exportedservices-ap1 + +patchesStrategicMerge: +- patch.yaml diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml new file mode 100644 index 0000000000..d71e8211ba --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/ap1-partition/patch.yaml @@ -0,0 +1,16 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ExportedServices +metadata: + name: ap1 +spec: + services: + - name: static-server + namespace: ns2 + consumers: + - samenessGroup: mine + - name: mesh-gateway + consumers: + - samenessGroup: mine diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml new file mode 100644 index 0000000000..05de6151fc --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/exportedservices-default + +patchesStrategicMerge: +- patch.yaml diff --git a/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml new file mode 100644 index 0000000000..9bb440637e --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/exported-services/default-partition/patch.yaml @@ -0,0 +1,16 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ExportedServices +metadata: + name: default +spec: + services: + - name: static-server + namespace: ns2 + consumers: + - samenessGroup: mine + - name: mesh-gateway + consumers: + - samenessGroup: mine diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml new file mode 100644 index 0000000000..227f223c9f --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/default/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml new file mode 100644 index 0000000000..1775e9abb1 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/default/patch.yaml @@ -0,0 +1,22 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'static-server.ns2.default:8080' + spec: + containers: + - name: static-client + image: anubhavmishra/tiny-tools:latest + # Just spin & wait forever, we'll use `kubectl exec` to demo + command: ['/bin/sh', '-c', '--'] + args: ['while true; do sleep 30; done;'] + # If ACLs are enabled, the serviceAccountName must match the Consul service name. + serviceAccountName: static-client \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml new file mode 100644 index 0000000000..227f223c9f --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-client + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml new file mode 100644 index 0000000000..c1a14c6070 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-client/partition/patch.yaml @@ -0,0 +1,22 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + 'consul.hashicorp.com/connect-inject': 'true' + 'consul.hashicorp.com/connect-service-upstreams': 'static-server.ns2.ap1:8080' + spec: + containers: + - name: static-client + image: anubhavmishra/tiny-tools:latest + # Just spin & wait forever, we'll use `kubectl exec` to demo + command: ['/bin/sh', '-c', '--'] + args: ['while true; do sleep 30; done;'] + # If ACLs are enabled, the serviceAccountName must match the Consul service name. + serviceAccountName: static-client diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml new file mode 100644 index 0000000000..c15bfe7ba7 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/default/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-server + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml new file mode 100644 index 0000000000..ca27b7ba42 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/default/patch.yaml @@ -0,0 +1,23 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + containers: + - name: static-server + image: docker.mirror.hashicorp.services/hashicorp/http-echo:alpine + args: + - -text="cluster-01-a" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + serviceAccountName: static-server diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml new file mode 100644 index 0000000000..c15bfe7ba7 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/partition/kustomization.yaml @@ -0,0 +1,8 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - ../../../../bases/static-server + +patchesStrategicMerge: + - patch.yaml \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml b/acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml new file mode 100644 index 0000000000..044115d1d1 --- /dev/null +++ b/acceptance/tests/fixtures/cases/sameness/static-server/partition/patch.yaml @@ -0,0 +1,23 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-server +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + spec: + containers: + - name: static-server + image: docker.mirror.hashicorp.services/hashicorp/http-echo:alpine + args: + - -text="cluster-01-b" + - -listen=:8080 + ports: + - containerPort: 8080 + name: http + serviceAccountName: static-server diff --git a/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go b/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go index ec4878df04..9edb1db010 100644 --- a/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go +++ b/acceptance/tests/ingress-gateway/ingress_gateway_namespaces_test.go @@ -69,7 +69,7 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { logger.Logf(t, "creating Kubernetes namespace %s", testNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", testNamespace) }) @@ -80,12 +80,12 @@ func TestIngressGatewaySingleNamespace(t *testing.T) { } logger.Logf(t, "creating server in %s namespace", testNamespace) - k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // We use the static-client pod so that we can make calls to the ingress gateway // via kubectl exec without needing a route into the cluster from the test machine. logger.Logf(t, "creating static-client in %s namespace", testNamespace) - k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") // With the cluster up, we can create our ingress-gateway config entry. logger.Log(t, "creating config entry") @@ -188,7 +188,7 @@ func TestIngressGatewayNamespaceMirroring(t *testing.T) { logger.Logf(t, "creating Kubernetes namespace %s", testNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", testNamespace) }) @@ -199,12 +199,12 @@ func TestIngressGatewayNamespaceMirroring(t *testing.T) { } logger.Logf(t, "creating server in %s namespace", testNamespace) - k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // We use the static-client pod so that we can make calls to the ingress gateway // via kubectl exec without needing a route into the cluster from the test machine. logger.Logf(t, "creating static-client in %s namespace", testNamespace) - k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) diff --git a/acceptance/tests/ingress-gateway/ingress_gateway_test.go b/acceptance/tests/ingress-gateway/ingress_gateway_test.go index b5df6287b6..d2b3d7193e 100644 --- a/acceptance/tests/ingress-gateway/ingress_gateway_test.go +++ b/acceptance/tests/ingress-gateway/ingress_gateway_test.go @@ -52,12 +52,12 @@ func TestIngressGateway(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating server") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // We use the static-client pod so that we can make calls to the ingress gateway // via kubectl exec without needing a route into the cluster from the test machine. logger.Log(t, "creating static-client pod") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") // With the cluster up, we can create our ingress-gateway config entry. logger.Log(t, "creating config entry") diff --git a/acceptance/tests/metrics/metrics_test.go b/acceptance/tests/metrics/metrics_test.go index 2a130f38db..3d40841e36 100644 --- a/acceptance/tests/metrics/metrics_test.go +++ b/acceptance/tests/metrics/metrics_test.go @@ -71,7 +71,7 @@ func TestComponentMetrics(t *testing.T) { // This simulates queries that would be made by a prometheus server that runs externally to the consul // components in the cluster. logger.Log(t, "creating static-client") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") // Server Metrics metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:8500/v1/agent/metrics?format=prometheus", fmt.Sprintf("%s-consul-server.%s.svc", releaseName, ns))) @@ -116,13 +116,13 @@ func TestAppMetrics(t *testing.T) { // Deploy service that will emit app and envoy metrics at merged metrics endpoint logger.Log(t, "creating static-metrics-app") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-metrics-app") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-metrics-app") // Create the static-client deployment so we can use it for in-cluster calls to metrics endpoints. // This simulates queries that would be made by a prometheus server that runs externally to the consul // components in the cluster. logger.Log(t, "creating static-client") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") // Merged App Metrics podList, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: "app=static-metrics-app"}) diff --git a/acceptance/tests/partitions/partitions_connect_test.go b/acceptance/tests/partitions/partitions_connect_test.go index aa73c17047..c53e5b6171 100644 --- a/acceptance/tests/partitions/partitions_connect_test.go +++ b/acceptance/tests/partitions/partitions_connect_test.go @@ -108,6 +108,7 @@ func TestPartitions_Connect(t *testing.T) { "dns.enableRedirection": strconv.FormatBool(cfg.EnableTransparentProxy), } + // Setup the default partition defaultPartitionHelmValues := make(map[string]string) // On Kind, there are no load balancers but since all clusters @@ -129,6 +130,7 @@ func TestPartitions_Connect(t *testing.T) { serverConsulCluster := consul.NewHelmCluster(t, defaultPartitionHelmValues, defaultPartitionClusterContext, cfg, releaseName) serverConsulCluster.Create(t) + // Copy secrets from the default partition to the secondary partition // Get the TLS CA certificate and key secret from the server cluster and apply it to the client cluster. caCertSecretName := fmt.Sprintf("%s-consul-ca-cert", releaseName) @@ -146,7 +148,7 @@ func TestPartitions_Connect(t *testing.T) { k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, secondaryPartitionClusterContext) - // Create client cluster. + // Create secondary partition cluster. secondaryPartitionHelmValues := map[string]string{ "global.enabled": "false", @@ -204,14 +206,14 @@ func TestPartitions_Connect(t *testing.T) { logger.Logf(t, "creating namespaces %s and %s in servers cluster", staticServerNamespace, StaticClientNamespace) k8s.RunKubectl(t, defaultPartitionClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) k8s.RunKubectl(t, defaultPartitionClusterContext.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, defaultPartitionClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace, StaticClientNamespace) }) logger.Logf(t, "creating namespaces %s and %s in clients cluster", staticServerNamespace, StaticClientNamespace) k8s.RunKubectl(t, secondaryPartitionClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) k8s.RunKubectl(t, secondaryPartitionClusterContext.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, secondaryPartitionClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace, StaticClientNamespace) }) @@ -271,37 +273,37 @@ func TestPartitions_Connect(t *testing.T) { kustomizeDir := "../fixtures/bases/mesh-gateway" k8s.KubectlApplyK(t, defaultPartitionClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, defaultPartitionClusterContext.KubectlOptions(t), kustomizeDir) }) k8s.KubectlApplyK(t, secondaryPartitionClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, secondaryPartitionClusterContext.KubectlOptions(t), kustomizeDir) }) // This section of the tests runs the in-partition networking tests. t.Run("in-partition", func(t *testing.T) { logger.Log(t, "test in-partition networking") logger.Log(t, "creating static-server and static-client deployments in server cluster") - k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { if c.destinationNamespace == defaultNamespace { - k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } else { - k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") + k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") } } logger.Log(t, "creating static-server and static-client deployments in client cluster") - k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { if c.destinationNamespace == defaultNamespace { - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } else { - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") } } // Check that both static-server and static-client have been injected and now have 2 containers in server cluster. @@ -383,7 +385,7 @@ func TestPartitions_Connect(t *testing.T) { require.NoError(t, err) _, _, err = consulClient.ConfigEntries().Set(intention, &api.WriteOptions{Partition: secondaryPartition}) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _, err := consulClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{Partition: defaultPartition}) require.NoError(t, err) _, err = consulClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{Partition: secondaryPartition}) @@ -424,25 +426,25 @@ func TestPartitions_Connect(t *testing.T) { t.Run("cross-partition", func(t *testing.T) { logger.Log(t, "test cross-partition networking") logger.Log(t, "creating static-server and static-client deployments in server cluster") - k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { if c.destinationNamespace == defaultNamespace { - k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/default-ns-partition") + k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/default-ns-partition") } else { - k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/ns-partition") + k8s.DeployKustomize(t, defaultPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/ns-partition") } } logger.Log(t, "creating static-server and static-client deployments in client cluster") - k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { if c.destinationNamespace == defaultNamespace { - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/default-ns-default-partition") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/default-ns-default-partition") } else { - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/ns-default-partition") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-partitions/ns-default-partition") } } // Check that both static-server and static-client have been injected and now have 2 containers in server cluster. @@ -496,14 +498,14 @@ func TestPartitions_Connect(t *testing.T) { if c.destinationNamespace == defaultNamespace { k8s.KubectlApplyK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-default") k8s.KubectlApplyK(t, secondaryPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/secondary-partition-default") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-default") k8s.KubectlDeleteK(t, secondaryPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/secondary-partition-default") }) } else { k8s.KubectlApplyK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-ns1") k8s.KubectlApplyK(t, secondaryPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/secondary-partition-ns1") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-ns1") k8s.KubectlDeleteK(t, secondaryPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/secondary-partition-ns1") }) @@ -551,7 +553,7 @@ func TestPartitions_Connect(t *testing.T) { intention.Sources[0].Partition = defaultPartition _, _, err = consulClient.ConfigEntries().Set(intention, &api.WriteOptions{Partition: secondaryPartition}) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _, err := consulClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{Partition: defaultPartition}) require.NoError(t, err) _, err = consulClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{Partition: secondaryPartition}) diff --git a/acceptance/tests/partitions/partitions_gateway_test.go b/acceptance/tests/partitions/partitions_gateway_test.go index cbd9152612..a90a790cb6 100644 --- a/acceptance/tests/partitions/partitions_gateway_test.go +++ b/acceptance/tests/partitions/partitions_gateway_test.go @@ -148,14 +148,14 @@ func TestPartitions_Gateway(t *testing.T) { logger.Logf(t, "creating namespaces %s and %s in servers cluster", staticServerNamespace, StaticClientNamespace) k8s.RunKubectl(t, defaultPartitionClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) k8s.RunKubectl(t, defaultPartitionClusterContext.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, defaultPartitionClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace, StaticClientNamespace) }) logger.Logf(t, "creating namespaces %s and %s in clients cluster", staticServerNamespace, StaticClientNamespace) k8s.RunKubectl(t, secondaryPartitionClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) k8s.RunKubectl(t, secondaryPartitionClusterContext.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, secondaryPartitionClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace, StaticClientNamespace) }) @@ -203,12 +203,12 @@ func TestPartitions_Gateway(t *testing.T) { kustomizeDir := "../fixtures/cases/api-gateways/mesh" k8s.KubectlApplyK(t, defaultPartitionClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, defaultPartitionClusterContext.KubectlOptions(t), kustomizeDir) }) k8s.KubectlApplyK(t, secondaryPartitionClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, secondaryPartitionClusterContext.KubectlOptions(t), kustomizeDir) }) @@ -217,12 +217,12 @@ func TestPartitions_Gateway(t *testing.T) { // Since we're deploying the gateway in the secondary cluster, we create the static client // in the secondary as well. logger.Log(t, "creating static-client pod in secondary partition cluster") - k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-client") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-client") logger.Log(t, "creating api-gateway resources") out, err := k8s.RunKubectlAndGetOutputE(t, secondaryPartitionClusterStaticServerOpts, "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, secondaryPartitionClusterStaticServerOpts, "delete", "-k", "../fixtures/bases/api-gateway") @@ -254,7 +254,7 @@ func TestPartitions_Gateway(t *testing.T) { t.Run("in-partition", func(t *testing.T) { logger.Log(t, "test in-partition networking") logger.Log(t, "creating target server in secondary partition cluster") - k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, secondaryPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // Check that static-server injected 2 containers. for _, labelSelector := range []string{"app=static-server"} { @@ -288,7 +288,7 @@ func TestPartitions_Gateway(t *testing.T) { logger.Log(t, "creating intention") _, _, err = consulClient.ConfigEntries().Set(intention, &api.WriteOptions{Partition: secondaryPartition}) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _, err = consulClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{Partition: secondaryPartition}) require.NoError(t, err) }) @@ -302,7 +302,7 @@ func TestPartitions_Gateway(t *testing.T) { logger.Log(t, "test cross-partition networking") logger.Log(t, "creating target server in default partition cluster") - k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, defaultPartitionClusterStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") // Check that static-server injected 2 containers. for _, labelSelector := range []string{"app=static-server"} { @@ -316,13 +316,13 @@ func TestPartitions_Gateway(t *testing.T) { logger.Log(t, "creating exported services") k8s.KubectlApplyK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-ns1") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, defaultPartitionClusterContext.KubectlOptions(t), "../fixtures/cases/crd-partitions/default-partition-ns1") }) logger.Log(t, "creating local service resolver") k8s.KubectlApplyK(t, secondaryPartitionClusterStaticServerOpts, "../fixtures/cases/api-gateways/resolver") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, secondaryPartitionClusterStaticServerOpts, "../fixtures/cases/api-gateways/resolver") }) @@ -349,7 +349,7 @@ func TestPartitions_Gateway(t *testing.T) { logger.Log(t, "creating intention") _, _, err = consulClient.ConfigEntries().Set(intention, &api.WriteOptions{Partition: defaultPartition}) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _, err = consulClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{Partition: defaultPartition}) require.NoError(t, err) }) diff --git a/acceptance/tests/partitions/partitions_sync_test.go b/acceptance/tests/partitions/partitions_sync_test.go index 8eaaff099e..da95d7f272 100644 --- a/acceptance/tests/partitions/partitions_sync_test.go +++ b/acceptance/tests/partitions/partitions_sync_test.go @@ -195,13 +195,13 @@ func TestPartitions_Sync(t *testing.T) { logger.Logf(t, "creating namespaces %s in servers cluster", staticServerNamespace) k8s.RunKubectl(t, primaryClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, primaryClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) logger.Logf(t, "creating namespaces %s in clients cluster", staticServerNamespace) k8s.RunKubectl(t, secondaryClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, secondaryClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) @@ -241,9 +241,9 @@ func TestPartitions_Sync(t *testing.T) { logger.Log(t, "creating a static-server with a service") // create service in default partition. - k8s.DeployKustomize(t, primaryStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, primaryStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") // create service in secondary partition. - k8s.DeployKustomize(t, secondaryStaticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, secondaryStaticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") logger.Log(t, "checking that the service has been synced to Consul") var services map[string][]string diff --git a/acceptance/tests/peering/peering_connect_namespaces_test.go b/acceptance/tests/peering/peering_connect_namespaces_test.go index 622e547091..618248c6a9 100644 --- a/acceptance/tests/peering/peering_connect_namespaces_test.go +++ b/acceptance/tests/peering/peering_connect_namespaces_test.go @@ -166,12 +166,12 @@ func TestPeering_ConnectNamespaces(t *testing.T) { kustomizeMeshDir := "../fixtures/bases/mesh-peering" k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) }) k8s.KubectlApplyK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) }) @@ -198,7 +198,7 @@ func TestPeering_ConnectNamespaces(t *testing.T) { // Create the peering acceptor on the client peer. k8s.KubectlApply(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDelete(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml") }) @@ -214,7 +214,7 @@ func TestPeering_ConnectNamespaces(t *testing.T) { // Create the peering dialer on the server peer. k8s.KubectlApply(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-dialer.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "delete", "secret", "api-token") k8s.KubectlDelete(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-dialer.yaml") }) @@ -232,13 +232,13 @@ func TestPeering_ConnectNamespaces(t *testing.T) { logger.Logf(t, "creating namespaces %s in server peer", staticServerNamespace) k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) logger.Logf(t, "creating namespaces %s in client peer", staticClientNamespace) k8s.RunKubectl(t, staticClientPeerClusterContext.KubectlOptions(t), "create", "ns", staticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticClientPeerClusterContext.KubectlOptions(t), "delete", "ns", staticClientNamespace) }) @@ -256,26 +256,26 @@ func TestPeering_ConnectNamespaces(t *testing.T) { kustomizeDir := "../fixtures/bases/mesh-gateway" k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeDir) }) k8s.KubectlApplyK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeDir) }) logger.Log(t, "creating static-server in server peer") - k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating static-client deployments in client peer") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { if c.destinationNamespace == defaultNamespace { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/default-namespace") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/default-namespace") } else { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/non-default-namespace") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/non-default-namespace") } } // Check that both static-server and static-client have been injected and now have 2 containers. @@ -312,12 +312,12 @@ func TestPeering_ConnectNamespaces(t *testing.T) { logger.Log(t, "creating exported services") if c.destinationNamespace == defaultNamespace { k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/default-namespace") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/default-namespace") }) } else { k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/non-default-namespace") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/non-default-namespace") }) } diff --git a/acceptance/tests/peering/peering_connect_test.go b/acceptance/tests/peering/peering_connect_test.go index a14cf3a805..d92ab59990 100644 --- a/acceptance/tests/peering/peering_connect_test.go +++ b/acceptance/tests/peering/peering_connect_test.go @@ -121,12 +121,12 @@ func TestPeering_Connect(t *testing.T) { kustomizeMeshDir := "../fixtures/bases/mesh-peering" k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) }) k8s.KubectlApplyK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) }) @@ -153,7 +153,7 @@ func TestPeering_Connect(t *testing.T) { // Create the peering acceptor on the client peer. k8s.KubectlApply(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDelete(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml") }) @@ -169,7 +169,7 @@ func TestPeering_Connect(t *testing.T) { // Create the peering dialer on the server peer. k8s.KubectlApply(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-dialer.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "delete", "secret", "api-token") k8s.KubectlDelete(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-dialer.yaml") }) @@ -187,13 +187,13 @@ func TestPeering_Connect(t *testing.T) { logger.Logf(t, "creating namespaces %s in server peer", staticServerNamespace) k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) logger.Logf(t, "creating namespaces %s in client peer", staticClientNamespace) k8s.RunKubectl(t, staticClientPeerClusterContext.KubectlOptions(t), "create", "ns", staticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticClientPeerClusterContext.KubectlOptions(t), "delete", "ns", staticClientNamespace) }) @@ -203,23 +203,23 @@ func TestPeering_Connect(t *testing.T) { kustomizeDir := "../fixtures/bases/mesh-gateway" k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeDir) }) k8s.KubectlApplyK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeDir) }) logger.Log(t, "creating static-server in server peer") - k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating static-client deployments in client peer") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/default") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/default") } // Check that both static-server and static-client have been injected and now have 2 containers. podList, err := staticServerPeerClusterContext.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{ @@ -249,7 +249,7 @@ func TestPeering_Connect(t *testing.T) { logger.Log(t, "creating exported services") k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/default") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/default") }) diff --git a/acceptance/tests/peering/peering_gateway_test.go b/acceptance/tests/peering/peering_gateway_test.go index 17824b8e69..3ba04980a9 100644 --- a/acceptance/tests/peering/peering_gateway_test.go +++ b/acceptance/tests/peering/peering_gateway_test.go @@ -113,12 +113,12 @@ func TestPeering_Gateway(t *testing.T) { kustomizeMeshDir := "../fixtures/bases/mesh-peering" k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) }) k8s.KubectlApplyK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeMeshDir) }) @@ -145,7 +145,7 @@ func TestPeering_Gateway(t *testing.T) { // Create the peering acceptor on the client peer. k8s.KubectlApply(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDelete(t, staticClientPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-acceptor.yaml") }) @@ -161,7 +161,7 @@ func TestPeering_Gateway(t *testing.T) { // Create the peering dialer on the server peer. k8s.KubectlApply(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-dialer.yaml") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "delete", "secret", "api-token") k8s.KubectlDelete(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/bases/peering/peering-dialer.yaml") }) @@ -179,13 +179,13 @@ func TestPeering_Gateway(t *testing.T) { logger.Logf(t, "creating namespaces %s in server peer", staticServerNamespace) k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticServerPeerClusterContext.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) logger.Logf(t, "creating namespaces %s in client peer", staticClientNamespace) k8s.RunKubectl(t, staticClientPeerClusterContext.KubectlOptions(t), "create", "ns", staticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, staticClientPeerClusterContext.KubectlOptions(t), "delete", "ns", staticClientNamespace) }) @@ -195,12 +195,12 @@ func TestPeering_Gateway(t *testing.T) { kustomizeDir := "../fixtures/cases/api-gateways/mesh" k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), kustomizeDir) }) k8s.KubectlApplyK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientPeerClusterContext.KubectlOptions(t), kustomizeDir) }) @@ -209,21 +209,21 @@ func TestPeering_Gateway(t *testing.T) { // Since we're deploying the gateway in the secondary cluster, we create the static client // in the secondary as well. logger.Log(t, "creating static-client pod in client peer") - k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/non-default-namespace") + k8s.DeployKustomize(t, staticClientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-peers/non-default-namespace") logger.Log(t, "creating static-server in server peer") - k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating exported services") k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/non-default-namespace") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/non-default-namespace") }) logger.Log(t, "creating api-gateway resources in client peer") out, err := k8s.RunKubectlAndGetOutputE(t, staticClientOpts, "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, staticClientOpts, "delete", "-k", "../fixtures/bases/api-gateway") @@ -253,7 +253,7 @@ func TestPeering_Gateway(t *testing.T) { logger.Log(t, "creating local service resolver") k8s.KubectlApplyK(t, staticClientOpts, "../fixtures/cases/api-gateways/peer-resolver") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, staticClientOpts, "../fixtures/cases/api-gateways/peer-resolver") }) @@ -280,7 +280,7 @@ func TestPeering_Gateway(t *testing.T) { logger.Log(t, "creating intention") _, _, err = staticServerPeerClient.ConfigEntries().Set(intention, &api.WriteOptions{}) require.NoError(t, err) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { _, err = staticServerPeerClient.ConfigEntries().Delete(api.ServiceIntentions, staticServerName, &api.WriteOptions{}) require.NoError(t, err) }) diff --git a/acceptance/tests/sameness/main_test.go b/acceptance/tests/sameness/main_test.go new file mode 100644 index 0000000000..67e6ee42b7 --- /dev/null +++ b/acceptance/tests/sameness/main_test.go @@ -0,0 +1,28 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package sameness + +import ( + "fmt" + "os" + "testing" + + testsuite "github.com/hashicorp/consul-k8s/acceptance/framework/suite" +) + +var suite testsuite.Suite + +func TestMain(m *testing.M) { + suite = testsuite.NewSuite(m) + + expectedNumberOfClusters := 4 + + if suite.Config().EnableMultiCluster && suite.Config().IsExpectedClusterCount(expectedNumberOfClusters) && suite.Config().UseKind { + os.Exit(suite.Run()) + } else { + fmt.Println(fmt.Sprintf("Skipping sameness tests because either -enable-multi-cluster is "+ + "not set, the number of clusters did not match the expected count of %d, or --useKind is false. "+ + "Sameness acceptance tests are currently only suopported on Kind clusters", expectedNumberOfClusters)) + } +} diff --git a/acceptance/tests/sameness/sameness_test.go b/acceptance/tests/sameness/sameness_test.go new file mode 100644 index 0000000000..a7a926cd42 --- /dev/null +++ b/acceptance/tests/sameness/sameness_test.go @@ -0,0 +1,454 @@ +package sameness + +import ( + "context" + "fmt" + "strconv" + "testing" + + terratestk8s "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/config" + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + primaryDatacenterPartition = "ap1" + primaryServerDatacenter = "dc1" + peer1Datacenter = "dc2" + peer2Datacenter = "dc3" + staticClientNamespace = "ns1" + staticServerNamespace = "ns2" + + keyPrimaryServer = "server" + keyPartition = "partition" + keyPeer1 = "peer1" + keyPeer2 = "peer2" + + staticServerDeployment = "deploy/static-server" + staticClientDeployment = "deploy/static-client" + + primaryServerClusterName = "cluster-01-a" + partitionClusterName = "cluster-01-b" +) + +func TestFailover_Connect(t *testing.T) { + env := suite.Environment() + cfg := suite.Config() + + if !cfg.EnableEnterprise { + t.Skipf("skipping this test because -enable-enterprise is not set") + } + + cases := []struct { + name string + ACLsEnabled bool + }{ + { + "default failover", + false, + }, + { + "secure failover", + true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + /* + Architecture Overview: + Primary Datacenter (DC1) + Default Partition + Peer -> DC2 (cluster-02-a) + Peer -> DC3 (cluster-03-a) + AP1 Partition + Peer -> DC2 (cluster-02-a) + Peer -> DC3 (cluster-03-a) + Datacenter 2 (DC2) + Default Partition + Peer -> DC1 (cluster-01-a) + Peer -> DC1 (cluster-01-b) + Peer -> DC3 (cluster-03-a) + Datacenter 3 (DC3) + Default Partition + Peer -> DC1 (cluster-01-a) + Peer -> DC1 (cluster-01-b) + Peer -> DC2 (cluster-02-a) + + + Architecture Diagram + failover scenarios from perspective of DC1 Default Partition Static-Server + +-------------------------------------------+ + | | + | DC1 | + | | + | +-----------------------------+ | +-----------------------------------+ + | | | | | DC2 | + | | +------------------+ | | Failover 2 | +------------------+ | + | | | +-------+--------+-----------------+------>| | | + | | | Static-Server | | | | | Static-Server | | + | | | +-------+---+ | | | | | + | | | | | | | | | | | + | | | | | | | | | | | + | | | +-------+---+----+-------------+ | | | | + | | +------------------+ | | | | | +------------------+ | + | | Admin Partitions: Default | | | | | | + | | Name: cluster-01-a | | | | | Admin Partitions: Default | + | | | | | | | Name: cluster-02-a | + | +-----------------------------+ | | | | | + | | | | +-----------------------------------+ + | Failover 1| | Failover 3 | + | +-------------------------------+ | | | +-----------------------------------+ + | | | | | | | DC3 | + | | +------------------+ | | | | | +------------------+ | + | | | | | | | | | | Static-Server | | + | | | Static-Server | | | | | | | | | + | | | | | | | | | | | | + | | | | | | | +---+------>| | | + | | | |<------+--+ | | | | | + | | | | | | | +------------------+ | + | | +------------------+ | | | | + | | Admin Partitions: ap1 | | | Admin Partitions: Default | + | | Name: cluster-01-b | | | Name: cluster-03-a | + | | | | | | + | +-------------------------------+ | | | + | | +-----------------------------------+ + +-------------------------------------------+ + */ + + members := map[string]*member{ + keyPrimaryServer: {context: env.DefaultContext(t), hasServer: true}, + keyPartition: {context: env.Context(t, 1), hasServer: false}, + keyPeer1: {context: env.Context(t, 2), hasServer: true}, + keyPeer2: {context: env.Context(t, 3), hasServer: true}, + } + + // Setup Namespaces. + for _, v := range members { + createNamespaces(t, cfg, v.context) + } + + // Create the Default Cluster. + commonHelmValues := map[string]string{ + "global.peering.enabled": "true", + + "global.tls.enabled": "true", + "global.tls.httpsOnly": strconv.FormatBool(c.ACLsEnabled), + + "global.enableConsulNamespaces": "true", + + "global.adminPartitions.enabled": "true", + + "global.logLevel": "debug", + + "global.acls.manageSystemACLs": strconv.FormatBool(c.ACLsEnabled), + + "connectInject.enabled": "true", + "connectInject.consulNamespaces.mirroringK8S": "true", + + "meshGateway.enabled": "true", + "meshGateway.replicas": "1", + + "dns.enabled": "true", + } + + defaultPartitionHelmValues := map[string]string{ + "global.datacenter": primaryServerDatacenter, + } + + // On Kind, there are no load balancers but since all clusters + // share the same node network (docker bridge), we can use + // a NodePort service so that we can access node(s) in a different Kind cluster. + if cfg.UseKind { + defaultPartitionHelmValues["meshGateway.service.type"] = "NodePort" + defaultPartitionHelmValues["meshGateway.service.nodePort"] = "30200" + defaultPartitionHelmValues["server.exposeService.type"] = "NodePort" + defaultPartitionHelmValues["server.exposeService.nodePort.https"] = "30000" + defaultPartitionHelmValues["server.exposeService.nodePort.grpc"] = "30100" + } + helpers.MergeMaps(defaultPartitionHelmValues, commonHelmValues) + + releaseName := helpers.RandomName() + members[keyPrimaryServer].helmCluster = consul.NewHelmCluster(t, defaultPartitionHelmValues, members[keyPrimaryServer].context, cfg, releaseName) + members[keyPrimaryServer].helmCluster.Create(t) + + // Get the TLS CA certificate and key secret from the server cluster and apply it to the client cluster. + caCertSecretName := fmt.Sprintf("%s-consul-ca-cert", releaseName) + + logger.Logf(t, "retrieving ca cert secret %s from the server cluster and applying to the client cluster", caCertSecretName) + k8s.CopySecret(t, members[keyPrimaryServer].context, members[keyPartition].context, caCertSecretName) + + // Create Secondary Partition Cluster which will apply the primary datacenter. + partitionToken := fmt.Sprintf("%s-consul-partitions-acl-token", releaseName) + if c.ACLsEnabled { + logger.Logf(t, "retrieving partition token secret %s from the server cluster and applying to the client cluster", partitionToken) + k8s.CopySecret(t, members[keyPrimaryServer].context, members[keyPartition].context, partitionToken) + } + + partitionServiceName := fmt.Sprintf("%s-consul-expose-servers", releaseName) + partitionSvcAddress := k8s.ServiceHost(t, cfg, members[keyPrimaryServer].context, partitionServiceName) + + k8sAuthMethodHost := k8s.KubernetesAPIServerHost(t, cfg, members[keyPartition].context) + + secondaryPartitionHelmValues := map[string]string{ + "global.enabled": "false", + "global.datacenter": primaryServerDatacenter, + + "global.adminPartitions.name": primaryDatacenterPartition, + + "global.tls.caCert.secretName": caCertSecretName, + "global.tls.caCert.secretKey": "tls.crt", + + "externalServers.enabled": "true", + "externalServers.hosts[0]": partitionSvcAddress, + "externalServers.tlsServerName": fmt.Sprintf("server.%s.consul", primaryServerDatacenter), + "global.server.enabled": "false", + } + + if c.ACLsEnabled { + // Setup partition token and auth method host if ACLs enabled. + secondaryPartitionHelmValues["global.acls.bootstrapToken.secretName"] = partitionToken + secondaryPartitionHelmValues["global.acls.bootstrapToken.secretKey"] = "token" + secondaryPartitionHelmValues["externalServers.k8sAuthMethodHost"] = k8sAuthMethodHost + } + + if cfg.UseKind { + secondaryPartitionHelmValues["externalServers.httpsPort"] = "30000" + secondaryPartitionHelmValues["externalServers.grpcPort"] = "30100" + secondaryPartitionHelmValues["meshGateway.service.type"] = "NodePort" + secondaryPartitionHelmValues["meshGateway.service.nodePort"] = "30200" + } + helpers.MergeMaps(secondaryPartitionHelmValues, commonHelmValues) + + members[keyPartition].helmCluster = consul.NewHelmCluster(t, secondaryPartitionHelmValues, members[keyPartition].context, cfg, releaseName) + members[keyPartition].helmCluster.Create(t) + + // Create Peer 1 Cluster. + PeerOneHelmValues := map[string]string{ + "global.datacenter": peer1Datacenter, + } + + if cfg.UseKind { + PeerOneHelmValues["server.exposeGossipAndRPCPorts"] = "true" + PeerOneHelmValues["meshGateway.service.type"] = "NodePort" + PeerOneHelmValues["meshGateway.service.nodePort"] = "30100" + } + helpers.MergeMaps(PeerOneHelmValues, commonHelmValues) + + members[keyPeer1].helmCluster = consul.NewHelmCluster(t, PeerOneHelmValues, members[keyPeer1].context, cfg, releaseName) + members[keyPeer1].helmCluster.Create(t) + + // Create Peer 2 Cluster. + PeerTwoHelmValues := map[string]string{ + "global.datacenter": peer2Datacenter, + } + + if cfg.UseKind { + PeerTwoHelmValues["server.exposeGossipAndRPCPorts"] = "true" + PeerTwoHelmValues["meshGateway.service.type"] = "NodePort" + PeerTwoHelmValues["meshGateway.service.nodePort"] = "30100" + } + helpers.MergeMaps(PeerTwoHelmValues, commonHelmValues) + + members[keyPeer2].helmCluster = consul.NewHelmCluster(t, PeerTwoHelmValues, members[keyPeer2].context, cfg, releaseName) + members[keyPeer2].helmCluster.Create(t) + + // Create a ProxyDefaults resource to configure services to use the mesh + // gateways and set server and client opts. + for k, v := range members { + logger.Logf(t, "applying resources on %s", v.context.KubectlOptions(t).ContextName) + + // Client will use the client namespace. + members[k].clientOpts = &terratestk8s.KubectlOptions{ + ContextName: v.context.KubectlOptions(t).ContextName, + ConfigPath: v.context.KubectlOptions(t).ConfigPath, + Namespace: staticClientNamespace, + } + + // Server will use the server namespace. + members[k].serverOpts = &terratestk8s.KubectlOptions{ + ContextName: v.context.KubectlOptions(t).ContextName, + ConfigPath: v.context.KubectlOptions(t).ConfigPath, + Namespace: staticServerNamespace, + } + + // Sameness Defaults need to be applied first so that the sameness group exists. + applyResources(t, cfg, "../fixtures/bases/mesh-gateway", members[k].context.KubectlOptions(t)) + applyResources(t, cfg, "../fixtures/bases/sameness/default-ns", members[k].context.KubectlOptions(t)) + applyResources(t, cfg, "../fixtures/bases/sameness/override-ns", members[k].serverOpts) + + // Only assign a client if the cluster is running a Consul server. + if v.hasServer { + members[k].client, _ = members[k].helmCluster.SetupConsulClient(t, c.ACLsEnabled) + } + } + + // TODO: Add further setup for peering, right now the rest of this test will only cover Partitions + // Create static server deployments. + logger.Log(t, "creating static-server and static-client deployments") + k8s.DeployKustomize(t, members[keyPrimaryServer].serverOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-server/default") + k8s.DeployKustomize(t, members[keyPartition].serverOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-server/partition") + + // Create static client deployments. + k8s.DeployKustomize(t, members[keyPrimaryServer].clientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-client/default") + k8s.DeployKustomize(t, members[keyPartition].clientOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, + "../fixtures/cases/sameness/static-client/partition") + + // Verify that both static-server and static-client have been injected and now have 2 containers in server cluster. + // Also get the server IP + for _, labelSelector := range []string{"app=static-server", "app=static-client"} { + podList, err := members[keyPrimaryServer].context.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), + metav1.ListOptions{LabelSelector: labelSelector}) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Len(t, podList.Items[0].Spec.Containers, 2) + if labelSelector == "app=static-server" { + ip := &podList.Items[0].Status.PodIP + require.NotNil(t, ip) + logger.Logf(t, "default-static-server-ip: %s", *ip) + members[keyPrimaryServer].staticServerIP = ip + } + + podList, err = members[keyPartition].context.KubernetesClient(t).CoreV1().Pods(metav1.NamespaceAll).List(context.Background(), + metav1.ListOptions{LabelSelector: labelSelector}) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Len(t, podList.Items[0].Spec.Containers, 2) + if labelSelector == "app=static-server" { + ip := &podList.Items[0].Status.PodIP + require.NotNil(t, ip) + logger.Logf(t, "partition-static-server-ip: %s", *ip) + members[keyPartition].staticServerIP = ip + } + } + + logger.Log(t, "creating exported services") + applyResources(t, cfg, "../fixtures/cases/sameness/exported-services/default-partition", members[keyPrimaryServer].context.KubectlOptions(t)) + applyResources(t, cfg, "../fixtures/cases/sameness/exported-services/ap1-partition", members[keyPartition].context.KubectlOptions(t)) + + // Setup DNS. + dnsService, err := members[keyPrimaryServer].context.KubernetesClient(t).CoreV1().Services("default").Get(context.Background(), fmt.Sprintf("%s-%s", releaseName, "consul-dns"), metav1.GetOptions{}) + require.NoError(t, err) + dnsIP := dnsService.Spec.ClusterIP + logger.Logf(t, "dnsIP: %s", dnsIP) + + // Setup Prepared Query. + definition := &api.PreparedQueryDefinition{ + Name: "my-query", + Service: api.ServiceQuery{ + Service: "static-server", + SamenessGroup: "mine", + Namespace: staticServerNamespace, + OnlyPassing: false, + }, + } + resp, _, err := members[keyPrimaryServer].client.PreparedQuery().Create(definition, &api.WriteOptions{}) + require.NoError(t, err) + logger.Logf(t, "PQ ID: %s", resp) + + logger.Log(t, "all infrastructure up and running") + logger.Log(t, "verifying failover scenarios") + + const dnsLookup = "static-server.service.ns2.ns.mine.sg.consul" + const dnsPQLookup = "my-query.query.consul" + + // Verify initial server. + serviceFailoverCheck(t, primaryServerClusterName, members[keyPrimaryServer]) + + // Verify initial dns. + dnsFailoverCheck(t, releaseName, dnsIP, dnsLookup, members[keyPrimaryServer], members[keyPrimaryServer]) + + // Verify initial dns with PQ. + dnsFailoverCheck(t, releaseName, dnsIP, dnsPQLookup, members[keyPrimaryServer], members[keyPrimaryServer]) + + // Scale down static-server on the server, will fail over to partition. + k8s.KubectlScale(t, members[keyPrimaryServer].serverOpts, staticServerDeployment, 0) + + // Verify failover to partition. + serviceFailoverCheck(t, partitionClusterName, members[keyPrimaryServer]) + + // Verify dns failover to partition. + dnsFailoverCheck(t, releaseName, dnsIP, dnsLookup, members[keyPrimaryServer], members[keyPartition]) + + // Verify prepared query failover. + dnsFailoverCheck(t, releaseName, dnsIP, dnsPQLookup, members[keyPrimaryServer], members[keyPartition]) + + logger.Log(t, "tests complete") + }) + } +} + +type member struct { + context environment.TestContext + helmCluster *consul.HelmCluster + client *api.Client + hasServer bool + serverOpts *terratestk8s.KubectlOptions + clientOpts *terratestk8s.KubectlOptions + staticServerIP *string +} + +func createNamespaces(t *testing.T, cfg *config.TestConfig, context environment.TestContext) { + logger.Logf(t, "creating namespaces in %s", context.KubectlOptions(t).ContextName) + k8s.RunKubectl(t, context.KubectlOptions(t), "create", "ns", staticServerNamespace) + k8s.RunKubectl(t, context.KubectlOptions(t), "create", "ns", staticClientNamespace) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.RunKubectl(t, context.KubectlOptions(t), "delete", "ns", staticClientNamespace, staticServerNamespace) + }) +} + +func applyResources(t *testing.T, cfg *config.TestConfig, kustomizeDir string, opts *terratestk8s.KubectlOptions) { + k8s.KubectlApplyK(t, opts, kustomizeDir) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, opts, kustomizeDir) + }) +} + +// serviceFailoverCheck verifies that the server failed over as expected by checking that curling the `static-server` +// using the `static-client` responds with the expected cluster name. Each static-server responds with a uniquue +// name so that we can verify failover occured as expected. +func serviceFailoverCheck(t *testing.T, expectedClusterName string, server *member) { + retry.Run(t, func(r *retry.R) { + resp, err := k8s.RunKubectlAndGetOutputE(t, server.clientOpts, "exec", "-i", + staticClientDeployment, "-c", "static-client", "--", "curl", "localhost:8080") + require.NoError(r, err) + assert.Contains(r, resp, expectedClusterName) + logger.Log(t, resp) + }) +} + +func dnsFailoverCheck(t *testing.T, releaseName string, dnsIP string, dnsQuery string, server, failover *member) { + retry.Run(t, func(r *retry.R) { + logs, err := k8s.RunKubectlAndGetOutputE(t, server.clientOpts, "exec", "-i", + staticClientDeployment, "-c", "static-client", "--", "dig", fmt.Sprintf("@%s-consul-dns.default", releaseName), dnsQuery) + require.NoError(r, err) + + // When the `dig` request is successful, a section of its response looks like the following: + // + // ;; ANSWER SECTION: + // static-server.service.mine.sg.ns2.ns.consul. 0 IN A + // + // ;; Query time: 2 msec + // ;; SERVER: #() + // ;; WHEN: Mon Aug 10 15:02:40 UTC 2020 + // ;; MSG SIZE rcvd: 98 + // + // We assert on the existence of the ANSWER SECTION, The consul-server IPs being present in + // the ANSWER SECTION and the DNS IP mentioned in the SERVER: field + + assert.Contains(r, logs, fmt.Sprintf("SERVER: %s", dnsIP)) + assert.Contains(r, logs, "ANSWER SECTION:") + assert.Contains(r, logs, *failover.staticServerIP) + }) +} diff --git a/acceptance/tests/sync/sync_catalog_namespaces_test.go b/acceptance/tests/sync/sync_catalog_namespaces_test.go index 7634220b6b..67123d6e4f 100644 --- a/acceptance/tests/sync/sync_catalog_namespaces_test.go +++ b/acceptance/tests/sync/sync_catalog_namespaces_test.go @@ -97,12 +97,12 @@ func TestSyncCatalogNamespaces(t *testing.T) { logger.Logf(t, "creating namespace %s", staticServerNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", staticServerNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", staticServerNamespace) }) logger.Log(t, "creating a static-server with a service") - k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, staticServerOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) diff --git a/acceptance/tests/sync/sync_catalog_test.go b/acceptance/tests/sync/sync_catalog_test.go index 2ca8b1ee1f..dc30570422 100644 --- a/acceptance/tests/sync/sync_catalog_test.go +++ b/acceptance/tests/sync/sync_catalog_test.go @@ -50,7 +50,7 @@ func TestSyncCatalog(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating a static-server with a service") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), suite.Config().NoCleanupOnFailure, suite.Config().DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), suite.Config().NoCleanupOnFailure, suite.Config().NoCleanup, suite.Config().DebugDirectory, "../fixtures/bases/static-server") consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) @@ -120,7 +120,7 @@ func TestSyncCatalogWithIngress(t *testing.T) { // endpoint fails initially. out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/ingress") require.NoError(r, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-k", "../fixtures/bases/ingress") @@ -130,7 +130,7 @@ func TestSyncCatalogWithIngress(t *testing.T) { consulCluster.Create(t) logger.Log(t, "creating a static-server with a service") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), suite.Config().NoCleanupOnFailure, suite.Config().DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), suite.Config().NoCleanupOnFailure, suite.Config().NoCleanup, suite.Config().DebugDirectory, "../fixtures/bases/static-server") consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go index 109975d731..62485c6e44 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go @@ -73,7 +73,7 @@ func TestTerminatingGatewayDestinations(t *testing.T) { // Deploy a static-server that will play the role of an external service. logger.Log(t, "creating static-server deployment") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server-https") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server-https") // If ACLs are enabled we need to update the role of the terminating gateway // with service:write permissions to the static-server service @@ -91,7 +91,7 @@ func TestTerminatingGatewayDestinations(t *testing.T) { // Deploy the static client logger.Log(t, "deploying static client") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") staticServerIP, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "get", "po", "-l", "app=static-server", `-o=jsonpath={.items[0].status.podIP}`) require.NoError(t, err) diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go index 7ad9f7da10..73250ea810 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go @@ -62,7 +62,7 @@ func TestTerminatingGatewaySingleNamespace(t *testing.T) { logger.Logf(t, "creating Kubernetes namespace %s", testNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", testNamespace) }) @@ -74,7 +74,7 @@ func TestTerminatingGatewaySingleNamespace(t *testing.T) { // Deploy a static-server that will play the role of an external service. logger.Log(t, "creating static-server deployment") - k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") // Register the external service. registerExternalService(t, consulClient, testNamespace) @@ -91,7 +91,7 @@ func TestTerminatingGatewaySingleNamespace(t *testing.T) { // Deploy the static client. logger.Log(t, "deploying static client") - k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") + k8s.DeployKustomize(t, nsK8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") // If ACLs are enabled, test that intentions prevent connections. if c.secure { @@ -159,14 +159,14 @@ func TestTerminatingGatewayNamespaceMirroring(t *testing.T) { logger.Logf(t, "creating Kubernetes namespace %s", testNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", testNamespace) }) StaticClientNamespace := "ns2" logger.Logf(t, "creating Kubernetes namespace %s", StaticClientNamespace) k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", StaticClientNamespace) }) @@ -183,7 +183,7 @@ func TestTerminatingGatewayNamespaceMirroring(t *testing.T) { // Deploy a static-server that will play the role of an external service. logger.Log(t, "creating static-server deployment") - k8s.DeployKustomize(t, ns1K8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, ns1K8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") // Register the external service registerExternalService(t, consulClient, testNamespace) @@ -200,7 +200,7 @@ func TestTerminatingGatewayNamespaceMirroring(t *testing.T) { // Deploy the static client logger.Log(t, "deploying static client") - k8s.DeployKustomize(t, ns2K8SOptions, cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") + k8s.DeployKustomize(t, ns2K8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") // If ACLs are enabled, test that intentions prevent connections. if c.secure { diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_test.go index 25792e9abb..6e7e95032f 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_test.go @@ -51,7 +51,7 @@ func TestTerminatingGateway(t *testing.T) { // Deploy a static-server that will play the role of an external service. logger.Log(t, "creating static-server deployment") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/bases/static-server") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") // Once the cluster is up, register the external service, then create the config entry. consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) @@ -71,7 +71,7 @@ func TestTerminatingGateway(t *testing.T) { // Deploy the static client logger.Log(t, "deploying static client") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") // If ACLs are enabled, test that intentions prevent connections. if c.secure { diff --git a/acceptance/tests/vault/vault_namespaces_test.go b/acceptance/tests/vault/vault_namespaces_test.go index 68fa819a84..4a9ca092d0 100644 --- a/acceptance/tests/vault/vault_namespaces_test.go +++ b/acceptance/tests/vault/vault_namespaces_test.go @@ -265,13 +265,13 @@ func TestVault_VaultNamespace(t *testing.T) { // Deploy two services and check that they can talk to each other. logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, ctx.KubectlOptions(t), "../fixtures/bases/intention") }) k8s.KubectlApplyK(t, ctx.KubectlOptions(t), "../fixtures/bases/intention") diff --git a/acceptance/tests/vault/vault_test.go b/acceptance/tests/vault/vault_test.go index 47d58b68c5..9dab0a3e71 100644 --- a/acceptance/tests/vault/vault_test.go +++ b/acceptance/tests/vault/vault_test.go @@ -350,13 +350,13 @@ func testVault(t *testing.T, testAutoBootstrap bool) { // Deploy two services and check that they can talk to each other. logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, ctx.KubectlOptions(t), "../fixtures/bases/intention") }) k8s.KubectlApplyK(t, ctx.KubectlOptions(t), "../fixtures/bases/intention") diff --git a/acceptance/tests/vault/vault_tls_auto_reload_test.go b/acceptance/tests/vault/vault_tls_auto_reload_test.go index c3a3ae4034..d5d4d33c4c 100644 --- a/acceptance/tests/vault/vault_tls_auto_reload_test.go +++ b/acceptance/tests/vault/vault_tls_auto_reload_test.go @@ -246,13 +246,13 @@ func TestVault_TLSAutoReload(t *testing.T) { // Deploy two services and check that they can talk to each other. logger.Log(t, "creating static-server and static-client deployments") - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") if cfg.EnableTransparentProxy { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-tproxy") } else { - k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") } - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, ctx.KubectlOptions(t), "../fixtures/bases/intention") }) k8s.KubectlApplyK(t, ctx.KubectlOptions(t), "../fixtures/bases/intention") diff --git a/acceptance/tests/vault/vault_wan_fed_test.go b/acceptance/tests/vault/vault_wan_fed_test.go index d8c00b732a..a2a18842b5 100644 --- a/acceptance/tests/vault/vault_wan_fed_test.go +++ b/acceptance/tests/vault/vault_wan_fed_test.go @@ -491,16 +491,17 @@ func TestVault_WANFederationViaGateways(t *testing.T) { logger.Log(t, "creating proxy-defaults config") kustomizeDir := "../fixtures/bases/mesh-gateway" k8s.KubectlApplyK(t, primaryCtx.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, primaryCtx.KubectlOptions(t), kustomizeDir) }) // Check that we can connect services over the mesh gateways. logger.Log(t, "creating static-server in dc2") - k8s.DeployKustomize(t, secondaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, secondaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating static-client in dc1") - k8s.DeployKustomize(t, primaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + k8s.DeployKustomize(t, primaryCtx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") logger.Log(t, "creating intention") _, _, err = primaryClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ diff --git a/acceptance/tests/wan-federation/wan_federation_gateway_test.go b/acceptance/tests/wan-federation/wan_federation_gateway_test.go index c87ee7197b..6abacda438 100644 --- a/acceptance/tests/wan-federation/wan_federation_gateway_test.go +++ b/acceptance/tests/wan-federation/wan_federation_gateway_test.go @@ -130,25 +130,25 @@ func TestWANFederation_Gateway(t *testing.T) { logger.Log(t, "creating proxy-defaults config in dc1") kustomizeDir := "../fixtures/cases/api-gateways/mesh" k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir) }) // these clients are just there so we can exec in and curl on them. logger.Log(t, "creating static-client in dc1") - k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") logger.Log(t, "creating static-client in dc2") - k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") t.Run("from primary to secondary", func(t *testing.T) { logger.Log(t, "creating static-server in dc2") - k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating api-gateway resources in dc1") out, err := k8s.RunKubectlAndGetOutputE(t, primaryContext.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, primaryContext.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") @@ -156,7 +156,7 @@ func TestWANFederation_Gateway(t *testing.T) { // create a service resolver for doing cross-dc redirects. k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc1-to-dc2-resolver") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc1-to-dc2-resolver") }) @@ -170,12 +170,12 @@ func TestWANFederation_Gateway(t *testing.T) { t.Run("from secondary to primary", func(t *testing.T) { // Check that we can connect services over the mesh gateways logger.Log(t, "creating static-server in dc1") - k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating api-gateway resources in dc2") out, err := k8s.RunKubectlAndGetOutputE(t, secondaryContext.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") require.NoError(t, err, out) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { // Ignore errors here because if the test ran as expected // the custom resources will have been deleted. k8s.RunKubectlAndGetOutputE(t, secondaryContext.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") @@ -183,7 +183,7 @@ func TestWANFederation_Gateway(t *testing.T) { // create a service resolver for doing cross-dc redirects. k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc2-to-dc1-resolver") - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc2-to-dc1-resolver") }) diff --git a/acceptance/tests/wan-federation/wan_federation_test.go b/acceptance/tests/wan-federation/wan_federation_test.go index 5d6200721c..8edc1f5d03 100644 --- a/acceptance/tests/wan-federation/wan_federation_test.go +++ b/acceptance/tests/wan-federation/wan_federation_test.go @@ -154,7 +154,7 @@ func TestWANFederation(t *testing.T) { logger.Log(t, "creating proxy-defaults config") kustomizeDir := "../fixtures/bases/mesh-gateway" k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), kustomizeDir) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), kustomizeDir) }) @@ -184,10 +184,10 @@ func TestWANFederation(t *testing.T) { // Check that we can connect services over the mesh gateways logger.Log(t, "creating static-server in dc2") - k8s.DeployKustomize(t, secondaryHelper.KubectlOptsForApp(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, secondaryHelper.KubectlOptsForApp(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") logger.Log(t, "creating static-client in dc1") - k8s.DeployKustomize(t, primaryHelper.KubectlOptsForApp(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + k8s.DeployKustomize(t, primaryHelper.KubectlOptsForApp(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") if c.secure { primaryHelper.CreateIntention(t)