From 7deca60ce8257d4e06eb8fc0f715c6ba4ea7962c Mon Sep 17 00:00:00 2001 From: Anita Akaeze Date: Wed, 22 Feb 2023 22:16:13 +0000 Subject: [PATCH] backport of commit e17298b6b82dd1a19e763d1964854f8c87cd9e60 --- .../libs/cluster/container.go | 2 + .../resolver_subset_redirect_test.go | 230 ++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go index bd4416a35bf..5aa3f023eeb 100644 --- a/test/integration/consul-container/libs/cluster/container.go +++ b/test/integration/consul-container/libs/cluster/container.go @@ -518,10 +518,12 @@ func newContainerRequest(config Config, opts containerOpts) (podRequest, consulR "8079/tcp", // Envoy App Listener - grpc port used by static-server "8078/tcp", // Envoy App Listener - grpc port used by static-server-v1 "8077/tcp", // Envoy App Listener - grpc port used by static-server-v2 + "8076/tcp", // Envoy App Listener - grpc port used by static-server-v3 "8080/tcp", // Envoy App Listener - http port used by static-server "8081/tcp", // Envoy App Listener - http port used by static-server-v1 "8082/tcp", // Envoy App Listener - http port used by static-server-v2 + "8083/tcp", // Envoy App Listener - http port used by static-server-v3 "9998/tcp", // Envoy App Listener "9999/tcp", // Envoy App Listener }, diff --git a/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go new file mode 100644 index 00000000000..3ba808057ac --- /dev/null +++ b/test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_subset_redirect_test.go @@ -0,0 +1,230 @@ +package upgrade + +import ( + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/hashicorp/consul/api" + libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert" + libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster" + libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service" + "github.com/hashicorp/consul/test/integration/consul-container/libs/topology" + "github.com/hashicorp/consul/test/integration/consul-container/libs/utils" + "github.com/hashicorp/go-version" + "github.com/stretchr/testify/require" + "gotest.tools/assert" +) + +// TestTrafficManagement_ServiceResolverSubsetRedirect Summary +// This test starts up 4 servers and 1 client in the same datacenter. +// +// Steps: +// - Create a single agent cluster. +// - Create 2 static-servers, 2 subset servers and 1 client and sidecars for all services, then register them with Consul +// - Validate traffic is successfully redirected from server 1 to sever2-v2 as defined in the service resolver +func TestTrafficManagement_ServiceResolverSubsetRedirect(t *testing.T) { + t.Parallel() + + type testcase struct { + oldversion string + targetVersion string + } + tcs := []testcase{ + { + oldversion: "1.13", + targetVersion: utils.TargetVersion, + }, + { + oldversion: "1.14", + targetVersion: utils.TargetVersion, + }, + } + + run := func(t *testing.T, tc testcase) { + buildOpts := &libcluster.BuildOptions{ + ConsulVersion: tc.oldversion, + Datacenter: "dc1", + InjectAutoEncryption: true, + } + // If version < 1.14 disable AutoEncryption + oldVersion, _ := version.NewVersion(tc.oldversion) + if oldVersion.LessThan(utils.Version_1_14) { + buildOpts.InjectAutoEncryption = false + } + cluster, _, _ := topology.NewPeeringCluster(t, 1, buildOpts) + node := cluster.Agents[0] + + // Register static-server service resolver + serviceResolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServer2ServiceName, + Subsets: map[string]api.ServiceResolverSubset{ + "v1": { + Filter: "Service.Meta.version == v1", + }, + "v2": { + Filter: "Service.Meta.version == v2", + }, + }, + } + err := cluster.ConfigEntryWrite(serviceResolver) + require.NoError(t, err) + + // Register static-server-2 service resolver to redirect traffic + // from static-server to static-server-2-v2 + service2Resolver := &api.ServiceResolverConfigEntry{ + Kind: api.ServiceResolver, + Name: libservice.StaticServerServiceName, + Redirect: &api.ServiceResolverRedirect{ + Service: libservice.StaticServer2ServiceName, + ServiceSubset: "v2", + }, + } + err = cluster.ConfigEntryWrite(service2Resolver) + require.NoError(t, err) + + // register agent services + agentServices := setupServiceAndSubsets(t, cluster) + assertionFn, proxyRestartFn := agentServices.validateAgentServices(t) + _, port := agentServices.client.GetAddr() + _, adminPort := agentServices.client.GetAdminAddr() + + // validate static-client is up and running + libassert.AssertContainerState(t, agentServices.client, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-2-v2") + assertionFn() + + // Upgrade cluster, restart sidecars then begin service traffic validation + require.NoError(t, cluster.StandardUpgrade(t, context.Background(), tc.targetVersion)) + proxyRestartFn() + + libassert.AssertContainerState(t, agentServices.client, "running") + libassert.HTTPServiceEchoes(t, "localhost", port, "") + libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server-2-v2") + assertionFn() + + // assert 3 static-server instances are healthy + libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServer2ServiceName, false, 3) + libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server-2.default", "HEALTHY", 1) + } + + for _, tc := range tcs { + t.Run(fmt.Sprintf("upgrade from %s to %s", tc.oldversion, tc.targetVersion), + func(t *testing.T) { + run(t, tc) + }) + // test sometimes fails with error: could not start or join all agents: could not add container index 0: port not found + time.Sleep(1 * time.Second) + } +} + +func (s *registeredServices) validateAgentServices(t *testing.T) (func(), func()) { + var ( + responseFormat = map[string]string{"format": "json"} + servicePort = make(map[string]int) + proxyRestartFn func() + assertionFn func() + ) + + for serviceName, proxies := range s.services { + for _, proxy := range proxies { + _, adminPort := proxy.GetAdminAddr() + servicePort[serviceName] = adminPort + } + } + + assertionFn = func() { + // validate services proxy admin is up + for serviceName, adminPort := range servicePort { + _, statusCode, err := libassert.GetEnvoyOutput(adminPort, "stats", responseFormat) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, statusCode, fmt.Sprintf("%s cannot be reached %v", serviceName, statusCode)) + + // certs are valid + libassert.AssertEnvoyPresentsCertURI(t, adminPort, serviceName) + } + } + + for _, serviceConnectProxy := range s.services { + for _, proxy := range serviceConnectProxy { + proxyRestartFn = func() { require.NoErrorf(t, proxy.Restart(), "%s", proxy.GetName()) } + } + } + return assertionFn, proxyRestartFn +} + +type registeredServices struct { + client libservice.Service + services map[string][]libservice.Service +} + +// create 3 servers and 1 client +func setupServiceAndSubsets(t *testing.T, cluster *libcluster.Cluster) *registeredServices { + node := cluster.Agents[0] + client := node.GetClient() + + // create static-servers and subsets + serviceOpts := &libservice.ServiceOpts{ + Name: libservice.StaticServerServiceName, + ID: "static-server", + HTTPPort: 8080, + GRPCPort: 8079, + } + _, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName) + + serviceOpts2 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2", + HTTPPort: 8081, + GRPCPort: 8078, + } + _, server2ConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + serviceOptsV1 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2-v1", + Meta: map[string]string{"version": "v1"}, + HTTPPort: 8082, + GRPCPort: 8077, + } + _, server2ConnectProxyV1, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV1) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + serviceOptsV2 := &libservice.ServiceOpts{ + Name: libservice.StaticServer2ServiceName, + ID: "static-server-2-v2", + Meta: map[string]string{"version": "v2"}, + HTTPPort: 8083, + GRPCPort: 8076, + } + _, server2ConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName) + + // Create a client proxy instance with the server as an upstream + clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false) + require.NoError(t, err) + libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName)) + + // return a map of all services created + tmpServices := map[string][]libservice.Service{} + tmpServices[libservice.StaticClientServiceName] = append(tmpServices[libservice.StaticClientServiceName], clientConnectProxy) + tmpServices[libservice.StaticServerServiceName] = append(tmpServices[libservice.StaticServerServiceName], serverConnectProxy) + tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxy) + tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxyV1) + tmpServices[libservice.StaticServer2ServiceName] = append(tmpServices[libservice.StaticServer2ServiceName], server2ConnectProxyV2) + + return ®isteredServices{ + client: clientConnectProxy, + services: tmpServices, + } +}