[Release/1.14.x] net 3914 gha consul container test no splitting

.github/workflows/reusable-lint.yml

@@ -48,7 +48,6 @@ jobs:
           working-directory: ${{ matrix.directory }}
           version: v1.51.1
           args: --build-tags="${{ env.GOTAGS }}" -v
-          skip-cache: true
       - name: Notify Slack
         if: ${{ failure() }}
         run: .github/scripts/notify_slack.sh

.github/workflows/test-integrations.yml

@@ -343,50 +343,14 @@ jobs:
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
-  generate-compatibility-job-matrices:
-    needs: [setup]
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
-    name: Generate Compatibility Job Matrices
-    outputs:
-      compatibility-matrix: ${{ steps.set-matrix.outputs.compatibility-matrix }}
-    steps:
-      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
-      - name: Generate Compatibility Job Matrix
-        id: set-matrix
-        env:
-          TOTAL_RUNNERS: 6
-          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
-        run: |
-          cd ./test/integration/consul-container
-          NUM_RUNNERS=$TOTAL_RUNNERS
-          NUM_DIRS=$(find ./test -mindepth 1 -maxdepth 2 -type d | wc -l)
-
-          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
-            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
-            NUM_RUNNERS=$((NUM_DIRS-1))
-          fi
-          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
-          NUM_RUNNERS=$((NUM_RUNNERS-1))
-          {
-            echo -n "compatibility-matrix="
-            find ./test -maxdepth 2 -type d -print0 | xargs -0 -n 1 \
-              | grep -v util | grep -v upgrade \
-              | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
-              | jq --compact-output 'map(join(" "))'
-          } >> "$GITHUB_OUTPUT"
   compatibility-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
     needs:
       - setup
       - dev-build
-      - generate-compatibility-job-matrices
     permissions:
       id-token: write # NOTE: this permission is explicitly required for Vault auth.
       contents: read
-    strategy:
-      fail-fast: false
-      matrix:
-        test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -416,12 +380,9 @@ jobs:
             mkdir -p "/tmp/test-results"
             cd ./test/integration/consul-container
             docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
-            echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
-            # shellcheck disable=SC2001
-            sed 's, ,\n,g' <<< "${{ matrix.test-cases }}"
             go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
               --raw-command \
-              --format=short-verbose \
+              --format=standard-verbose \
               --debug \
               --rerun-fails=3 \
               -- \
@@ -430,7 +391,7 @@ jobs:
               -tags "${{ env.GOTAGS }}" \
               -timeout=30m \
               -json \
-              ${{ matrix.test-cases }} \
+              `go list ./... | grep -v upgrade` \
               --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
               --target-version local \
               --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
@@ -474,39 +435,6 @@ jobs:
           DD_ENV: ci
         run: datadog-ci junit upload --service "$GITHUB_REPOSITORY" $TEST_RESULTS_DIR/results.xml
 
-  generate-upgrade-job-matrices:
-    needs: [setup]
-    runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
-    name: Generate Upgrade Job Matrices
-    outputs:
-      upgrade-matrix: ${{ steps.set-matrix.outputs.upgrade-matrix }}
-    steps:
-      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
-      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
-        with:
-          go-version-file: 'go.mod'
-      - name: Generate Updgrade Job Matrix
-        id: set-matrix
-        env:
-          TOTAL_RUNNERS: 5
-          JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
-        run: |
-          cd ./test/integration/consul-container/test/upgrade
-          NUM_RUNNERS=$TOTAL_RUNNERS
-          NUM_DIRS=$(go test ./...  -list=. -json | jq -r '.Output | select (. !=null) |  select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' | wc -l)
-
-          if [ "$NUM_DIRS" -lt "$NUM_RUNNERS" ]; then
-            echo "TOTAL_RUNNERS is larger than the number of tests/packages to split."
-            NUM_RUNNERS=$((NUM_DIRS-1))
-          fi
-          # fix issue where test splitting calculation generates 1 more split than TOTAL_RUNNERS.
-          NUM_RUNNERS=$((NUM_RUNNERS-1))
-          {
-            echo -n "upgrade-matrix="
-            go test ./...  -list=. -json | jq -r '.Output | select (. !=null) |  select(. | startswith("Test")) | gsub("[\\n\\t]"; "")' \
-            | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
-            | jq --compact-output 'map(join("|"))'
-          } >> "$GITHUB_OUTPUT"
 
   upgrade-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
@@ -516,14 +444,12 @@ jobs:
     needs:
       - setup
       - dev-build
-      - generate-upgrade-job-matrices
     strategy:
       fail-fast: false
       matrix:
-        consul-version: [ "1.14", "1.15"]
-        test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }}
+        consul-version: ["1.14"]
     env:
-      CONSUL_VERSION: ${{ matrix.consul-version }}
+      CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -579,9 +505,6 @@ jobs:
           mkdir -p "${{ env.TEST_RESULTS_DIR }}"
           cd ./test/integration/consul-container/test/upgrade
           docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
-          echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
-          # shellcheck disable=SC2001
-          sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
           go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
             --raw-command \
             --format=short-verbose \
@@ -594,11 +517,10 @@ jobs:
             -tags "${{ env.GOTAGS }}" \
             -timeout=30m \
             -json ./... \
-            -run "${{ matrix.test-cases }}" \
             --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
             --target-version local \
             --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --latest-version "${{ env.CONSUL_VERSION }}"
+            --latest-version "${{ env.CONSUL_LATEST_VERSION }}"
           ls -lrt
         env:
           # this is needed because of incompatibility between RYUK container and GHA
@@ -620,9 +542,7 @@ jobs:
     - vault-integration-test
     - generate-envoy-job-matrices
     - envoy-integration-test
-    - generate-compatibility-job-matrices
     - compatibility-integration-test
-    - generate-upgrade-job-matrices
     - upgrade-integration-test
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
     if: ${{ always() }}

test/integration/consul-container/libs/cluster/cluster.go

@@ -213,7 +213,7 @@ func (c *Cluster) Clients() ([]libagent.Agent, error) {
 	return clients, nil
 }
 
-const retryTimeout = 20 * time.Second
+const retryTimeout = 90 * time.Second
 const retryFrequency = 500 * time.Millisecond
 
 func LongFailer() *retry.Timer {
@@ -250,6 +250,6 @@ func WaitForMembers(t *testing.T, client *api.Client, expectN int) {
 			}
 		}
 		require.NoError(r, err)
-		require.Equal(r, activeMembers, expectN)
+		require.Equal(r, expectN, activeMembers)
 	})
 }

test/integration/consul-container/libs/service/connect.go

@@ -85,7 +85,7 @@ func NewConnectService(ctx context.Context, name string, serviceName string, ser
 
 	req := testcontainers.ContainerRequest{
 		FromDockerfile: dockerfileCtx,
-		WaitingFor:     wait.ForLog("").WithStartupTimeout(10 * time.Second),
+		WaitingFor:     wait.ForLog("").WithStartupTimeout(100 * time.Second),
 		AutoRemove:     false,
 		Name:           containerName,
 		Cmd: []string{

test/integration/consul-container/libs/service/examples.go

@@ -66,7 +66,7 @@ func NewExampleService(ctx context.Context, name string, httpPort int, grpcPort
 
 	req := testcontainers.ContainerRequest{
 		Image:      hashicorpDockerProxy + "/fortio/fortio",
-		WaitingFor: wait.ForLog("").WithStartupTimeout(10 * time.Second),
+		WaitingFor: wait.ForLog("").WithStartupTimeout(100 * time.Second),
 		AutoRemove: false,
 		Name:       containerName,
 		Cmd:        []string{"server", "-http-port", fmt.Sprintf("%d", httpPort), "-grpc-port", fmt.Sprintf("%d", grpcPort), "-redirect-port", "-disabled"},

test/integration/consul-container/libs/service/gateway.go

@@ -79,7 +79,7 @@ func NewGatewayService(ctx context.Context, name string, kind string, node libno
 
 	req := testcontainers.ContainerRequest{
 		FromDockerfile: dockerfileCtx,
-		WaitingFor:     wait.ForLog("").WithStartupTimeout(10 * time.Second),
+		WaitingFor:     wait.ForLog("").WithStartupTimeout(100 * time.Second),
 		AutoRemove:     false,
 		Name:           containerName,
 		Cmd: []string{

test/integration/consul-container/test/upgrade/healthcheck_test.go

@@ -22,7 +22,7 @@ func TestTargetServersWithLatestGAClients(t *testing.T) {
 		numClients = 1
 	)
 
-	cluster := serversCluster(t, numServers, utils.TargetVersion, utils.TargetImage)
+	cluster := serversCluster(t, numServers, utils.TargetImage, utils.TargetVersion)
 	defer terminate(t, cluster)
 
 	clients := clientsCreate(t, numClients, utils.LatestImage, utils.LatestVersion, cluster)
@@ -268,13 +268,15 @@ func serviceCreate(t *testing.T, client *api.Client, serviceName string) uint64
 	return meta.LastIndex
 }
 
-func serversCluster(t *testing.T, numServers int, version string, image string) *libcluster.Cluster {
+func serversCluster(t *testing.T, numServers int, image string, version string) *libcluster.Cluster {
 	var configs []libagent.Config
 
 	conf, err := libagent.NewConfigBuilder(nil).
 		Bootstrap(3).
 		ToAgentConfig()
 	require.NoError(t, err)
+	conf.Image = image
+	conf.Version = version
 
 	for i := 0; i < numServers; i++ {
 		configs = append(configs, *conf)