RATIS-1961. Create workflow for running tests repeatedly (#985)

.github/workflows/repeat-test.yml

@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: repeat-test
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Git Ref (branch, tag or commit SHA)
+        default: master
+        required: true
+      test-class:
+        description: Test Class
+        required: true
+      test-method:
+        description: Test Method
+        default: ''
+        required: false
+      splits:
+        description: Number of splits
+        default: 10
+        required: true
+      iterations:
+        description: Number of iterations per split
+        default: 10
+        required: true
+      fail-fast:
+        description: Stop after first failure
+        default: false
+        required: true
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  TEST_CLASS: ${{ github.event.inputs.test-class }}
+  TEST_METHOD: ${{ github.event.inputs.test-method }}
+  ITERATIONS: ${{ github.event.inputs.iterations }}
+  FAIL_FAST: ${{ github.event.inputs.fail-fast }}
+run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}#{1}[{2}]-{3}x{4}', inputs.test-class, inputs.test-method, inputs.ref, inputs.splits, inputs.iterations) || '' }}
+jobs:
+  prepare:
+    runs-on: ubuntu-20.04
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+      test-spec: ${{ steps.test-spec.outputs.test-spec }}
+    steps:
+      - id: generate
+        name: Generate test matrix
+        run: |
+          splits=()
+          for ((i = 1; i <= ${{ github.event.inputs.splits }}; i++)); do
+              splits+=("$i")
+          done
+          printf -v x "%s," "${splits[@]}"
+          split_matrix="[${x%,}]"
+          echo "matrix=$split_matrix" >> $GITHUB_OUTPUT
+      - name: Define test spec
+        id: test-spec
+        run: |
+          if [[ -z "$TEST_METHOD" ]]; then
+            test_spec="$TEST_CLASS"
+          else
+            test_spec="$TEST_CLASS#$TEST_METHOD"
+          fi
+          echo "Test to be run: $test_spec"
+          echo "test-spec=$test_spec" >> $GITHUB_OUTPUT
+  test:
+    if: ${{ always() }}
+    needs:
+      - prepare
+    runs-on: ubuntu-20.04
+    env:
+      TEST_SPEC: ${{ needs.prepare.outputs.test-spec }}
+    strategy:
+      matrix:
+        split: ${{ fromJson(needs.prepare.outputs.matrix) }}
+      fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - name: Cache for maven dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ratis
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Validate test spec
+        run: if [[ -z "$TEST_SPEC" ]]; then exit 1; fi
+      - name: Execute ${{ env.TEST_SPEC }}
+        run: dev-support/checks/unit.sh -Dtest="$TEST_SPEC"
+        continue-on-error: true
+      - name: Summary of failures
+        run: dev-support/checks/_summary.sh target/unit/summary.txt
+        if: ${{ !cancelled() }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: result-${{ env.TEST_CLASS }}-split-${{ matrix.split }}
+          path: target/unit
+  count-failures:
+    if: ${{ always() }}
+    needs: test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Download build results
+        uses: actions/download-artifact@v3
+      - name: Count failures
+        run: |
+          failures=$(find . -name 'summary.txt' | grep -v 'iteration' | xargs grep -v 'exit code: 0' | wc -l)
+          echo "Total failures: $failures"
+          if [[ $failures -gt 0 ]]; then
+            echo ""
+            echo "Failed runs:"
+            grep 'exit code: 1' */summary.txt | grep -o 'split.*teration [0-9]*' | sed -e 's/.summary.txt:/ /' -e 's/-/ /' | sort -g -k2 -k4
+            echo ""
+            exit 1
+          fi

dev-support/checks/_summary.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REPORT_FILE="$1"
+
+: ${ITERATIONS:="1"}
+
+declare -i ITERATIONS
+
+rc=0
+
+if [[ ! -e "${REPORT_FILE}" ]]; then
+  echo "Report file missing, check logs for details"
+  rc=255
+
+elif [[ ${ITERATIONS} -gt 1 ]]; then
+  cat "${REPORT_FILE}"
+
+  if grep -q 'exit code: [^0]' "${REPORT_FILE}"; then
+    rc=1
+  fi
+
+elif [[ -s "${REPORT_FILE}" ]]; then
+  cat "${REPORT_FILE}"
+  rc=1
+fi
+
+exit ${rc}

dev-support/checks/unit.sh

@@ -21,6 +21,7 @@ cd "$DIR/../.." || exit 1
 
 source "${DIR}/../find_maven.sh"
 
+: ${FAIL_FAST:="false"}
 : ${ITERATIONS:="1"}
 
 declare -i ITERATIONS
@@ -32,6 +33,13 @@ REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../target/unit"}
 mkdir -p "$REPORT_DIR"
 
 export MAVEN_OPTS="-Xmx4096m"
+MAVEN_OPTIONS='-B --no-transfer-progress'
+
+if [[ "${FAIL_FAST}" == "true" ]]; then
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast -Dsurefire.skipAfterFailureCount=1"
+else
+  MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-at-end"
+fi
 
 rc=0
 for i in $(seq 1 ${ITERATIONS}); do
@@ -41,7 +49,7 @@ for i in $(seq 1 ${ITERATIONS}); do
     mkdir -p "${REPORT_DIR}"
   fi
 
-  ${MVN} -B -fae --no-transfer-progress test "$@" \
+  ${MVN} ${MAVEN_OPTIONS} test "$@" \
     | tee "${REPORT_DIR}/output.log"
   irc=$?
 
@@ -52,13 +60,21 @@ for i in $(seq 1 ${ITERATIONS}); do
   fi
 
   if [[ ${ITERATIONS} -gt 1 ]]; then
+    if [[ ${irc} == 0 ]]; then
+      rm -fr "${REPORT_DIR}"
+    fi
+
     REPORT_DIR="${original_report_dir}"
     echo "Iteration ${i} exit code: ${irc}" | tee -a "${REPORT_DIR}/summary.txt"
   fi
 
   if [[ ${rc} == 0 ]]; then
     rc=${irc}
   fi
+
+  if [[ ${rc} != 0 ]] && [[ "${FAIL_FAST}" == "true" ]]; then
+    break
+  fi
 done
 
 exit ${rc}