From 8a67ea930f0fa345338c7a6ac89a7d78a8d653a7 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 3 Jun 2025 09:07:31 +0000 Subject: [PATCH 1/6] HDDS-13169. Intermittent failure in testSnapshotOperationsNotBlockedDuringCompaction --- .../apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java index 195bcd105a92..947d14e1b8b2 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotCache.java @@ -384,6 +384,7 @@ void testSnapshotOperationsNotBlockedDuringCompaction() throws IOException, Inte final Semaphore compactionLock = new Semaphore(1); final AtomicBoolean table1Compacting = new AtomicBoolean(false); final AtomicBoolean table1CompactedFinish = new AtomicBoolean(false); + final AtomicBoolean table2CompactedFinish = new AtomicBoolean(false); org.apache.hadoop.hdds.utils.db.DBStore store1 = snapshot1.get().getMetadataManager().getStore(); doAnswer(invocation -> { table1Compacting.set(true); @@ -392,6 +393,10 @@ void testSnapshotOperationsNotBlockedDuringCompaction() throws IOException, Inte table1CompactedFinish.set(true); return null; }).when(store1).compactTable("table1"); + doAnswer(invocation -> { + table2CompactedFinish.set(true); + return null; + }).when(store1).compactTable("table2"); compactionLock.acquire(); final UUID dbKey2 = UUID.randomUUID(); @@ -411,6 +416,7 @@ void testSnapshotOperationsNotBlockedDuringCompaction() throws IOException, Inte assertFalse(table1CompactedFinish.get()); compactionLock.release(); GenericTestUtils.waitFor(() -> table1CompactedFinish.get(), 50, 3000); + GenericTestUtils.waitFor(() -> table2CompactedFinish.get(), 50, 3000); verify(store1, times(1)).compactTable("table1"); verify(store1, times(1)).compactTable("table2"); From ee9c260addb4c044516229cab5ca6322f729abd5 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 3 Jun 2025 09:45:36 +0000 Subject: [PATCH 2/6] Modify junit.sh to run 'clean verify' for subsequent iterations in test execution --- hadoop-ozone/dev-support/checks/junit.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh index 7c25293c5d74..6f360359c9ab 100755 --- a/hadoop-ozone/dev-support/checks/junit.sh +++ b/hadoop-ozone/dev-support/checks/junit.sh @@ -68,8 +68,13 @@ for i in $(seq 1 ${ITERATIONS}); do mkdir -p "${REPORT_DIR}" fi - mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" verify \ - | tee "${REPORT_DIR}/output.log" + if [[ $i -eq 1 ]]; then + mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" verify \ + | tee "${REPORT_DIR}/output.log" + else + mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" clean verify \ + | tee "${REPORT_DIR}/output.log" + fi irc=$? # shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh From 0cf95b351802eb46d518d0fc884fd658ef9d6724 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 3 Jun 2025 16:38:02 +0000 Subject: [PATCH 3/6] Update intermittent-test-check.yml and junit.sh to use TARGET_DIR for output paths --- .github/workflows/intermittent-test-check.yml | 9 +++++---- hadoop-ozone/dev-support/checks/junit.sh | 10 +++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 6ad316e797fa..2b1f87a9352c 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -65,6 +65,7 @@ env: RATIS_REPO: ${{ github.event.inputs.ratis-repo }} RATIS_VERSION: ${{ github.event.inputs.ratis-ref }} JAVA_VERSION: ${{ github.event.inputs.java-version }} + TARGET_DIR: flaky-test-target # Surefire 3.0.0-M4 is used because newer versions do not reliably kill the fork on timeout # SUREFIRE-1722, SUREFIRE-1815 SUREFIRE_VERSION: 3.0.0-M4 @@ -131,8 +132,8 @@ jobs: - name: Find modules id: modules run: | - grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' target/unit/output.log | grep -B1 'Running org.apache' - modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' target/unit/output.log | grep -B1 'Running org.apache' \ + grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.TARGET_DIR }}/unit/output.log | grep -B1 'Running org.apache' + modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.TARGET_DIR }}/unit/output.log | grep -B1 'Running org.apache' \ | grep surefire | cut -f2 -d'@' | awk '{ print $1 }' | sed 's/^/:/' | xargs | sed -e 's/ /,/g') echo "modules=$modules" >> $GITHUB_OUTPUT if: ${{ !cancelled() }} @@ -275,14 +276,14 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh target/unit/summary.txt + run: hadoop-ozone/dev-support/checks/_summary.sh ${{ env.TARGET_DIR }}/unit/summary.txt if: ${{ !cancelled() }} - name: Archive build results uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: result-${{ github.run_number }}-${{ github.run_id }}-split-${{ matrix.split }} - path: target/unit + path: ${{ env.TARGET_DIR }}/unit count-failures: if: ${{ failure() }} needs: run-test diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh index 6f360359c9ab..5b79ae72f2e1 100755 --- a/hadoop-ozone/dev-support/checks/junit.sh +++ b/hadoop-ozone/dev-support/checks/junit.sh @@ -24,6 +24,7 @@ cd "$DIR/../../.." || exit 1 : ${ITERATIONS:="1"} : ${OZONE_WITH_COVERAGE:="false"} : ${OZONE_REPO_CACHED:="false"} +: ${TARGET_DIR:="target"} declare -i ITERATIONS if [[ ${ITERATIONS} -le 0 ]]; then @@ -56,7 +57,7 @@ if [[ ${ITERATIONS} -gt 1 ]] && [[ ${OZONE_REPO_CACHED} == "false" ]]; then mvn ${MAVEN_OPTIONS} -DskipTests install fi -REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/${CHECK}"} +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../${TARGET_DIR}/${CHECK}"} REPORT_FILE="${REPORT_DIR}/summary.txt" mkdir -p "$REPORT_DIR" @@ -68,13 +69,8 @@ for i in $(seq 1 ${ITERATIONS}); do mkdir -p "${REPORT_DIR}" fi - if [[ $i -eq 1 ]]; then - mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" verify \ + mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" clean verify \ | tee "${REPORT_DIR}/output.log" - else - mvn ${MAVEN_OPTIONS} -Dmaven-surefire-plugin.argLineAccessArgs="${OZONE_MODULE_ACCESS_ARGS}" "$@" clean verify \ - | tee "${REPORT_DIR}/output.log" - fi irc=$? # shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh From 2373cb9a1787d6653665f82655effe6a9d96b5e2 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 4 Jun 2025 10:36:07 +0000 Subject: [PATCH 4/6] Refactor intermittent-test-check.yml and junit.sh to replace TARGET_DIR with OUTPUT_DIR for consistency in output paths --- .github/workflows/intermittent-test-check.yml | 10 +++++----- hadoop-ozone/dev-support/checks/junit.sh | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 2b1f87a9352c..d3a252659aea 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -65,7 +65,7 @@ env: RATIS_REPO: ${{ github.event.inputs.ratis-repo }} RATIS_VERSION: ${{ github.event.inputs.ratis-ref }} JAVA_VERSION: ${{ github.event.inputs.java-version }} - TARGET_DIR: flaky-test-target + OUTPUT_DIR: flaky-test-check-ci-results # Surefire 3.0.0-M4 is used because newer versions do not reliably kill the fork on timeout # SUREFIRE-1722, SUREFIRE-1815 SUREFIRE_VERSION: 3.0.0-M4 @@ -132,8 +132,8 @@ jobs: - name: Find modules id: modules run: | - grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.TARGET_DIR }}/unit/output.log | grep -B1 'Running org.apache' - modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.TARGET_DIR }}/unit/output.log | grep -B1 'Running org.apache' \ + grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/unit/output.log | grep -B1 'Running org.apache' + modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/unit/output.log | grep -B1 'Running org.apache' \ | grep surefire | cut -f2 -d'@' | awk '{ print $1 }' | sed 's/^/:/' | xargs | sed -e 's/ /,/g') echo "modules=$modules" >> $GITHUB_OUTPUT if: ${{ !cancelled() }} @@ -276,14 +276,14 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh ${{ env.TARGET_DIR }}/unit/summary.txt + run: hadoop-ozone/dev-support/checks/_summary.sh ${{ env.OUTPUT_DIR }}/unit/summary.txt if: ${{ !cancelled() }} - name: Archive build results uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: result-${{ github.run_number }}-${{ github.run_id }}-split-${{ matrix.split }} - path: ${{ env.TARGET_DIR }}/unit + path: ${{ env.OUTPUT_DIR }}/unit count-failures: if: ${{ failure() }} needs: run-test diff --git a/hadoop-ozone/dev-support/checks/junit.sh b/hadoop-ozone/dev-support/checks/junit.sh index 5b79ae72f2e1..ce93510f1f3e 100755 --- a/hadoop-ozone/dev-support/checks/junit.sh +++ b/hadoop-ozone/dev-support/checks/junit.sh @@ -24,7 +24,6 @@ cd "$DIR/../../.." || exit 1 : ${ITERATIONS:="1"} : ${OZONE_WITH_COVERAGE:="false"} : ${OZONE_REPO_CACHED:="false"} -: ${TARGET_DIR:="target"} declare -i ITERATIONS if [[ ${ITERATIONS} -le 0 ]]; then @@ -57,7 +56,7 @@ if [[ ${ITERATIONS} -gt 1 ]] && [[ ${OZONE_REPO_CACHED} == "false" ]]; then mvn ${MAVEN_OPTIONS} -DskipTests install fi -REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../${TARGET_DIR}/${CHECK}"} +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/${CHECK}"} REPORT_FILE="${REPORT_DIR}/summary.txt" mkdir -p "$REPORT_DIR" From df64f509bb213c338af8587727c0265de28c7793 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 4 Jun 2025 17:03:35 +0000 Subject: [PATCH 5/6] Correct output log paths in intermittent-test-check.yml --- .github/workflows/intermittent-test-check.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index d3a252659aea..181758639cc7 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -132,8 +132,8 @@ jobs: - name: Find modules id: modules run: | - grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/unit/output.log | grep -B1 'Running org.apache' - modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/unit/output.log | grep -B1 'Running org.apache' \ + grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/output.log | grep -B1 'Running org.apache' + modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/output.log | grep -B1 'Running org.apache' \ | grep surefire | cut -f2 -d'@' | awk '{ print $1 }' | sed 's/^/:/' | xargs | sed -e 's/ /,/g') echo "modules=$modules" >> $GITHUB_OUTPUT if: ${{ !cancelled() }} @@ -276,14 +276,14 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures - run: hadoop-ozone/dev-support/checks/_summary.sh ${{ env.OUTPUT_DIR }}/unit/summary.txt + run: hadoop-ozone/dev-support/checks/_summary.sh ${{ env.OUTPUT_DIR }}/summary.txt if: ${{ !cancelled() }} - name: Archive build results uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: result-${{ github.run_number }}-${{ github.run_id }}-split-${{ matrix.split }} - path: ${{ env.OUTPUT_DIR }}/unit + path: ${{ env.OUTPUT_DIR }} count-failures: if: ${{ failure() }} needs: run-test From 028f77e7eee0be1b02b9917ba973985706c7afdd Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 4 Jun 2025 17:45:32 +0000 Subject: [PATCH 6/6] ensure text processing for grep commands in intermittent-test-check.yml --- .github/workflows/intermittent-test-check.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/intermittent-test-check.yml b/.github/workflows/intermittent-test-check.yml index 181758639cc7..296b0e0c50b1 100644 --- a/.github/workflows/intermittent-test-check.yml +++ b/.github/workflows/intermittent-test-check.yml @@ -132,9 +132,9 @@ jobs: - name: Find modules id: modules run: | - grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/output.log | grep -B1 'Running org.apache' - modules=$(grep -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/output.log | grep -B1 'Running org.apache' \ - | grep surefire | cut -f2 -d'@' | awk '{ print $1 }' | sed 's/^/:/' | xargs | sed -e 's/ /,/g') + grep --text -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/output.log | grep --text -B1 'Running org.apache' + modules=$(grep --text -e 'surefire:${{ env.SUREFIRE_VERSION }}:test' -e 'Running org.apache' ${{ env.OUTPUT_DIR }}/output.log | grep --text -B1 'Running org.apache' \ + | grep --text surefire | cut -f2 -d'@' | awk '{ print $1 }' | sed 's/^/:/' | xargs | sed -e 's/ /,/g') echo "modules=$modules" >> $GITHUB_OUTPUT if: ${{ !cancelled() }} build: @@ -293,12 +293,12 @@ jobs: uses: actions/download-artifact@v4 - name: Count failures run: | - failures=$(find . -name 'summary.txt' | grep -v 'iteration' | xargs grep -v 'exit code: 0' | wc -l) + failures=$(find . -name 'summary.txt' | grep --text -v 'iteration' | xargs grep --text -v 'exit code: 0' | wc -l) echo "Total failures: $failures" if [[ $failures -gt 0 ]]; then echo "" echo "Failed runs:" - grep 'exit code: 1' */summary.txt | grep -o 'split.*teration [0-9]*' | sed -e 's/.summary.txt:/ /' -e 's/-/ /' | sort -g -k2 -k4 + grep --text 'exit code: 1' */summary.txt | grep --text -o 'split.*teration [0-9]*' | sed -e 's/.summary.txt:/ /' -e 's/-/ /' | sort -g -k2 -k4 echo "" exit 1 fi