From bb23d306146b0d90208d89464c780dab5cb7cae5 Mon Sep 17 00:00:00 2001 From: fl4via Date: Mon, 28 Jul 2025 12:03:25 -0300 Subject: [PATCH 01/11] [UNDERTOW-2587] At GitHub CI, add a timeout to the tests and take a thread dump in case the timeout expires Signed-off-by: fl4via --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfcc9a84ad..362c938c20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,8 @@ jobs: - name: Print Version run: mvn -v - name: Run Tests - run: mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} + run: | + timeout 5400s mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} || (jstack -l $(pgrep -f 'java') > thread-dump.txt && exit 1) - uses: actions/upload-artifact@v4 if: failure() with: @@ -156,7 +157,8 @@ jobs: - name: Print Version run: mvn -v - name: Run Tests - run: mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true + run: | + timeout 5400s mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true || (jstack -l $(pgrep -f 'java') > thread-dump.txt && exit 1) - uses: actions/upload-artifact@v4 if: failure() with: From 593a82ae0a387367ef42e8f654169fbfbb31657f Mon Sep 17 00:00:00 2001 From: fl4via Date: Mon, 28 Jul 2025 12:08:59 -0300 Subject: [PATCH 02/11] DO NOT MERGE: cause the test to run forever, to test the ci command that takes the thread dump Signed-off-by: fl4via --- core/src/test/java/io/undertow/util/ByteRangeTestCase.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/test/java/io/undertow/util/ByteRangeTestCase.java b/core/src/test/java/io/undertow/util/ByteRangeTestCase.java index 4b2d1ff3c8..8eba5f2bfe 100644 --- a/core/src/test/java/io/undertow/util/ByteRangeTestCase.java +++ b/core/src/test/java/io/undertow/util/ByteRangeTestCase.java @@ -232,6 +232,12 @@ public void testGetResponseResult6() { new Date(1559820153000L), "foo").getStatusCode()); } + @Test + public void testRunForever() throws InterruptedException { + // sleep for 90 minutes, just to trigger the thread dump and see if it is working okay + Thread.sleep(5400000); + } + @Test public void testGetResponseResultNull() { ByteRange byteRange = new ByteRange(new ArrayList<>()); From d8611fc0fca2191a4f48d35128d938f7e7c67ec7 Mon Sep 17 00:00:00 2001 From: fl4via Date: Mon, 28 Jul 2025 15:16:29 -0300 Subject: [PATCH 03/11] [UNDERTOW-2587] Add Windows Script Signed-off-by: fl4via --- .github/workflows/ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 362c938c20..e73c88d7df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,13 @@ jobs: - name: Print Version run: mvn -v - name: Run Tests + if: matrix.os == 'windows-latest' + run: | + mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} > output.txt + timeout /t 180 + "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') + - name: Run Tests + if: matrix.os != 'windows-latest' run: | timeout 5400s mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} || (jstack -l $(pgrep -f 'java') > thread-dump.txt && exit 1) - uses: actions/upload-artifact@v4 @@ -157,6 +164,13 @@ jobs: - name: Print Version run: mvn -v - name: Run Tests + if: matrix.os == 'windows-latest' + run: | + mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} > output.txt + timeout /t 180 + "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') + - name: Run Tests + if: matrix.os != 'windows-latest' run: | timeout 5400s mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true || (jstack -l $(pgrep -f 'java') > thread-dump.txt && exit 1) - uses: actions/upload-artifact@v4 From e4e3861699126384d58a5288d5004dca2a6a1c84 Mon Sep 17 00:00:00 2001 From: fl4via Date: Tue, 29 Jul 2025 01:14:03 -0300 Subject: [PATCH 04/11] [UNDERTOW-2587] Try to improve the linux command Signed-off-by: fl4via --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e73c88d7df..ca9c11108e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,13 +106,13 @@ jobs: - name: Run Tests if: matrix.os == 'windows-latest' run: | - mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} > output.txt + mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & timeout /t 180 "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') - name: Run Tests if: matrix.os != 'windows-latest' run: | - timeout 5400s mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} || (jstack -l $(pgrep -f 'java') > thread-dump.txt && exit 1) + timeout 5400s mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} || jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt && exit 1) - uses: actions/upload-artifact@v4 if: failure() with: @@ -166,13 +166,13 @@ jobs: - name: Run Tests if: matrix.os == 'windows-latest' run: | - mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} > output.txt + mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & timeout /t 180 "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') - name: Run Tests if: matrix.os != 'windows-latest' run: | - timeout 5400s mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true || (jstack -l $(pgrep -f 'java') > thread-dump.txt && exit 1) + timeout 5400s mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true || jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt && exit 1) - uses: actions/upload-artifact@v4 if: failure() with: From a8520c9755663691a547125ebee41cdb21805480 Mon Sep 17 00:00:00 2001 From: fl4via Date: Tue, 29 Jul 2025 01:27:12 -0300 Subject: [PATCH 05/11] Continue Signed-off-by: fl4via --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca9c11108e..700e1fe796 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,7 +107,7 @@ jobs: if: matrix.os == 'windows-latest' run: | mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & - timeout /t 180 + timeout /t 180 /NoBreak 1>NUL "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') - name: Run Tests if: matrix.os != 'windows-latest' @@ -167,7 +167,7 @@ jobs: if: matrix.os == 'windows-latest' run: | mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & - timeout /t 180 + timeout /t 180 /NoBreak 1>NUL "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') - name: Run Tests if: matrix.os != 'windows-latest' From d0c7b578cf3011f1beee49022dd4dae5753b4b2c Mon Sep 17 00:00:00 2001 From: fl4via Date: Tue, 29 Jul 2025 01:36:13 -0300 Subject: [PATCH 06/11] More changes Signed-off-by: fl4via --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 700e1fe796..5494b6aa23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,8 +107,8 @@ jobs: if: matrix.os == 'windows-latest' run: | mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & - timeout /t 180 /NoBreak 1>NUL - "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') + Start-Sleep 180 + "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') & exit 1 - name: Run Tests if: matrix.os != 'windows-latest' run: | @@ -167,8 +167,8 @@ jobs: if: matrix.os == 'windows-latest' run: | mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & - timeout /t 180 /NoBreak 1>NUL - "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') + Start-Sleep 180 + "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') || exit 1 - name: Run Tests if: matrix.os != 'windows-latest' run: | From d2bf4adc9fbde24f4f2e688ecff95fa7cadd41eb Mon Sep 17 00:00:00 2001 From: Flavia Rainone Date: Tue, 29 Jul 2025 16:50:47 -0300 Subject: [PATCH 07/11] [UNDERTOW-2587] Fix the linux script Signed-off-by: Flavia Rainone --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5494b6aa23..4e2a4275dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,10 @@ jobs: - name: Run Tests if: matrix.os != 'windows-latest' run: | - timeout 5400s mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} || jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt && exit 1) + timeout --foreground 180s sh -c 'mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }};exit' || echo "Hanging test found, we will proceed with a thread dump" + timeout_occurred=$? + hanging_test_pid=$(jps | awk '$2 ~ /surefirebooter/ {print $1}') + [ ! -z "$hanging_test_pid" ] && echo "Hanging test found. Pid is $hanging_test_pid" && jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') | tee thread-dump.txt && exit 1 || echo "No hanging test found, tests ran successfully" && exit $timeout_occurred - uses: actions/upload-artifact@v4 if: failure() with: @@ -172,7 +175,10 @@ jobs: - name: Run Tests if: matrix.os != 'windows-latest' run: | - timeout 5400s mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true || jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt && exit 1) + timeout --foreground 180s sh -c 'mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true;exit' || echo "Hanging test found, we will proceed with a thread dump" + timeout_occurred=$? + hanging_test_pid=$(jps | awk '$2 ~ /surefirebooter/ {print $1}') + [ ! -z "$hanging_test_pid" ] && echo "Hanging test found. Pid is $hanging_test_pid" && jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') | tee thread-dump.txt && exit 1 || echo "No hanging test found, tests ran successfully" && exit $timeout_occurred - uses: actions/upload-artifact@v4 if: failure() with: From 94b533b78e5e6d369a63310a36b2284d5b8519e4 Mon Sep 17 00:00:00 2001 From: Flavia Rainone Date: Tue, 29 Jul 2025 23:57:49 -0300 Subject: [PATCH 08/11] [UNDERTOW-2587] Install coreutils to use timeout with macos Signed-off-by: Flavia Rainone --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e2a4275dd..ef79fb5e5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,11 @@ jobs: repositories: '[{ "id": "jboss", "name": "JBoss", "url": "https://repository.jboss.org/nexus/content/groups/public" }]' - name: Print Version run: mvn -v + - name: Install coreutils + if: matrix.os == 'macos-latest' + run: | + brew install coreutils + alias timeout=gtimeout - name: Run Tests if: matrix.os == 'windows-latest' run: | From 0fb09ce0587159dd60015440f57bac32bdd893ee Mon Sep 17 00:00:00 2001 From: fl4via Date: Fri, 1 Aug 2025 01:12:10 -0300 Subject: [PATCH 09/11] [UNDERTOW-2587] Add a script to run the build with a timeout (in minutes) and take a thread dump if the timeout expires Signed-off-by: fl4via --- .github/workflows/ci.yml | 4 +--- scripts/run_command_with_timeout.bat | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 scripts/run_command_with_timeout.bat diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef79fb5e5b..9b581e051a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,9 +111,7 @@ jobs: - name: Run Tests if: matrix.os == 'windows-latest' run: | - mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & - Start-Sleep 180 - "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') & exit 1 + .\scripts\run_command_with_timeout.bat 2 "mvn -U -B -fae test -Pproxy -DfailIfNoTests=false -pl ${{ matrix.module }}" - name: Run Tests if: matrix.os != 'windows-latest' run: | diff --git a/scripts/run_command_with_timeout.bat b/scripts/run_command_with_timeout.bat new file mode 100644 index 0000000000..9ca7303fba --- /dev/null +++ b/scripts/run_command_with_timeout.bat @@ -0,0 +1,28 @@ +@echo off +REM usage: run_command_with_timeout.bat timeout command +REM where: +REM timeout is the timeout in minutes +REM command is the command to run +REM author: Flavia Rainone + + +set "command=%~2" +echo %command% +set "lock=%temp%\%~nx0.lock" + +start "" /b %command% 9>"%lock%" +set /a count=0 + +:wait +REM waited too long, kill the test if it is there +if %count%==%1 (goto :killHangingTest) +timeout /t 60 /nobreak >nul +set /a count=%count% + 1 +2>nul (9>"%lock%" (call ) && del "%lock%" || goto :wait) +echo No hanging test found, tests ran successfully +exit 0 + +:killHangingTest +FOR /F "tokens=*" %%A IN ('jps^|awk "$2 ~ /surefirebooter/ {print $1}"') DO SET PID=%%A +echo Hanging test found. Pid is %PID% +jstack -l %PID% > thread-dump.txt & taskkill /F /PID %PID% & exit 1 \ No newline at end of file From 9bda3f3c306b9a8165cf3263a7638484b583f412 Mon Sep 17 00:00:00 2001 From: fl4via Date: Fri, 1 Aug 2025 01:22:52 -0300 Subject: [PATCH 10/11] [UNDERTOW-2587] Replace timeout /t /nobreak by Start-Sleep: CI does not run the former Signed-off-by: fl4via --- scripts/run_command_with_timeout.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_command_with_timeout.bat b/scripts/run_command_with_timeout.bat index 9ca7303fba..4436f1f1a7 100644 --- a/scripts/run_command_with_timeout.bat +++ b/scripts/run_command_with_timeout.bat @@ -16,7 +16,7 @@ set /a count=0 :wait REM waited too long, kill the test if it is there if %count%==%1 (goto :killHangingTest) -timeout /t 60 /nobreak >nul +powershell -Command "Start-Sleep -Seconds 60" set /a count=%count% + 1 2>nul (9>"%lock%" (call ) && del "%lock%" || goto :wait) echo No hanging test found, tests ran successfully From fed90d912be5145a5e1ab0aa2c3b2d33836a2a7c Mon Sep 17 00:00:00 2001 From: fl4via Date: Fri, 1 Aug 2025 01:32:49 -0300 Subject: [PATCH 11/11] fixup --- .github/workflows/ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b581e051a..4930200d31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,7 @@ jobs: path: | **/surefire*-reports/*.txt **/*.dump* + thread-dump.txt test-matrix-ipv6: name: JDK ${{ matrix.jdk }} - ipv6 - ${{ matrix.module }} ${{ matrix.proxy }} - ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -170,13 +171,6 @@ jobs: - name: Print Version run: mvn -v - name: Run Tests - if: matrix.os == 'windows-latest' - run: | - mvn -U -B -fae test -Pproxy '-DfailIfNoTests=false' -pl ${{ matrix.module }} & - Start-Sleep 180 - "jstack -l $(jps | awk '$2 ~ /surefirebooter/ {print $1}') > thread-dump.txt" & taskkill /F /PID $(jps | awk '$2 ~ /surefirebooter/ {print $1}') || exit 1 - - name: Run Tests - if: matrix.os != 'windows-latest' run: | timeout --foreground 180s sh -c 'mvn -U -B -fae test ${{ matrix.proxy }} '-DfailIfNoTests=false' -pl ${{ matrix.module }} -Dtest.ipv6=true;exit' || echo "Hanging test found, we will proceed with a thread dump" timeout_occurred=$? @@ -189,3 +183,4 @@ jobs: path: | **/surefire*-reports/*.txt **/*.dump* + thread-dump.txt \ No newline at end of file