Fix retry logic for unit test

.github/workflows/ci.yml

@@ -79,7 +79,7 @@ jobs:
             platform: x86_64
             test-category: full
             full-gpu-tests: true
-            runs-on: [Windows, self-hosted, "GCP-T4"]
+            runs-on: [Windows, self-hosted, GCP-T4]
             has-gpu: true
             server-count: 8
       fail-fast: false
@@ -231,23 +231,12 @@ jobs:
               -skip-reference-image-generation \
               -show-adapter-info \
               -enable-debug-layers false
-          elif [[ "${{matrix.has-gpu}}" == "true" ]]; then
-            "$bin_dir/slang-test" \
-              -use-test-server \
-              -category ${{ matrix.test-category }} \
-              -api all-dx12 \
-              -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-github-runner.txt \
-              -skip-reference-image-generation \
-              -show-adapter-info  \
-              -enable-debug-layers false
           else
             "$bin_dir/slang-test" \
               -use-test-server \
               -category ${{ matrix.test-category }} \
-              -api all-dx12 \
               -expected-failure-list tests/expected-failure-github.txt \
-              -expected-failure-list tests/expected-failure-github-runner.txt \
+              -expected-failure-list tests/expected-failure-no-gpu.txt \
               -skip-reference-image-generation \
               -show-adapter-info \
               -enable-debug-layers false
@@ -276,7 +265,7 @@ jobs:
             -category ${{ matrix.test-category }} \
             -emit-spirv-via-glsl \
             -api vk \
-            -expected-failure-list tests/expected-failure.txt \
+            -expected-failure-list tests/expected-failure-via-glsl.txt \
             -skip-reference-image-generation \
             -show-adapter-info
       - name: Run slang-rhi tests

tests/expected-failure-no-gpu.txt

@@ -0,0 +1,6 @@
+slang-unit-test-tool/RecordReplay_triangle.internal
+slang-unit-test-tool/RecordReplay_ray_tracing.internal
+slang-unit-test-tool/RecordReplay_ray_tracing_pipeline.internal
+slang-unit-test-tool/RecordReplay_autodiff_texture.internal
+slang-unit-test-tool/RecordReplay_gpu_printing.internal
+slang-unit-test-tool/cudaCodeGenBug.internal

tools/slang-test/slang-test-main.cpp

@@ -949,28 +949,48 @@ static Result _executeRPC(
     JSONRPCConnection* rpcConnection = context->getOrCreateJSONRPCConnection();
     if (!rpcConnection)
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: getOrCreateJSONRPCConnection()");
         return SLANG_FAIL;
     }
 
     // Execute
     if (SLANG_FAILED(rpcConnection->sendCall(method, rttiInfo, args)))
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: sendCall()");
+
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
 
     // Wait for the result
-    rpcConnection->waitForResult(context->connectionTimeOutInMs);
+    if (SLANG_FAILED(rpcConnection->waitForResult(context->connectionTimeOutInMs)))
+    {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: waitForResult()");
+    }
 
     if (!rpcConnection->hasMessage())
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: hasMessage()");
+
         // We can assume somethings gone wrong. So lets kill the connection and fail.
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
 
     if (rpcConnection->getMessageType() != JSONRPCMessageType::Result)
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: getMessageType() != JSONRPCMessageType::Result");
+
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
@@ -979,6 +999,10 @@ static Result _executeRPC(
     TestServerProtocol::ExecutionResult exeRes;
     if (SLANG_FAILED(rpcConnection->getMessage(&exeRes)))
     {
+        context->getTestReporter()->messageFormat(
+            TestMessageType::RunError,
+            "JSON RPC failure: getMessage()");
+
         context->destroyRPCConnection();
         return SLANG_FAIL;
     }
@@ -4780,8 +4804,8 @@ static SlangResult runUnitTestModule(
                 }
 
                 // If the test failed and it is not an expected failure, add it to the list of
-                // failed unit tests.
-                if (isFailed &&
+                // failed unit tests so that we can retry.
+                if (isFailed && !context->isRetry &&
                     !context->getTestReporter()->m_expectedFailureList.contains(test.testName))
                 {
                     std::lock_guard lock(context->mutexFailedTests);
@@ -5142,7 +5166,7 @@ SlangResult innerMain(int argc, char** argv)
 
 int main(int argc, char** argv)
 {
-    const SlangResult res = innerMain(argc, argv);
+    SlangResult res = innerMain(argc, argv);
     slang::shutdown();
     Slang::RttiInfo::deallocateAll();
 

tools/slang-unit-test/unit-test-record-replay.cpp

@@ -256,6 +256,16 @@ static SlangResult runExample(
         hashLines.add(line);
     }
 
+    if (hashLines.getCount() == 0)
+    {
+        msgBuilder << "Hash value is not found for '" << exampleName << "'\n";
+        msgBuilder << "Process ret code: " << exeRes.resultCode << "\n";
+        msgBuilder << "Standard output:\n" << exeRes.standardOutput << "\n";
+        msgBuilder << "Standard error:\n" << exeRes.standardError << "\n";
+        getTestReporter()->message(TestMessageType::TestFailure, msgBuilder.toString().getBuffer());
+        return SLANG_FAIL;
+    }
+
     res = parseHashes(hashLines, outHashes);
     if (SLANG_FAILED(res))
     {