Failed tests are now listed in the total (#3073)

* Failed tests are now listed in the total
rh-hideout · Jul 14, 2023 · 5f29ae6 · 5f29ae6
1 parent 98ae1be
commit 5f29ae6
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 7 deletions.
diff --git a/include/test_runner.h b/include/test_runner.h
@@ -3,11 +3,10 @@
 
 extern const bool8 gTestRunnerEnabled;
 extern const bool8 gTestRunnerHeadless;
+extern const bool8 gTestRunnerSkipIsFail;
 
 #if TESTING
 
-extern const bool8 gTestRunnerSkipIsFail;
-
 void TestRunner_Battle_RecordAbilityPopUp(u32 battlerId, u32 ability);
 void TestRunner_Battle_RecordAnimation(u32 animType, u32 animId);
 void TestRunner_Battle_RecordHP(u32 battlerId, u32 oldHP, u32 newHP);

diff --git a/test/test_runner_battle.c b/test/test_runner_battle.c
@@ -10,6 +10,17 @@
 #include "test_battle.h"
 #include "window.h"
 
+#if defined(__INTELLISENSE__)
+#undef TestRunner_Battle_RecordAbilityPopUp
+#undef TestRunner_Battle_RecordAnimation
+#undef TestRunner_Battle_RecordHP
+#undef TestRunner_Battle_RecordMessage
+#undef TestRunner_Battle_RecordStatus1
+#undef TestRunner_Battle_AfterLastTurn
+#undef TestRunner_Battle_CheckBattleRecordActionType
+#undef TestRunner_Battle_GetForcedAbility
+#endif
+
 #define INVALID(fmt, ...) Test_ExitWithResult(TEST_RESULT_INVALID, "%s:%d: " fmt, gTestRunnerState.test->filename, sourceLine, ##__VA_ARGS__)
 #define INVALID_IF(c, fmt, ...) do { if (c) Test_ExitWithResult(TEST_RESULT_INVALID, "%s:%d: " fmt, gTestRunnerState.test->filename, sourceLine, ##__VA_ARGS__); } while (0)
 

diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c
@@ -31,7 +31,11 @@
 #include <sys/wait.h>
 #include <unistd.h>
 
-#define MAX_PROCESSES 32 // See also test/test.h
+#define min(a, b) ((a) < (b) ? (a) : (b))
+
+#define MAX_PROCESSES               32 // See also test/test.h
+#define MAX_FAILED_TESTS_TO_LIST    100
+#define MAX_TEST_LIST_BUFFER_LENGTH 256
 
 struct Runner
 {
@@ -51,6 +55,7 @@ struct Runner
     int assumptionFails;
     int fails;
     int results;
+    char failedTestNames[MAX_FAILED_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH];
 };
 
 static unsigned nrunners = 0;
@@ -99,6 +104,8 @@ static void handle_read(int i, struct Runner *runner)
                     runner->assumptionFails++;
                     goto add_to_results;
                 case 'F':
+                    if (runner->fails < MAX_FAILED_TESTS_TO_LIST)
+                        strcpy(runner->failedTestNames[runner->fails], runner->test_name);
                     runner->fails++;
 add_to_results:
                     runner->results++;
@@ -182,6 +189,14 @@ static void exit2(int _)
     exit(2);
 }
 
+int compare_strings(const void * a, const void * b)
+{
+    const char *arg1 = (const char *) a;
+    const char *arg2 = (const char *) b;
+
+    return strcmp(arg1, arg2);
+}
+
 int main(int argc, char *argv[])
 {
     if (argc < 4)
@@ -476,6 +491,9 @@ int main(int argc, char *argv[])
     int assumptionFails = 0;
     int fails = 0;
     int results = 0;
+
+    char failedTestNames[MAX_FAILED_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH];
+
     for (int i = 0; i < nrunners; i++)
     {
         int wstatus;
@@ -492,26 +510,45 @@ int main(int argc, char *argv[])
         knownFails += runners[i].knownFails;
         todos += runners[i].todos;
         assumptionFails += runners[i].assumptionFails;
-        fails += runners[i].fails;
+        for (int j = 0; j < runners[i].fails; j++)
+        {
+            if (j < MAX_FAILED_TESTS_TO_LIST)
+                strcpy(failedTestNames[fails], runners[i].failedTestNames[j]);
+            fails++;
+        }
         results += runners[i].results;
     }
 
+    qsort(failedTestNames, min(fails, MAX_FAILED_TESTS_TO_LIST), sizeof(char) * MAX_TEST_LIST_BUFFER_LENGTH, compare_strings);
+
     if (results == 0)
     {
         fprintf(stdout, "\nNo tests found.\n");
     }
     else
     {
-        fprintf(stdout, "\n- Tests TOTAL:         %d\n", results);
+        if (fails > 0)
+        {
+            fprintf(stdout, "\n- Tests \e[31mFAILED\e[0m :       %d    Add TESTS='X' to run tests with the defined prefix.\n", fails);
+            for (int i = 0; i < fails; i++)
+            {
+                if (i >= MAX_FAILED_TESTS_TO_LIST)
+                {
+                    fprintf(stdout, "  - \e[31mand %d more...\e[0m\n", fails - MAX_FAILED_TESTS_TO_LIST);
+                    break;
+                }
+                fprintf(stdout, "  - \e[31m%s\e[0m.\n", failedTestNames[i]);
+            }
+        }
         fprintf(stdout, "- Tests \e[32mPASSED\e[0m:        %d\n", passes);
         if (knownFails > 0)
             fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails);
         if (todos > 0)
             fprintf(stdout, "- Tests \e[33mTO_DO\e[0m:         %d\n", todos);
-        if (fails > 0)
-            fprintf(stdout, "- Tests \e[31mFAILED\e[0m :       %d\n", fails);
         if (assumptionFails > 0)
             fprintf(stdout, "- \e[33mASSUMPTIONS_FAILED\e[0m:  %d\n", assumptionFails);
+
+        fprintf(stdout, "- Tests \e[34mTOTAL\e[0m:         %d\n", results);
     }
     fprintf(stdout, "\n");