Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Detect test runner crashes #3129

Merged
merged 2 commits into from
Jul 15, 2023
Merged

Detect test runner crashes #3129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4637d7e
Recover from test crashes
mrgriffin Jul 14, 2023
25986be
Prevent corrupted heaps causing infinite loops
mrgriffin Jul 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ld_script_test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ SECTIONS {
test/*.o(COMMON);
*libc.a:sbrkr.o(COMMON);
end = .;

/* .persistent starts at 0x3007F00 */
/* WARNING: This is the end of the IRQ stack, if there's too
* much data it WILL be overwritten. */
. = 0x7F00;
test/*.o(.persistent);

. = 0x8000;
}

Expand Down
3 changes: 1 addition & 2 deletions test/test.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ enum TestResult
TEST_RESULT_INVALID,
TEST_RESULT_ERROR,
TEST_RESULT_TIMEOUT,
TEST_RESULT_CRASH,
TEST_RESULT_TODO,
};

Expand All @@ -38,8 +39,6 @@ struct TestRunnerState
{
u8 state;
u8 exitCode;
s32 tests;
s32 passes;
const char *skipFilename;
const struct Test *test;
u32 processCosts[MAX_PROCESSES];
Expand Down
155 changes: 120 additions & 35 deletions test/test_runner.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,16 @@ void CB2_TestRunner(void);
EWRAM_DATA struct TestRunnerState gTestRunnerState;
EWRAM_DATA struct FunctionTestRunnerState *gFunctionTestRunnerState;

enum {
CURRENT_TEST_STATE_ESTIMATE,
CURRENT_TEST_STATE_RUN,
};

__attribute__((section(".persistent"))) static struct {
u32 address:28;
u32 state:1;
} sCurrentTest = {0};

void TestRunner_Battle(const struct Test *);

static bool32 MgbaOpen_(void);
Expand Down Expand Up @@ -51,6 +61,47 @@ enum
STATE_EXIT,
};

static u32 MinCostProcess(void)
{
u32 i;
u32 minCost, minCostProcess;

minCost = gTestRunnerState.processCosts[0];
minCostProcess = 0;
for (i = 1; i < gTestRunnerN; i++)
{
if (gTestRunnerState.processCosts[i] < minCost)
{
minCost = gTestRunnerState.processCosts[i];
minCostProcess = i;
}
}

return minCostProcess;
}

// Greedily assign tests to processes based on estimated cost.
// TODO: Make processCosts a min heap.
static u32 AssignCostToRunner(void)
{
u32 minCostProcess;

if (gTestRunnerState.test->runner == &gAssumptionsRunner)
return TRUE;

minCostProcess = MinCostProcess();

// XXX: If estimateCost returns only on some processes, or
// returns inconsistent results then processCosts will be
// inconsistent and some tests may not run.
if (gTestRunnerState.test->runner->estimateCost)
gTestRunnerState.processCosts[minCostProcess] += gTestRunnerState.test->runner->estimateCost(gTestRunnerState.test->data);
else
gTestRunnerState.processCosts[minCostProcess] += 1;

return minCostProcess;
}

void CB2_TestRunner(void)
{
switch (gTestRunnerState.state)
Expand All @@ -65,12 +116,43 @@ void CB2_TestRunner(void)

gIntrTable[7] = Intr_Timer2;

gTestRunnerState.state = STATE_NEXT_TEST;
// The current test restarted the ROM (e.g. by jumping to NULL).
if (sCurrentTest.address != 0)
{
gTestRunnerState.test = __start_tests;
while ((uintptr_t)gTestRunnerState.test != sCurrentTest.address)
{
AssignCostToRunner();
gTestRunnerState.test++;
}
if (sCurrentTest.state == CURRENT_TEST_STATE_ESTIMATE)
{
u32 runner = MinCostProcess();
gTestRunnerState.processCosts[runner] += 1;
if (runner == gTestRunnerI)
{
gTestRunnerState.state = STATE_REPORT_RESULT;
gTestRunnerState.result = TEST_RESULT_CRASH;
}
else
{
gTestRunnerState.state = STATE_NEXT_TEST;
}
}
else
{
gTestRunnerState.state = STATE_REPORT_RESULT;
gTestRunnerState.result = TEST_RESULT_CRASH;
}
}
else
{
gTestRunnerState.state = STATE_NEXT_TEST;
gTestRunnerState.test = __start_tests - 1;
}
gTestRunnerState.exitCode = 0;
gTestRunnerState.tests = 0;
gTestRunnerState.passes = 0;
gTestRunnerState.skipFilename = NULL;
gTestRunnerState.test = __start_tests - 1;

break;

case STATE_NEXT_TEST:
Expand Down Expand Up @@ -108,40 +190,19 @@ void CB2_TestRunner(void)
return;
}

// Greedily assign tests to processes based on estimated cost.
// TODO: Make processCosts a min heap.
if (gTestRunnerState.test->runner != &gAssumptionsRunner)
{
u32 i;
u32 minCost, minCostProcess;
minCost = gTestRunnerState.processCosts[0];
minCostProcess = 0;
for (i = 1; i < gTestRunnerN; i++)
{
if (gTestRunnerState.processCosts[i] < minCost)
{
minCost = gTestRunnerState.processCosts[i];
minCostProcess = i;
}
}

if (minCostProcess == gTestRunnerI)
gTestRunnerState.state = STATE_RUN_TEST;
else
gTestRunnerState.state = STATE_NEXT_TEST;
sCurrentTest.address = (uintptr_t)gTestRunnerState.test;
sCurrentTest.state = CURRENT_TEST_STATE_ESTIMATE;

// XXX: If estimateCost exits only on some processes then
// processCosts will be inconsistent.
if (gTestRunnerState.test->runner->estimateCost)
gTestRunnerState.processCosts[minCostProcess] += gTestRunnerState.test->runner->estimateCost(gTestRunnerState.test->data);
else
gTestRunnerState.processCosts[minCostProcess] += 1;
}
if (AssignCostToRunner() == gTestRunnerI)
gTestRunnerState.state = STATE_RUN_TEST;
else
gTestRunnerState.state = STATE_NEXT_TEST;

break;

case STATE_RUN_TEST:
gTestRunnerState.state = STATE_REPORT_RESULT;
sCurrentTest.state = CURRENT_TEST_STATE_RUN;
if (gTestRunnerState.test->runner->setUp)
gTestRunnerState.test->runner->setUp(gTestRunnerState.test->data);
gTestRunnerState.test->runner->run(gTestRunnerState.test->data);
Expand All @@ -162,6 +223,15 @@ void CB2_TestRunner(void)
const struct MemBlock *block = head;
do
{
if (block->magic != MALLOC_SYSTEM_ID
|| !(EWRAM_START <= (uintptr_t)block->next && (uintptr_t)block->next < EWRAM_END)
|| (block->next <= block && block->next != head))
{
MgbaPrintf_("gHeap corrupted block at 0x%p", block);
gTestRunnerState.result = TEST_RESULT_ERROR;
break;
}

if (block->allocated)
{
const char *location = MemBlockLocation(block);
Expand All @@ -186,11 +256,8 @@ void CB2_TestRunner(void)
const char *color;
const char *result;

gTestRunnerState.tests++;

if (gTestRunnerState.result == gTestRunnerState.expectedResult)
{
gTestRunnerState.passes++;
color = "\e[32m";
MgbaPrintf_(":N%s", gTestRunnerState.test->name);
}
Expand Down Expand Up @@ -243,6 +310,9 @@ void CB2_TestRunner(void)
case TEST_RESULT_TIMEOUT:
result = "TIMEOUT";
break;
case TEST_RESULT_CRASH:
result = "CRASH";
break;
default:
result = "UNKNOWN";
break;
Expand Down Expand Up @@ -434,6 +504,7 @@ static s32 MgbaVPrintf_(const char *fmt, va_list va)
{
s32 i = 0;
s32 c, d;
u32 p;
const char *s;
while (*fmt)
{
Expand Down Expand Up @@ -467,6 +538,20 @@ static s32 MgbaVPrintf_(const char *fmt, va_list va)
i = MgbaPutchar_(i, buffer[--n]);
}
break;
case 'p':
p = va_arg(va, unsigned);
{
s32 n;
for (n = 0; n < 7; n++)
{
unsigned nybble = (p >> (24 - (4*n))) & 0xF;
if (nybble <= 9)
i = MgbaPutchar_(i, '0' + nybble);
else
i = MgbaPutchar_(i, 'a' + nybble - 10);
}
}
break;
case 'q':
d = va_arg(va, int);
{
Expand Down