ztest: Add Support for Multiple Test Runs.

Introduces new kconfig options to enable multiple test runs in the ztest.

Signed-off-by: Arkadiusz Cholewinski <[email protected]>

Author: gbarkadiusz

doc/develop/test/ztest.rst

@@ -602,11 +602,15 @@ randomize the order. The output from the test will display the seed for failed
 tests.  For native simulator builds you can provide the seed as an argument to
 twister with ``--seed``.
 
-Static configuration of ZTEST_SHUFFLE contains:
-
- - :kconfig:option:`CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT` - Number of iterations the test suite will run.
- - :kconfig:option:`CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT` - Number of iterations the test will run.
 
+Repeating Tests
+***********************
+By default the tests are executed once. The test cases and test suites
+may be executed multiple times. Enable :kconfig:option:`CONFIG_ZTEST_REPEAT` to
+execute the tests multiple times. By default the multiplication factors are 3, which
+means every test suite is executed 3 times and every test case is executed 3 times. This can
+be changed by the :kconfig:option:`CONFIG_ZTEST_SUITE_REPEAT_COUNT` and
+:kconfig:option:`CONFIG_ZTEST_TEST_REPEAT_COUNT` Kconfig options.
 
 Test Selection
 **************

subsys/testsuite/ztest/Kconfig

@@ -150,22 +150,41 @@ config ZTEST_SHUFFLE
 
 if ZTEST_SHUFFLE
 config ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
-	int "Number of iterations the test suite will run"
+	int "[DEPRECATED] Number of iterations the test suite will run"
 	default 3
 	help
-	  This rule will execute a test suite N number of times. The tests
-	  per suite will be shuffled on each iteration.  The test order will likely
-	  be different per iteration.
+	  This is used to execute a test suite N number of times.
+	  [DEPRECATED] use ZTEST_SUITE_REPEAT_COUNT instead.
 
 config ZTEST_SHUFFLE_TEST_REPEAT_COUNT
-	int "Number of iterations the test will run"
+	int "[DEPRECATED] Number of iterations the test will run"
 	default 3
 	help
-	  This rule will execute a test N number of times.  The test order will
-	  likely be different per iteration.
+	  This is used to execute a test case N number of times.
+	  [DEPRECATED] use ZTEST_TEST_REPEAT_COUNT instead.
 
 endif #ZTEST_SHUFFLE
 
+config ZTEST_REPEAT
+	bool "Repeat the tests and suites"
+	help
+	  This rule will repeat the tests and the test suites.
+
+if ZTEST_REPEAT
+config ZTEST_SUITE_REPEAT_COUNT
+	int "Number of iterations the test suite will run"
+	default 3
+	help
+	  This rule will execute a test suite N number of times.
+
+config ZTEST_TEST_REPEAT_COUNT
+	int "Number of iterations the test will run"
+	default 3
+	help
+	  This rule will execute a test N number of times.
+
+endif #ZTEST_REPEAT
+
 config ZTEST_SUMMARY
 	bool "Display test summary"
 	default y

subsys/testsuite/ztest/src/ztest.c

@@ -30,9 +30,15 @@ static bool failed_expectation;
 #include <stdlib.h>
 #include <time.h>
 #include <zephyr/random/random.h>
-
+#ifndef CONFIG_ZTEST_REPEAT
 #define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT
 #define NUM_ITER_PER_TEST  CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT
+#endif
+#endif /* CONFIG_ZTEST_SHUFFLE */
+
+#ifdef CONFIG_ZTEST_REPEAT
+#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SUITE_REPEAT_COUNT
+#define NUM_ITER_PER_TEST  CONFIG_ZTEST_TEST_REPEAT_COUNT
 #else
 #define NUM_ITER_PER_SUITE 1
 #define NUM_ITER_PER_TEST  1

tests/net/lib/lwm2m/block_transfer/prj.conf

@@ -8,6 +8,7 @@ CONFIG_TEST_RANDOM_GENERATOR=y
 CONFIG_LWM2M_LOG_ENCODE_BUFFER_ALLOCATIONS=y
 
 CONFIG_ZTEST_SHUFFLE=y
+CONFIG_ZTEST_REPEAT=y
 
 CONFIG_LWM2M=y
 CONFIG_LWM2M_COAP_BLOCK_TRANSFER=y

tests/subsys/ipc/pbuf/prj.conf

@@ -1,6 +1,7 @@
 CONFIG_ZTEST=y
 CONFIG_ZTRESS=y
 CONFIG_ZTEST_SHUFFLE=y
+CONFIG_ZTEST_REPEAT=y
 
 CONFIG_IPC_SERVICE=y
 CONFIG_IPC_SERVICE_ICMSG=y

tests/subsys/modem/backends/uart/prj.conf

@@ -7,4 +7,5 @@ CONFIG_SERIAL=y
 CONFIG_ZTEST=y
 CONFIG_LOG=y
 CONFIG_ZTEST_SHUFFLE=y
-CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=3
+CONFIG_ZTEST_REPEAT=y
+CONFIG_ZTEST_TEST_REPEAT_COUNT=3

tests/ztest/base/prj_verbose_0.conf

@@ -6,5 +6,6 @@ CONFIG_TEST_RANDOM_GENERATOR=y
 CONFIG_TIMER_RANDOM_GENERATOR=y
 
 CONFIG_ZTEST_SHUFFLE=y
-CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
-CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2
+CONFIG_ZTEST_REPEAT=y
+CONFIG_ZTEST_SUITE_REPEAT_COUNT=2
+CONFIG_ZTEST_TEST_REPEAT_COUNT=2

tests/ztest/base/prj_verbose_1.conf

@@ -2,6 +2,7 @@ CONFIG_ZTEST=y
 CONFIG_ZTEST_ASSERT_VERBOSE=1
 
 CONFIG_ZTEST_SHUFFLE=y
-CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT=2
-CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT=2
+CONFIG_ZTEST_REPEAT=y
+CONFIG_ZTEST_SUITE_REPEAT_COUNT=2
+CONFIG_ZTEST_TEST_REPEAT_COUNT=2
 CONFIG_ENTROPY_GENERATOR=y

tests/ztest/base/src/main.c

@@ -153,8 +153,8 @@ static void rule_test_teardown(void *data)
 	 * after_each function was called.
 	 */
 	zassert_equal(fixture->state, RULE_STATE_AFTER_EACH, "Unexpected state");
-#ifdef CONFIG_ZTEST_SHUFFLE
-	zassert_equal(fixture->run_count, CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT);
+#ifdef CONFIG_ZTEST_REPEAT
+	zassert_equal(fixture->run_count, CONFIG_ZTEST_TEST_REPEAT_COUNT);
 #endif
 }