[wip] testgen reproducibility with MODE_MULTIPROCESSING mode

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -140,7 +140,7 @@ def should_skip_case_dir(case_dir, is_force, diagnostics_obj):
     return is_skip, diagnostics_obj
 
 
-def run_generator(generator_name, test_providers: Iterable[TestProvider]):
+def run_generator(generator_name, test_providers: Iterable[TestProvider], generator_mode: str=None):
     """
     Implementation for a general test generator.
     :param generator_name: The name of the generator. (lowercase snake_case)
@@ -227,7 +227,10 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
     diagnostics_obj = Diagnostics()
     provider_start = time.time()
 
-    if GENERATOR_MODE == MODE_MULTIPROCESSING:
+    if generator_mode is None:
+        generator_mode = GENERATOR_MODE
+
+    if generator_mode == MODE_MULTIPROCESSING:
         all_test_case_params = []
 
     for tprov in test_providers:
@@ -252,14 +255,14 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]):
             if is_skip:
                 continue
 
-            if GENERATOR_MODE == MODE_SINGLE_PROCESS:
+            if generator_mode == MODE_SINGLE_PROCESS:
                 result = generate_test_vector(test_case, case_dir, log_file, file_mode)
                 write_result_into_diagnostics_obj(result, diagnostics_obj)
-            elif GENERATOR_MODE == MODE_MULTIPROCESSING:
+            elif generator_mode == MODE_MULTIPROCESSING:
                 item = TestCaseParams(test_case, case_dir, log_file, file_mode)
                 all_test_case_params.append(item)
 
-    if GENERATOR_MODE == MODE_MULTIPROCESSING:
+    if generator_mode == MODE_MULTIPROCESSING:
         with Pool(processes=NUM_PROCESS) as pool:
             results = pool.map(worker_function, iter(all_test_case_params))
 

tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py

@@ -5,7 +5,7 @@
 MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS'
 MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING'
 # Test generator mode
-GENERATOR_MODE = MODE_SINGLE_PROCESS
+GENERATOR_MODE = MODE_MULTIPROCESSING
 # Number of subprocesses when using MODE_MULTIPROCESSING
 NUM_PROCESS = multiprocessing.cpu_count() // 2 - 1
 

tests/core/pyspec/eth2spec/gen_helpers/gen_from_tests/gen.py

@@ -10,6 +10,10 @@
 from eth2spec.gen_helpers.gen_base.gen_typing import TestCase, TestProvider
 
 
+def generate_case_fn(tfn, generator_mode, phase, preset, bls_active):
+    return lambda: tfn(generator_mode=generator_mode, phase=phase, preset=preset, bls_active=bls_active)
+
+
 def generate_from_tests(runner_name: str, handler_name: str, src: Any,
                         fork_name: SpecForkName, preset_name: PresetBaseName,
                         bls_active: bool = True,
@@ -52,7 +56,7 @@ def generate_from_tests(runner_name: str, handler_name: str, src: Any,
             suite_name=getattr(tfn, 'suite_name', 'pyspec_tests'),
             case_name=case_name,
             # TODO: with_all_phases and other per-phase tooling, should be replaced with per-fork equivalent.
-            case_fn=lambda: tfn(generator_mode=True, phase=phase, preset=preset_name, bls_active=bls_active)
+            case_fn=generate_case_fn(tfn, generator_mode=True, phase=phase, preset=preset_name, bls_active=bls_active)
         )
 
 

tests/core/pyspec/eth2spec/test/altair/transition/test_transition.py

@@ -83,7 +83,7 @@ def test_normal_transition(state, fork_epoch, spec, post_spec, pre_tag, post_tag
 
 @with_fork_metas([ForkMeta(pre_fork_name=pre, post_fork_name=post, fork_epoch=8) for pre, post in ALL_PRE_POST_FORKS])
 def test_transition_randomized_state(state, fork_epoch, spec, post_spec, pre_tag, post_tag):
-    randomize_state(spec, state)
+    randomize_state(spec, state, rng=random.Random(5566))
 
     transition_until_fork(spec, state, fork_epoch)
 

tests/core/pyspec/eth2spec/test/bellatrix/fork_choice/test_on_merge_block.py

@@ -1,3 +1,5 @@
+from random import Random
+
 from eth2spec.utils.ssz.ssz_typing import uint256
 from eth2spec.test.exceptions import BlockNotFoundException
 from eth2spec.test.context import spec_state_test, with_phases, BELLATRIX
@@ -63,9 +65,9 @@ def test_all_valid(spec, state):
     on_tick_and_append_step(spec, store, current_time, test_steps)
     assert store.time == current_time
 
-    pow_block_parent = prepare_random_pow_block(spec)
+    pow_block_parent = prepare_random_pow_block(spec, rng=Random(1111))
     pow_block_parent.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY - uint256(1)
-    pow_block = prepare_random_pow_block(spec)
+    pow_block = prepare_random_pow_block(spec, rng=Random(2222))
     pow_block.parent_hash = pow_block_parent.block_hash
     pow_block.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY
     pow_blocks = [pow_block, pow_block_parent]
@@ -98,7 +100,7 @@ def test_block_lookup_failed(spec, state):
     on_tick_and_append_step(spec, store, current_time, test_steps)
     assert store.time == current_time
 
-    pow_block = prepare_random_pow_block(spec)
+    pow_block = prepare_random_pow_block(spec, rng=Random(3333))
     pow_block.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY - uint256(1)
     pow_blocks = [pow_block]
     for pb in pow_blocks:
@@ -129,9 +131,9 @@ def test_too_early_for_merge(spec, state):
     on_tick_and_append_step(spec, store, current_time, test_steps)
     assert store.time == current_time
 
-    pow_block_parent = prepare_random_pow_block(spec)
+    pow_block_parent = prepare_random_pow_block(spec, rng=Random(4444))
     pow_block_parent.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY - uint256(2)
-    pow_block = prepare_random_pow_block(spec)
+    pow_block = prepare_random_pow_block(spec, rng=Random(5555))
     pow_block.parent_hash = pow_block_parent.block_hash
     pow_block.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY - uint256(1)
     pow_blocks = [pow_block, pow_block_parent]
@@ -162,9 +164,9 @@ def test_too_late_for_merge(spec, state):
     on_tick_and_append_step(spec, store, current_time, test_steps)
     assert store.time == current_time
 
-    pow_block_parent = prepare_random_pow_block(spec)
+    pow_block_parent = prepare_random_pow_block(spec, rng=Random(6666))
     pow_block_parent.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY
-    pow_block = prepare_random_pow_block(spec)
+    pow_block = prepare_random_pow_block(spec, rng=Random(7777))
     pow_block.parent_hash = pow_block_parent.block_hash
     pow_block.total_difficulty = spec.config.TERMINAL_TOTAL_DIFFICULTY + uint256(1)
     pow_blocks = [pow_block, pow_block_parent]