refactor(tests),feat(test-forks): add filter_combinations pytest marker to avoid runtime skips

packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py

@@ -487,6 +487,17 @@ def pytest_configure(config: pytest.Config) -> None:
             "function values_fn(fork)"
         ),
     )
+    config.addinivalue_line(
+        "markers",
+        (
+            "filter_combinations(predicate, *, reason): deselect "
+            "parametrized test cases whose parameter combination does not "
+            "satisfy predicate. The predicate receives parameter values as "
+            "keyword arguments and must return True to keep the "
+            "combination. *reason* is a short human-readable explanation "
+            "shown in verbose collection output."
+        ),
+    )
     for d in fork_covariant_decorators:
         config.addinivalue_line("markers", f"{d.marker_name}: {d.description}")
 
@@ -1348,39 +1359,96 @@ def blob_params_changed_at_transition(fork: Fork) -> bool:
     return False
 
 
+def _get_item_params(
+    item: pytest.Item,
+) -> Dict[str, Any] | None:
+    """Return the callspec params for a collected test item, if any."""
+    if hasattr(item, "callspec"):
+        return item.callspec.params
+    if hasattr(item, "params"):
+        return item.params
+    return None
+
+
+def _combination_filter_reason(
+    item: pytest.Item,
+    params: Dict[str, Any],
+) -> str | None:
+    """
+    Return the rejection reason if any ``filter_combinations`` marker
+    rejects *params*, otherwise ``None``.
+    """
+    for marker in item.iter_markers("filter_combinations"):
+        if not marker.args:
+            continue
+        predicate = marker.args[0]
+        if not callable(predicate):
+            pytest.exit(
+                f"filter_combinations predicate for "
+                f"'{item.nodeid}' is not callable: "
+                f"{predicate!r}",
+                returncode=pytest.ExitCode.USAGE_ERROR,
+            )
+        if not predicate(**params):
+            return marker.kwargs.get(
+                "reason", "rejected by filter_combinations"
+            )
+    return None
+
+
 def pytest_collection_modifyitems(
     config: pytest.Config, items: List[pytest.Item]
 ) -> None:
     """
-    Filter tests based on param-level validity markers.
+    Filter tests after parametrization.
 
-    The pytest_generate_tests hook only considers function-level validity
-    markers. This hook runs after parametrization and can access all markers
-    including param-level ones, allowing us to properly filter tests based on
-    param-level valid_from/valid_until markers.
+    Two kinds of filtering are applied:
+
+    1. **Validity markers** — param-level ``valid_from`` / ``valid_until``
+       markers that the ``pytest_generate_tests`` hook cannot see.
+    2. **Combination filters** — the ``filter_combinations`` marker lets
+       test authors reject specific cross-parameter tuples at collection
+       time instead of calling ``pytest.skip()`` at runtime.
     """
-    del config
     items_to_remove = []
+    deselected: List[pytest.Item] = []
+    # function name -> (reason, total, deselected_count)
+    filter_stats: Dict[str, Tuple[str, int, int]] = {}
 
     for i, item in enumerate(items):
-        # Get fork from params if available
-        params = None
-        if hasattr(item, "callspec"):
-            params = item.callspec.params
-        elif hasattr(item, "params"):
-            params = item.params
-
-        if not params or "fork" not in params or params["fork"] is None:
+        params = _get_item_params(item)
+        if not params:
             continue
 
-        fork: Fork = params["fork"]
+        # --- combination filter ---
+        marker = next(item.iter_markers("filter_combinations"), None)
+        if marker is not None:
+            fn_name = item.nodeid.split("[")[0]
+            if fn_name not in filter_stats:
+                reason = marker.kwargs.get(
+                    "reason", "rejected by filter_combinations"
+                )
+                filter_stats[fn_name] = (reason, 0, 0)
+            r, total, dc = filter_stats[fn_name]
+            total += 1
+
+            filter_reason = _combination_filter_reason(item, params)
+            if filter_reason is not None:
+                items_to_remove.append(i)
+                deselected.append(item)
+                dc += 1
+
+            filter_stats[fn_name] = (r, total, dc)
+            if filter_reason is not None:
+                continue
+
+        # --- validity markers ---
+        fork = params.get("fork")
+        if fork is None:
+            continue
 
-        # Get all markers including param-level ones
         markers = item.iter_markers()
 
-        # Calculate valid fork set from all markers
-        # If this raises (e.g., duplicate markers from combining function-level
-        # and param-level), exit immediately with error
         try:
             valid_fork_set = ValidityMarker.get_test_fork_set_from_markers(
                 markers
@@ -1391,10 +1459,37 @@ def pytest_collection_modifyitems(
                 returncode=pytest.ExitCode.USAGE_ERROR,
             )
 
-        # If the fork is not in the valid set, mark for removal
         if fork not in valid_fork_set:
             items_to_remove.append(i)
 
+    # Fail if a filter_combinations predicate eliminated every case
+    # for a test function — the predicate is almost certainly wrong.
+    for fn_name, (reason, total, dc) in filter_stats.items():
+        if total > 0 and dc == total:
+            pytest.exit(
+                f"filter_combinations deselected all {total} "
+                f"parametrizations of {fn_name} "
+                f"(reason: {reason}). "
+                f"Check the predicate logic.",
+                returncode=pytest.ExitCode.USAGE_ERROR,
+            )
+
     # Remove items in reverse order to maintain indices
     for i in reversed(items_to_remove):
         del items[i]
+
+    if deselected:
+        config.hook.pytest_deselected(items=deselected)
+        if config.option.verbose >= 0:
+            writer = config.get_terminal_writer()
+            writer.line("")
+            writer.sep(
+                "-",
+                f"{len(deselected)} deselected by filter_combinations",
+            )
+            for fn_name, (reason, _, dc) in sorted(filter_stats.items()):
+                if dc:
+                    writer.line(f"  {fn_name}: {dc} deselected ({reason})")
+            if config.option.verbose >= 2:
+                for item in deselected:
+                    writer.line(f"    {item.nodeid}")

packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_covariant_markers.py

@@ -482,6 +482,148 @@ def test_fork_covariant_markers(
     )
     result = pytester.runpytest("-c", "pytest-fill.ini")
     result.assert_outcomes(**outcomes)
-    if outcomes["errors"]:
+    if outcomes.get("errors"):
         assert error_string is not None
         assert error_string in "\n".join(result.stdout.lines)
+
+
+# -- filter_combinations marker tests --
+
+
+@pytest.mark.parametrize(
+    "test_function,expected_outcomes,error_string",
+    [
+        pytest.param(
+            """
+            import pytest
+            from execution_testing import Op
+
+            @pytest.mark.with_all_call_opcodes()
+            @pytest.mark.parametrize("value", [0, 1])
+            @pytest.mark.filter_combinations(
+                lambda call_opcode, value, **_: (
+                    "value" in call_opcode.kwargs or value == 0
+                ),
+                reason="opcode does not support value argument",
+            )
+            @pytest.mark.valid_from("Cancun")
+            @pytest.mark.valid_until("Cancun")
+            @pytest.mark.state_test_only
+            def test_case(state_test, call_opcode, value):
+                pass
+            """,
+            # 4 opcodes × 2 values = 8; STATICCALL and DELEGATECALL
+            # lack "value" kwarg → 2 deselected (value=1 for each).
+            {"passed": 6, "failed": 0, "deselected": 2},
+            None,
+            id="filter_combinations_covariant_cross_parametrize",
+        ),
+        pytest.param(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("a", [1, 2, 3])
+            @pytest.mark.parametrize("b", [10, 20])
+            @pytest.mark.filter_combinations(
+                lambda a, b, **_: a + b != 12,
+                reason="a + b must not equal 12",
+            )
+            @pytest.mark.valid_from("Cancun")
+            @pytest.mark.valid_until("Cancun")
+            @pytest.mark.state_test_only
+            def test_case(state_test, a, b):
+                pass
+            """,
+            # 3 × 2 = 6; (2, 10) rejected → 5 passed, 1 deselected.
+            {"passed": 5, "failed": 0, "deselected": 1},
+            None,
+            id="filter_combinations_regular_parametrize_only",
+        ),
+        pytest.param(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("a", [1, 2, 3])
+            @pytest.mark.filter_combinations(
+                lambda a, **_: True,
+                reason="keep all",
+            )
+            @pytest.mark.valid_from("Cancun")
+            @pytest.mark.valid_until("Cancun")
+            @pytest.mark.state_test_only
+            def test_case(state_test, a):
+                pass
+            """,
+            {"passed": 3, "failed": 0, "deselected": 0},
+            None,
+            id="filter_combinations_keeps_all",
+        ),
+        pytest.param(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("a", [1, 2, 3])
+            @pytest.mark.parametrize("b", [10, 20])
+            @pytest.mark.filter_combinations(
+                lambda a, **_: a != 3,
+                reason="exclude a=3",
+            )
+            @pytest.mark.filter_combinations(
+                lambda b, **_: b != 20,
+                reason="exclude b=20",
+            )
+            @pytest.mark.valid_from("Cancun")
+            @pytest.mark.valid_until("Cancun")
+            @pytest.mark.state_test_only
+            def test_case(state_test, a, b):
+                pass
+            """,
+            # 3 × 2 = 6; first marker removes a=3 (2 items),
+            # second removes b=20 (2 of the remaining 4) → 2 passed.
+            {"passed": 2, "failed": 0, "deselected": 4},
+            None,
+            id="filter_combinations_stacked_and_logic",
+        ),
+        pytest.param(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("a", [1, 2])
+            @pytest.mark.filter_combinations(
+                lambda a, **_: False,
+                reason="reject everything",
+            )
+            @pytest.mark.valid_from("Cancun")
+            @pytest.mark.valid_until("Cancun")
+            @pytest.mark.state_test_only
+            def test_case(state_test, a):
+                pass
+            """,
+            {},
+            "filter_combinations deselected all",
+            id="filter_combinations_empty_set_error",
+        ),
+    ],
+)
+def test_filter_combinations(
+    pytester: pytest.Pytester,
+    test_function: str,
+    expected_outcomes: dict,
+    error_string: str | None,
+) -> None:
+    """Test the ``filter_combinations`` marker in an isolated fill session."""
+    pytester.makepyfile(test_function)
+    pytester.copy_example(
+        name="src/execution_testing/cli/pytest_commands/"
+        "pytest_ini_files/pytest-fill.ini"
+    )
+    result = pytester.runpytest("-c", "pytest-fill.ini")
+    if error_string is not None:
+        assert result.ret == pytest.ExitCode.USAGE_ERROR
+        assert error_string in "\n".join(result.stdout.lines)
+        return
+    outcomes = result.parseoutcomes()
+    for key, expected in expected_outcomes.items():
+        assert outcomes.get(key, 0) == expected, (
+            f"{key}: expected {expected}, got {outcomes.get(key, 0)}"
+        )

tests/frontier/create/test_create_suicide_during_init.py

@@ -44,6 +44,12 @@ def __int__(self) -> int:
     "operation",
     [Operation.SUICIDE, Operation.SUICIDE_TO_ITSELF],
 )
+@pytest.mark.filter_combinations(
+    lambda create_opcode, transaction_create, **_: (
+        create_opcode == Op.CREATE or not transaction_create
+    ),
+    reason="transaction_create only valid with CREATE",
+)
 @pytest.mark.eels_base_coverage
 def test_create_suicide_during_transaction_create(
     state_test: StateTestFiller,
@@ -54,9 +60,6 @@ def test_create_suicide_during_transaction_create(
     transaction_create: bool,
 ) -> None:
     """Contract init code calls suicide then measures different metrics."""
-    if create_opcode != Op.CREATE and transaction_create:
-        pytest.skip(f"Excluded: {create_opcode} with transaction_create=True")
-
     sender = pre.fund_eoa()
     contract_deploy = pre.deploy_contract(
         code=Op.CALLDATACOPY(0, 0, Op.CALLDATASIZE)

tests/prague/eip7702_set_code_tx/test_set_code_txs.py

@@ -772,16 +772,18 @@ def test_set_code_to_contract_creator(
     [0, 1],
 )
 @pytest.mark.with_all_call_opcodes
+@pytest.mark.filter_combinations(
+    lambda call_opcode, value, **_: "value" in call_opcode.kwargs
+    or value == 0,
+    reason="opcode does not support value argument",
+)
 def test_set_code_to_self_caller(
     state_test: StateTestFiller,
     pre: Alloc,
     call_opcode: Op,
     value: int,
 ) -> None:
     """Test the executing a self-call in a set-code transaction."""
-    if "value" not in call_opcode.kwargs and value != 0:
-        pytest.skip(f"Call opcode {call_opcode} does not support value")
-
     storage = Storage()
     auth_signer = pre.fund_eoa(auth_account_start_balance)
 
@@ -905,6 +907,11 @@ def test_set_code_max_depth_call_stack(
     "value",
     [0, 1],
 )
+@pytest.mark.filter_combinations(
+    lambda call_opcode, value, **_: "value" in call_opcode.kwargs
+    or value == 0,
+    reason="opcode does not support value argument",
+)
 @pytest.mark.eels_base_coverage
 def test_set_code_call_set_code(
     state_test: StateTestFiller,
@@ -913,9 +920,6 @@ def test_set_code_call_set_code(
     value: int,
 ) -> None:
     """Test the calling a set-code account from another set-code account."""
-    if "value" not in call_opcode.kwargs and value != 0:
-        pytest.skip(f"Call opcode {call_opcode} does not support value")
-
     auth_signer_1 = pre.fund_eoa(auth_account_start_balance)
     storage_1 = Storage()