diff --git a/tests/compile/fullgraph/test_simple.py b/tests/compile/fullgraph/test_simple.py index ed9c7a351e42..f1ea0e414d76 100644 --- a/tests/compile/fullgraph/test_simple.py +++ b/tests/compile/fullgraph/test_simple.py @@ -161,7 +161,14 @@ def _run_simple_model( @pytest.mark.parametrize("intermediate_unbacked", [True, False]) @torch.inference_mode() @create_new_process_for_each_test("spawn") -def test_simple_piecewise_compile(backend, intermediate_unbacked): +def test_simple_piecewise_compile(backend, intermediate_unbacked, monkeypatch): + # `intermediate_unbacked` flips a control-flow branch inside + # `SillyModel.forward`, but the AOT-compile cache key only hashes the + # forward function's qualname + line number, so both parametrize variants + # share the same cache slot. Disabling the cache forces each variant to + # compile fresh; otherwise the second-running variant loads the first's + # artifact and segfaults with an illegal memory access. + monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1") _run_simple_model( splitting_ops=["silly::attention"], use_inductor_graph_partition=False, diff --git a/tests/utils.py b/tests/utils.py index e8fd3f1e8152..07e3c79b914b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1511,43 +1511,72 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: return wrapper +# Set on the spawn-child interpreter so the wrapper short-circuits when the +# child resolves `module.qualname` back to its own decorated form, instead of +# launching another subprocess. +_SPAWN_CHILD_ENV = "VLLM_TEST_SPAWN_CHILD" + + def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]: """Decorator to spawn a new process for each test function. - Uses subprocess with cloudpickle to serialize the test function and - propagates exceptions back to the parent, so test failures are never - silently swallowed (fixes https://github.com/vllm-project/vllm/issues/41415). + Uses subprocess to run each test in a fresh interpreter and propagates + exceptions back to the parent, so test failures are never silently + swallowed (fixes https://github.com/vllm-project/vllm/issues/41415). + + The child resolves the test function by importing its module and looking + it up by qualified name, rather than reconstructing it from a cloudpickle + blob. Pickling the function by value would also pickle its ``__globals__`` + by value — turning module-level singletons (e.g. + ``vllm.compilation.counter.compilation_counter``) into stale clones in + the child, so increments performed by the production code in the child + would never be observable to the test. """ @functools.wraps(f) def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: + if os.environ.get(_SPAWN_CHILD_ENV) == "1": + return f(*args, **kwargs) + with tempfile.NamedTemporaryFile(delete=False, suffix=".tb", mode="wb") as tmp: tb_file = tmp.name try: - # Serialize the function + args with cloudpickle so closures work - payload = cloudpickle.dumps((f, args, kwargs, tb_file)) + payload = cloudpickle.dumps( + { + "module": f.__module__, + "qualname": f.__qualname__, + "args": args, + "kwargs": kwargs, + "tb_file": tb_file, + } + ) child_script = ( - "import sys, cloudpickle, traceback\n" + "import sys, importlib, cloudpickle, traceback\n" "try:\n" " from _pytest.outcomes import Skipped\n" "except ImportError:\n" " class Skipped(BaseException): pass\n" - "f, args, kwargs, tb_file = " - "cloudpickle.loads(sys.stdin.buffer.read())\n" + "data = cloudpickle.loads(sys.stdin.buffer.read())\n" + "mod = importlib.import_module(data['module'])\n" + "target = mod\n" + "for name in data['qualname'].split('.'):\n" + " target = getattr(target, name)\n" "try:\n" - " f(*args, **kwargs)\n" + " target(*data['args'], **data['kwargs'])\n" "except Skipped:\n" " sys.exit(0)\n" "except BaseException:\n" - " open(tb_file, 'w').write(traceback.format_exc())\n" + " with open(data['tb_file'], 'w') as fp:\n" + " fp.write(traceback.format_exc())\n" " sys.exit(1)\n" ) repo_root = str(VLLM_PATH.resolve()) env = os.environ.copy() env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "") + env[_SPAWN_CHILD_ENV] = "1" result = subprocess.run( [sys.executable, "-c", child_script], @@ -1557,12 +1586,14 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: ) if result.returncode != 0: - # Read traceback written by child, fall back to stderr - tb = "" - if os.path.exists(tb_file) and os.path.getsize(tb_file) > 0: + # Prefer the child's traceback file; fall back to stderr if + # the child crashed before its except handler ran. + try: with open(tb_file) as fp: tb = fp.read() - else: + except OSError: + tb = "" + if not tb: tb = result.stderr.decode() raise RuntimeError( f"Test subprocess '{f.__name__}' failed "