[CI][Bugfix] Fix failure CI step "PyTorch Fullgraph Smoke Test"

tests/compile/fullgraph/test_simple.py

@@ -161,7 +161,14 @@ def _run_simple_model(
 @pytest.mark.parametrize("intermediate_unbacked", [True, False])
 @torch.inference_mode()
 @create_new_process_for_each_test("spawn")
-def test_simple_piecewise_compile(backend, intermediate_unbacked):
+def test_simple_piecewise_compile(backend, intermediate_unbacked, monkeypatch):
+    # `intermediate_unbacked` flips a control-flow branch inside
+    # `SillyModel.forward`, but the AOT-compile cache key only hashes the
+    # forward function's qualname + line number, so both parametrize variants
+    # share the same cache slot. Disabling the cache forces each variant to
+    # compile fresh; otherwise the second-running variant loads the first's
+    # artifact and segfaults with an illegal memory access.
+    monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
     _run_simple_model(
         splitting_ops=["silly::attention"],
         use_inductor_graph_partition=False,

tests/utils.py

@@ -1511,43 +1511,72 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
     return wrapper
 
 
+# Set on the spawn-child interpreter so the wrapper short-circuits when the
+# child resolves `module.qualname` back to its own decorated form, instead of
+# launching another subprocess.
+_SPAWN_CHILD_ENV = "VLLM_TEST_SPAWN_CHILD"
+
+
 def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
     """Decorator to spawn a new process for each test function.
 
-    Uses subprocess with cloudpickle to serialize the test function and
-    propagates exceptions back to the parent, so test failures are never
-    silently swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
+    Uses subprocess to run each test in a fresh interpreter and propagates
+    exceptions back to the parent, so test failures are never silently
+    swallowed (fixes https://github.com/vllm-project/vllm/issues/41415).
+
+    The child resolves the test function by importing its module and looking
+    it up by qualified name, rather than reconstructing it from a cloudpickle
+    blob. Pickling the function by value would also pickle its ``__globals__``
+    by value — turning module-level singletons (e.g.
+    ``vllm.compilation.counter.compilation_counter``) into stale clones in
+    the child, so increments performed by the production code in the child
+    would never be observable to the test.
     """
 
     @functools.wraps(f)
     def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
+        if os.environ.get(_SPAWN_CHILD_ENV) == "1":
+            return f(*args, **kwargs)
+
         with tempfile.NamedTemporaryFile(delete=False, suffix=".tb", mode="wb") as tmp:
             tb_file = tmp.name
 
         try:
-            # Serialize the function + args with cloudpickle so closures work
-            payload = cloudpickle.dumps((f, args, kwargs, tb_file))
+            payload = cloudpickle.dumps(
+                {
+                    "module": f.__module__,
+                    "qualname": f.__qualname__,
+                    "args": args,
+                    "kwargs": kwargs,
+                    "tb_file": tb_file,
+                }
+            )
 
             child_script = (
-                "import sys, cloudpickle, traceback\n"
+                "import sys, importlib, cloudpickle, traceback\n"
                 "try:\n"
                 "    from _pytest.outcomes import Skipped\n"
                 "except ImportError:\n"
                 "    class Skipped(BaseException): pass\n"
-                "f, args, kwargs, tb_file = "
-                "cloudpickle.loads(sys.stdin.buffer.read())\n"
+                "data = cloudpickle.loads(sys.stdin.buffer.read())\n"
+                "mod = importlib.import_module(data['module'])\n"
+                "target = mod\n"
+                "for name in data['qualname'].split('.'):\n"
+                "    target = getattr(target, name)\n"
                 "try:\n"
-                "    f(*args, **kwargs)\n"
+                "    target(*data['args'], **data['kwargs'])\n"
                 "except Skipped:\n"
                 "    sys.exit(0)\n"
                 "except BaseException:\n"
-                "    open(tb_file, 'w').write(traceback.format_exc())\n"
+                "    with open(data['tb_file'], 'w') as fp:\n"
+                "        fp.write(traceback.format_exc())\n"
                 "    sys.exit(1)\n"
             )
 
             repo_root = str(VLLM_PATH.resolve())
             env = os.environ.copy()
             env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
+            env[_SPAWN_CHILD_ENV] = "1"
 
             result = subprocess.run(
                 [sys.executable, "-c", child_script],
@@ -1557,12 +1586,14 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
             )
 
             if result.returncode != 0:
-                # Read traceback written by child, fall back to stderr
-                tb = ""
-                if os.path.exists(tb_file) and os.path.getsize(tb_file) > 0:
+                # Prefer the child's traceback file; fall back to stderr if
+                # the child crashed before its except handler ran.
+                try:
                     with open(tb_file) as fp:
                         tb = fp.read()
-                else:
+                except OSError:
+                    tb = ""
+                if not tb:
                     tb = result.stderr.decode()
                 raise RuntimeError(
                     f"Test subprocess '{f.__name__}' failed "