Skip to content

Commit 3e59f72

Browse files
committed
[monarch] Add bootstrap command hint when proc bootstrap fails
If proc bootstrap (e.g., `monarch/_src/actor/bootstrap_main.py`) fails, provide both the original error message and a hint that indicates that the user might need to look at their proc bootstrap command, and points them to the correct API. Differential Revision: [D85903691](https://our.internmc.facebook.com/intern/diff/D85903691/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D85903691/)! ghstack-source-id: 319926115 Pull Request resolved: #1719
1 parent 77f1e63 commit 3e59f72

File tree

1 file changed

+40
-32
lines changed

1 file changed

+40
-32
lines changed

python/monarch/_src/actor/bootstrap_main.py

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -36,40 +36,48 @@ async def main():
3636

3737

3838
def invoke_main():
39-
# if this is invoked with the stdout piped somewhere, then print
40-
# changes its buffering behavior. So we default to the standard
41-
# behavior of std out as if it were a terminal.
42-
sys.stdout.reconfigure(line_buffering=True)
43-
global bootstrap_main
44-
45-
# TODO: figure out what from worker_main.py we should reproduce here.
46-
47-
from monarch._src.actor.telemetry import TracingForwarder # noqa
48-
49-
if os.environ.get("MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING") == "1":
50-
raise RuntimeError("Error during bootstrap for testing")
51-
52-
# forward logs to rust tracing. Defaults to on.
53-
if os.environ.get("MONARCH_PYTHON_LOG_TRACING", "1") == "1":
54-
# we can stream python logs now; no need to forward them to rust processes
55-
pass
56-
# install opentelemetry tracing
57-
5839
try:
59-
with (
60-
importlib.resources.as_file(
61-
importlib.resources.files("monarch") / "py-spy"
62-
) as pyspy,
63-
):
64-
if pyspy.exists():
65-
os.environ["PYSPY_BIN"] = str(pyspy)
66-
# fallback to using local py-spy
40+
# if this is invoked with the stdout piped somewhere, then print
41+
# changes its buffering behavior. So we default to the standard
42+
# behavior of std out as if it were a terminal.
43+
sys.stdout.reconfigure(line_buffering=True)
44+
global bootstrap_main
45+
46+
# TODO: figure out what from worker_main.py we should reproduce here.
47+
48+
from monarch._src.actor.telemetry import TracingForwarder # noqa
49+
50+
if os.environ.get("MONARCH_ERROR_DURING_BOOTSTRAP_FOR_TESTING") == "1":
51+
raise RuntimeError("Error during bootstrap for testing")
52+
53+
# forward logs to rust tracing. Defaults to on.
54+
if os.environ.get("MONARCH_PYTHON_LOG_TRACING", "1") == "1":
55+
# we can stream python logs now; no need to forward them to rust processes
56+
pass
57+
# install opentelemetry tracing
58+
59+
try:
60+
with (
61+
importlib.resources.as_file(
62+
importlib.resources.files("monarch") / "py-spy"
63+
) as pyspy,
64+
):
65+
if pyspy.exists():
66+
os.environ["PYSPY_BIN"] = str(pyspy)
67+
# fallback to using local py-spy
68+
except Exception as e:
69+
logging.warning(f"Failed to set up py-spy: {e}")
70+
71+
from monarch._src.actor.debugger.breakpoint import remote_breakpointhook
72+
73+
sys.breakpointhook = remote_breakpointhook
6774
except Exception as e:
68-
logging.warning(f"Failed to set up py-spy: {e}")
69-
70-
from monarch._src.actor.debugger.breakpoint import remote_breakpointhook
71-
72-
sys.breakpointhook = remote_breakpointhook
75+
bootstrap_err = RuntimeError(
76+
f"Failed to bootstrap proc due to: {e}\nMake sure your proc bootstrap command is correct. "
77+
f"Provided command:\n{' '.join([sys.executable, *sys.argv])}\nTo specify your proc bootstrap command, use the "
78+
f"`bootstrap_cmd` kwarg in `monarch.actor.host_mesh.HostMesh.allocate_nonblocking(...)`."
79+
)
80+
raise bootstrap_err from e
7381

7482
# Start an event loop for PythonActors to use.
7583
asyncio.run(main())

0 commit comments

Comments
 (0)