diff --git a/components/src/dynamo/frontend/main.py b/components/src/dynamo/frontend/main.py index 13b60007486b..5a5ee173bfe1 100644 --- a/components/src/dynamo/frontend/main.py +++ b/components/src/dynamo/frontend/main.py @@ -51,6 +51,48 @@ MIN_INITIAL_WORKERS_ENV = "DYN_ROUTER_MIN_INITIAL_WORKERS" +# The frontend's TCP listener can exhaust the default soft RLIMIT_NOFILE (1024 on +# most distros) under high concurrency (see the accept() loop in +# lib/runtime/src/pipeline/network/tcp/server.rs), causing accept() to fail with +# EMFILE. At startup we raise the soft limit toward FRONTEND_FD_LIMIT_TARGET, +# overridable per-deployment via DYN_FRONTEND_FD_LIMIT_TARGET. A non-positive or +# non-integer value disables the raise, so operators can opt out. +FRONTEND_FD_LIMIT_ENV = "DYN_FRONTEND_FD_LIMIT_TARGET" +FRONTEND_FD_LIMIT_TARGET = 8192 + + +def _raise_fd_limit(target: Optional[int] = None) -> None: + """Best-effort: raise the process's soft RLIMIT_NOFILE toward `target` + (default: the DYN_FRONTEND_FD_LIMIT_TARGET env var, else FRONTEND_FD_LIMIT_TARGET), + bounded by the hard limit. A target <= 0 (or a non-integer env value) disables + it; also a no-op if already sufficient, if the Unix-only `resource` module is + unavailable (e.g. Windows), or if the raise is denied.""" + if target is None: + raw = os.getenv(FRONTEND_FD_LIMIT_ENV, str(FRONTEND_FD_LIMIT_TARGET)) + try: + target = int(raw) + except ValueError: + target = 0 + if target <= 0: + return + try: + import resource + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft == resource.RLIM_INFINITY: + return # already unlimited; a finite target would only reduce it + new_soft = target if hard == resource.RLIM_INFINITY else min(target, hard) + if new_soft > soft: + resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard)) + logger.info( + f"Raised RLIMIT_NOFILE soft limit {soft} -> {new_soft} (hard={hard})" + ) + except Exception: + # Best-effort hardening; ignore failures (Windows lacks `resource`, or + # setrlimit may be denied in a restricted environment). + # logger.debug("Could not raise RLIMIT_NOFILE; continuing") + pass + def setup_engine_factory( config: FrontendConfig, @@ -318,6 +360,7 @@ async def graceful_shutdown(runtime: DistributedRuntime) -> None: def main() -> None: """Entry point for the Dynamo frontend CLI.""" + _raise_fd_limit() uvloop.run(async_main()) diff --git a/components/src/dynamo/frontend/tests/test_fd_limit.py b/components/src/dynamo/frontend/tests/test_fd_limit.py new file mode 100644 index 000000000000..791f4a55f28e --- /dev/null +++ b/components/src/dynamo/frontend/tests/test_fd_limit.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Platform-safety tests for the frontend fd-limit hardening (`_raise_fd_limit`). + +`resource` is a Unix-only stdlib module, so the hardening must degrade to a +no-op on platforms without it (e.g. Windows) and must never break startup. +""" + +import builtins + +import pytest + +from dynamo.frontend.main import FRONTEND_FD_LIMIT_TARGET, _raise_fd_limit + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def test_noop_when_resource_missing(monkeypatch): + """Simulate Windows: `import resource` fails -> clean no-op, not ImportError.""" + real_import = builtins.__import__ + + def _no_resource(name, *args, **kwargs): + if name == "resource": + raise ImportError("simulated Windows: no `resource` module") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_resource) + _raise_fd_limit() # must return cleanly + + +def test_swallows_setrlimit_failure(monkeypatch): + """A denied setrlimit (restricted environment) is swallowed best-effort.""" + resource = pytest.importorskip("resource") + monkeypatch.setattr(resource, "getrlimit", lambda _res: (64, 1_000_000)) + + def _deny(*args, **kwargs): + raise ValueError("simulated: not permitted to raise the limit") + + monkeypatch.setattr(resource, "setrlimit", _deny) + _raise_fd_limit() # must return cleanly despite setrlimit raising + + +def test_raises_limit_on_unix(monkeypatch): + """On Unix, the soft limit is lifted toward the target (bounded by hard).""" + resource = pytest.importorskip("resource") + captured = {} + monkeypatch.setattr(resource, "getrlimit", lambda _res: (64, 1_000_000)) + monkeypatch.setattr( + resource, "setrlimit", lambda _res, limits: captured.update(limits=limits) + ) + _raise_fd_limit(target=4096) + assert captured["limits"] == (4096, 1_000_000) + + +def test_noop_when_already_sufficient(monkeypatch): + """No setrlimit call when the soft limit already meets the target.""" + resource = pytest.importorskip("resource") + monkeypatch.setattr( + resource, "getrlimit", lambda _res: (FRONTEND_FD_LIMIT_TARGET, 1_000_000) + ) + + def _fail(*args, **kwargs): + raise AssertionError("setrlimit must not be called when already sufficient") + + monkeypatch.setattr(resource, "setrlimit", _fail) + _raise_fd_limit() # soft already == target -> no setrlimit