Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion nemo_gym/server_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import atexit
import json
import resource
import socket
import sys
import time
from abc import abstractmethod
Expand Down Expand Up @@ -100,14 +101,36 @@ def get_global_aiohttp_client(
return set_global_aiohttp_client(cfg)


class _TCPKeepAliveConnector(TCPConnector):
_KEEPALIVE_IDLE_SECONDS: int = 60

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we parameterize this via GlobalAIOHTTPAsyncClientConfig? and what are the defaults for these values if we don't specify them, can we default to those same defaults or at least display them in a comment here?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea. I moved the three TCP keepalive values (TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT) to GlobalAIOHTTPAsyncClientConfig fields so they're overridable per deployment. Linux kernel defaults are 7200s / 75s / 9 probes, so ~2h15m before TCP gives up. I think that's longer than most benchmarks need, so I used shorter times.

_KEEPALIVE_INTERVAL_SECONDS: int = 10
_KEEPALIVE_PROBES: int = 3

async def _wrap_create_connection(self, *args, **kwargs):
Comment thread
ananthsub marked this conversation as resolved.
Resolved
transport, protocol = await super()._wrap_create_connection(*args, **kwargs)
sock = transport.get_extra_info("socket")
if sock is None:
raise RuntimeError("_TCPKeepAliveConnector: transport has no underlying socket")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

RISK — fail-hard on a keepalive optimization inverts the intended failure mode.

WHAT BREAKS: _wrap_create_connection runs on every new connection. If get_extra_info("socket") ever returns None for a transport type in production, this raise propagates out of client.request(), gets caught by the generic except Exception in request() (line 227), retries MAX_NUM_TRIES, then re-raises — permanently. The condition is deterministic per transport type, so it wouldn't fail one request, it would fail all of them.

BLAST RADIUS: a feature added to prevent stalled rollouts becomes a total-outage switch. Losing keepalive is a degraded state (back to today's behavior); losing all HTTP is an outage. The downside of the guard is strictly worse than the problem it guards against.

FIX: degrade gracefully instead of raising — log/warn once and return transport, protocol without setting sockopts. Keep the loud raise only in the test (assert the warning), not in the hot path. On Linux+asyncio/uvloop with TCP/SSL transports socket is reliably present, so this is defense-in-depth, but the asymmetry (trivial fix, catastrophic failure mode) argues for not raising.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Switched the raise RuntimeError in _wrap_create_connection to a log-once warning that returns the transport unchanged. If a transport ever exposes no underlying socket, we now just skip keepalive on that connection instead of taking down the whole client through the retry loop. Test updated to assert graceful degradation and a single warning per connector.

sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
for opt_name, opt_value in (
("TCP_KEEPIDLE", self._KEEPALIVE_IDLE_SECONDS),
("TCP_KEEPINTVL", self._KEEPALIVE_INTERVAL_SECONDS),
("TCP_KEEPCNT", self._KEEPALIVE_PROBES),
):
opt = getattr(socket, opt_name, None)
if opt is not None:
sock.setsockopt(socket.IPPROTO_TCP, opt, opt_value)
return transport, protocol


def set_global_aiohttp_client(cfg: GlobalAIOHTTPAsyncClientConfig) -> ClientSession: # pragma: no cover
assert not is_global_aiohttp_client_setup(), (
"There is already a global aiohttp client setup. Please refactor your code or call `global_aiohttp_client_exit` if you want to explicitly re-make the client!"
)

num_workers = get_nemo_gym_fastapi_num_workers()
client_session = ClientSession(
connector=TCPConnector(
connector=_TCPKeepAliveConnector(
limit=cfg.global_aiohttp_connector_limit // num_workers,
limit_per_host=cfg.global_aiohttp_connector_limit_per_host // num_workers,
keepalive_timeout=15.0,
Expand Down
65 changes: 65 additions & 0 deletions tests/unit_tests/test_server_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import socket
from unittest.mock import AsyncMock, MagicMock

from aiohttp import TCPConnector
from pytest import MonkeyPatch, raises

import nemo_gym.global_config
Expand All @@ -29,6 +31,7 @@
HeadServer,
ServerClient,
SimpleServer,
_TCPKeepAliveConnector,
initialize_ray,
)

Expand Down Expand Up @@ -228,6 +231,68 @@ def test_initialize_ray_without_address(self, monkeypatch: MonkeyPatch) -> None:
ray_init_mock.assert_called_once_with(ignore_reinit_error=True)
ray_get_runtime_context_mock.assert_called_once()

async def test_TCPKeepAliveConnector_sets_keepalive_sockopts(self, monkeypatch: MonkeyPatch) -> None:
mock_socket = MagicMock()
mock_transport = MagicMock()
mock_transport.get_extra_info.return_value = mock_socket
mock_protocol = MagicMock()

super_wrap_mock = AsyncMock(return_value=(mock_transport, mock_protocol))
monkeypatch.setattr(TCPConnector, "_wrap_create_connection", super_wrap_mock)

connector = _TCPKeepAliveConnector()
try:
transport, protocol = await connector._wrap_create_connection()
finally:
await connector.close()

assert transport is mock_transport
assert protocol is mock_protocol
mock_transport.get_extra_info.assert_called_once_with("socket")
mock_socket.setsockopt.assert_any_call(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
for opt_name, opt_value in (
("TCP_KEEPIDLE", _TCPKeepAliveConnector._KEEPALIVE_IDLE_SECONDS),
("TCP_KEEPINTVL", _TCPKeepAliveConnector._KEEPALIVE_INTERVAL_SECONDS),
("TCP_KEEPCNT", _TCPKeepAliveConnector._KEEPALIVE_PROBES),
):
opt = getattr(socket, opt_name, None)
if opt is not None:
mock_socket.setsockopt.assert_any_call(socket.IPPROTO_TCP, opt, opt_value)

async def test_TCPKeepAliveConnector_skips_missing_platform_sockopts(self, monkeypatch: MonkeyPatch) -> None:
mock_socket = MagicMock()
mock_transport = MagicMock()
mock_transport.get_extra_info.return_value = mock_socket
mock_protocol = MagicMock()

super_wrap_mock = AsyncMock(return_value=(mock_transport, mock_protocol))
monkeypatch.setattr(TCPConnector, "_wrap_create_connection", super_wrap_mock)
for opt_name in ("TCP_KEEPIDLE", "TCP_KEEPINTVL", "TCP_KEEPCNT"):
monkeypatch.delattr(socket, opt_name, raising=False)

connector = _TCPKeepAliveConnector()
try:
await connector._wrap_create_connection()
finally:
await connector.close()

mock_socket.setsockopt.assert_called_once_with(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

async def test_TCPKeepAliveConnector_raises_when_no_socket(self, monkeypatch: MonkeyPatch) -> None:
mock_transport = MagicMock()
mock_transport.get_extra_info.return_value = None
mock_protocol = MagicMock()

super_wrap_mock = AsyncMock(return_value=(mock_transport, mock_protocol))
monkeypatch.setattr(TCPConnector, "_wrap_create_connection", super_wrap_mock)

connector = _TCPKeepAliveConnector()
try:
with raises(RuntimeError):
await connector._wrap_create_connection()
finally:
await connector.close()

def test_dry_run_skips_webserver_spinup(self, monkeypatch: MonkeyPatch) -> None:
self._mock_ray_return_value(monkeypatch, True)

Expand Down
Loading