From a38aa3441514281ad1b8f2ccba0c17f8e7cd274e Mon Sep 17 00:00:00 2001 From: NolenLiang Date: Sun, 16 Aug 2026 07:23:44 +0000 Subject: [PATCH 1/2] feat(gym): route rollouts through an externally managed vLLM router When env.nemo_gym.router_url is set, register each vLLM replica with the router (POST /workers blocks until the router's own health probe of the replica succeeds, so a 200 means registered and routable) and point Gym's policy_base_url at the router instead of the per-replica URL list. Unset router_url keeps the current direct-to-replica behavior. Co-Authored-By: Claude Fable 5 Signed-off-by: aoshen02 --- nemo_rl/environments/nemo_gym.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nemo_rl/environments/nemo_gym.py b/nemo_rl/environments/nemo_gym.py index a341c4acb23..2a549012c08 100644 --- a/nemo_rl/environments/nemo_gym.py +++ b/nemo_rl/environments/nemo_gym.py @@ -11,10 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import math import os import subprocess import sys +import urllib.request from collections import Counter from collections.abc import AsyncGenerator from copy import deepcopy @@ -491,7 +493,21 @@ def _spinup(self) -> None: initial_global_config_dict["policy_api_key"] = ( "dummy_key" # No key necessary for training. ) - initial_global_config_dict["policy_base_url"] = self.cfg["base_urls"] + router_url = initial_global_config_dict.pop("router_url", None) + if router_url: + # POST /workers blocks until the router's own health probe of the + # replica succeeds, so a 200 means registered and routable. + for base_url in filter(None, self.cfg["base_urls"]): + request = urllib.request.Request( + f"{router_url.removesuffix('/v1')}/workers", + data=json.dumps({"url": base_url.removesuffix("/v1")}).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + urllib.request.urlopen(request, timeout=660).close() + initial_global_config_dict["policy_base_url"] = ( + [router_url.rstrip("/") + "/v1"] if router_url else self.cfg["base_urls"] + ) # In multinode runs, Gym-managed service configs must advertise a real node IP # rather than falling back to localhost, or remote workers will connect to # their own loopback interface instead of the actor-hosted service. From f4a0408e8ce1d4a0b06da4782d41377041409f5e Mon Sep 17 00:00:00 2001 From: NolenLiang Date: Sun, 16 Aug 2026 07:56:06 +0000 Subject: [PATCH 2/2] feat(vllm): health endpoint on the async worker HTTP server GET /health lets an external router health-probe a replica while registering it as a dynamic backend. Co-Authored-By: Claude Fable 5 Signed-off-by: aoshen02 --- nemo_rl/models/generation/vllm/vllm_worker_async.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nemo_rl/models/generation/vllm/vllm_worker_async.py b/nemo_rl/models/generation/vllm/vllm_worker_async.py index 4a0a2c02401..4d082ff71fc 100644 --- a/nemo_rl/models/generation/vllm/vllm_worker_async.py +++ b/nemo_rl/models/generation/vllm/vllm_worker_async.py @@ -910,6 +910,11 @@ def _setup_vllm_server(self) -> "tuple[threading.Thread, str, uvicorn.Server]": # e.g. last-run middleware. app = FastAPI() + # vLLM Router uses GET /health while registering a dynamic backend. + @app.get("/health") + async def _health() -> dict[str, str]: + return {"status": "ok"} + app = self._setup_vllm_openai_api_server(app) if self._sparse_refit_receiver is not None: self._sparse_refit_receiver.setup_api_server(app)