List running server health and status

docs/reference/cli-commands.md

@@ -584,6 +584,49 @@ ng_pip_list +entrypoint=resources_servers/example_single_tool_call +format=json
 ng_pip_list +entrypoint=resources_servers/example_single_tool_call +outdated=true
 ```
 
+---
+
+### `ng_status` / `nemo_gym_status`
+
+View all currently running NeMo Gym servers and their health status.
+
+**Example**
+
+```bash
+ng_status
+
+NeMo Gym Server Status:
+
+[1] ✓ example_single_tool_call (resources_servers/example_single_tool_call)
+{
+    'server_type': 'resources_servers',
+    'name': 'example_single_tool_call',
+    'port': 58117,
+    'pid': 89904,
+    'uptime_seconds': '0d 0h 0m 41.5s',
+}
+[2] ✓ example_single_tool_call_simple_agent (responses_api_agents/simple_agent)
+{
+    'server_type': 'responses_api_agents',
+    'name': 'simple_agent',
+    'port': 58118,
+    'pid': 89905,
+    'uptime_seconds': '0d 0h 0m 41.5s',
+}
+[3] ✓ policy_model (responses_api_models/openai_model)
+{
+    'server_type': 'responses_api_models',
+    'name': 'openai_model',
+    'port': 58119,
+    'pid': 89907,
+    'uptime_seconds': '0d 0h 0m 41.5s',
+}
+
+3 servers found (3 healthy, 0 unhealthy)
+
+```
+
+---
 
 ## Getting Help
 

docs/reference/faq.md

@@ -296,6 +296,61 @@ ng_dump_config "+config_paths=[$config_paths]"
 ```
 
 
+# How To: ng_version - Check NeMo Gym version and system information
+Check your NeMo Gym installation version and environment details for troubleshooting or support.
+
+**Standard format:**
+```bash
+ng_version
+```
+
+**Example output:**
+```bash
+NeMo Gym v0.2.0rc0
+Python 3.13.5 (/Users/user/nemo-gym/.venv/bin/python3)
+Installation: /Users/user/nemo-gym
+
+Key Dependencies:
+  openai: 2.6.1
+  ray: 2.50.1
+
+System:
+  OS: Darwin 25.1.0
+  Platform: macOS-26.1-arm64-arm-64bit-Mach-O
+  Architecture: arm64
+  Processor: arm
+  CPUs: 14
+  Memory: 48.0 GB
+```
+
+**Machine-readable JSON format:**
+```bash
+ng_version +json=true
+```
+
+**Example output:**
+```bash
+{
+  "nemo_gym": "0.2.0rc0",
+  "python": "3.13.5",
+  "python_path": "/Users/user/nemo-gym/.venv/bin/python3",
+  "installation_path": "/Users/user/nemo-gym",
+  "dependencies": {
+    "openai": "2.6.1",
+    "ray": "2.50.1"
+  },
+  "system": {
+    "os": "Darwin 25.1.0",
+    "platform": "macOS-26.1-arm64-arm-64bit-Mach-O",
+    "architecture": "arm64",
+    "processor": "arm",
+    "cpus": 14,
+    "memory_gb": 48.0
+  }
+}
+```
+
+
 # How To: Use NeMo Gym with a non-Responses compatible API endpoint like vLLM
 As of Sep 05, 2025, not many models have been trained with middlewares or chat templates that are easily parseable to OpenAI Responses API schema, with the notable exception of OpenAI's own open source model GPT-OSS. Since Gym is first-party Responses API, this makes Gym very difficult to use with basically any model.
 

nemo_gym/cli.py

@@ -27,15 +27,15 @@
 from signal import SIGINT
 from subprocess import Popen
 from threading import Thread
-from time import sleep
+from time import sleep, time
 from typing import Dict, List, Optional, Tuple
 
 import psutil
 import rich
 import uvicorn
 from devtools import pprint
 from omegaconf import DictConfig, OmegaConf
-from pydantic import BaseModel, Field
+from pydantic import Field
 from tqdm.auto import tqdm
 
 from nemo_gym import PARENT_DIR, __version__
@@ -49,10 +49,12 @@
     GlobalConfigDictParserConfig,
     get_global_config_dict,
 )
+from nemo_gym.server_status import StatusCommand
 from nemo_gym.server_utils import (
     HEAD_SERVER_KEY_NAME,
     HeadServer,
     ServerClient,
+    ServerInstanceDisplayConfig,
     ServerStatus,
     initialize_ray,
 )
@@ -146,22 +148,10 @@ def dir_path(self) -> Path:
         return self._dir_path
 
 
-class ServerInstanceDisplayConfig(BaseModel):
-    process_name: str
-    server_type: str
-    name: str
-    dir_path: Path
-    entrypoint: str
-    host: Optional[str] = None
-    port: Optional[int] = None
-    pid: Optional[int] = None
-    config_path: str
-    url: Optional[str] = None
-
-
 class RunHelper:  # pragma: no cover
     _head_server: uvicorn.Server
     _head_server_thread: Thread
+    _head_server_instance: HeadServer
 
     _processes: Dict[str, Popen]
     _server_instance_display_configs: List[ServerInstanceDisplayConfig]
@@ -178,13 +168,15 @@ def start(self, global_config_dict_parser_config: GlobalConfigDictParserConfig)
         escaped_config_dict_yaml_str = shlex.quote(OmegaConf.to_yaml(global_config_dict))
 
         # We always run the head server in this `run` command.
-        self._head_server, self._head_server_thread = HeadServer.run_webserver()
+        self._head_server, self._head_server_thread, self._head_server_instance = HeadServer.run_webserver()
 
         top_level_paths = [k for k in global_config_dict.keys() if k not in NEMO_GYM_RESERVED_TOP_LEVEL_KEYS]
 
         self._processes: Dict[str, Popen] = dict()
         self._server_instance_display_configs: List[ServerInstanceDisplayConfig] = []
 
+        start_time = time()
+
         # TODO there is a better way to resolve this that uses nemo_gym/global_config.py::ServerInstanceConfig
         for top_level_path in top_level_paths:
             server_config_dict = global_config_dict[top_level_path]
@@ -232,9 +224,14 @@ def start(self, global_config_dict_parser_config: GlobalConfigDictParserConfig)
                     url=f"http://{host}:{port}" if host and port else None,
                     pid=process.pid,
                     config_path=top_level_path,
+                    start_time=start_time,
                 )
             )
 
+        self._head_server_instance.set_server_instances(
+            [inst.model_dump(mode="json") for inst in self._server_instance_display_configs]
+        )
+
         self._server_client = ServerClient(
             head_server_config=ServerClient.load_head_server_config(),
             global_config_dict=global_config_dict,
@@ -267,7 +264,7 @@ def display_server_instance_info(self) -> None:
 
         for i, inst in enumerate(self._server_instance_display_configs, 1):
             print(f"[{i}] {inst.process_name} ({inst.server_type}/{inst.name})")
-            pprint(inst.model_dump(mode="json"))
+            pprint(inst.model_dump(mode="json", exclude={"start_time", "status", "uptime_seconds"}))
         print(f"{'#' * 100}\n")
 
     def poll(self) -> None:
@@ -791,6 +788,15 @@ def display_help():  # pragma: no cover
         print(script)
 
 
+def status():  # pragma: no cover
+    global_config_dict = get_global_config_dict()
+    BaseNeMoGymCLIConfig.model_validate(global_config_dict)
+
+    status_cmd = StatusCommand()
+    servers = status_cmd.discover_servers()
+    status_cmd.display_status(servers)
+
+
 class PipListConfig(RunConfig):
     format: Optional[str] = Field(
         default=None,

nemo_gym/dataset_orchestrator.py

@@ -1,10 +1,11 @@
-# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

nemo_gym/hf_utils.py

@@ -1,10 +1,11 @@
-# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,

nemo_gym/server_status.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from time import time
+from typing import List
+
+import requests
+from devtools import pprint
+
+from nemo_gym.server_utils import ServerClient, ServerInstanceDisplayConfig, ServerStatus
+
+
+class StatusCommand:
+    """Main class to check server status"""
+
+    def check_health(self, server_info: ServerInstanceDisplayConfig) -> ServerStatus:
+        """Check if server is responding"""
+        if not server_info.url:
+            return "unknown_error"
+
+        try:
+            requests.get(server_info.url, timeout=2)
+            return "success"
+        except requests.exceptions.ConnectionError:
+            return "connection_error"
+        except requests.exceptions.Timeout:
+            return "timeout"
+        except Exception:
+            return "unknown_error"
+
+    def discover_servers(self) -> List[ServerInstanceDisplayConfig]:
+        """Find all running NeMo Gym server processes"""
+
+        try:
+            head_server_config = ServerClient.load_head_server_config()
+            head_url = f"http://{head_server_config.host}:{head_server_config.port}"
+
+            response = requests.get(f"{head_url}/server_instances", timeout=5)
+            response.raise_for_status()
+            instances = response.json()
+
+            servers = []
+            current_time = time()
+
+            for inst in instances:
+                uptime = current_time - inst.get("start_time", current_time)
+                server_info = ServerInstanceDisplayConfig(
+                    process_name=inst["process_name"],
+                    server_type=inst["server_type"],
+                    name=inst["name"],
+                    host=inst.get("host"),
+                    port=inst.get("port"),
+                    url=inst.get("url"),
+                    entrypoint=inst.get("entrypoint"),
+                    pid=inst.get("pid"),
+                    uptime_seconds=uptime,
+                    status="unknown_error",
+                )
+                server_info.status = self.check_health(server_info)
+                servers.append(server_info)
+
+            return servers
+
+        except (requests.RequestException, ConnectionError) as e:
+            print(f"""
+Could not connect to head server: {e}
+Is the head server running? Start it with: `ng_run`
+            """)
+            return []
+
+    def display_status(self, servers: List[ServerInstanceDisplayConfig]) -> None:
+        """Show server info in a table"""
+
+        def format_uptime(uptime_seconds: float) -> str:
+            """Format uptime in a human readable format"""
+            minutes, seconds = divmod(uptime_seconds, 60)
+            hours, minutes = divmod(minutes, 60)
+            days, hours = divmod(hours, 24)
+            return f"{int(days)}d {int(hours)}h {int(minutes)}m {seconds:.1f}s"
+
+        if not servers:
+            print("No NeMo Gym servers found running.")
+            return
+
+        print("\nNeMo Gym Server Status:\n")
+
+        for i, server in enumerate(servers, 1):
+            status_icon = "✓" if server.status == "success" else "✗"
+            print(f"[{i}] {status_icon} {server.process_name} ({server.server_type}/{server.name})")
+            display_dict = {
+                "server_type": server.server_type,
+                "name": server.name,
+                "port": server.port,
+                "pid": server.pid,
+                "uptime_seconds": format_uptime(server.uptime_seconds),
+            }
+            pprint(display_dict)
+
+        healthy_count = sum(1 for s in servers if s.status == "success")
+        print(f"""
+{len(servers)} servers found ({healthy_count} healthy, {len(servers) - healthy_count} unhealthy)
+""")