fix(config): actionable error for unknown server cross-references

nemo_gym/config_types.py

@@ -136,6 +136,10 @@ def is_server_ref(config_dict: DictConfig) -> Optional[ServerRef]:
         return None
 
 
+class ServerRefNotFoundError(ValueError):
+    """A server cross-reference points to an instance that is not defined in the merged config."""
+
+
 ########################################
 # Dataset configs for handling and upload/download
 ########################################

nemo_gym/global_config.py

@@ -15,6 +15,7 @@
 from argparse import ArgumentParser
 from collections import defaultdict
 from copy import deepcopy
+from difflib import get_close_matches
 from importlib import import_module
 from os import environ, getenv
 from pathlib import Path
@@ -36,6 +37,7 @@
 from nemo_gym import CACHE_DIR, PARENT_DIR, RESULTS_DIR, WORKING_DIR
 from nemo_gym.config_types import (
     ServerInstanceConfig,
+    ServerRefNotFoundError,
     WANDBConfig,
     is_almost_server,
     is_server_ref,
@@ -253,14 +255,23 @@ def validate_and_populate_defaults(
             run_server_config_dict = server_instance_config.get_inner_run_server_config_dict()
 
             # Check server refs
-            for v in run_server_config_dict.values():
+            for field_name, v in run_server_config_dict.items():
                 maybe_server_ref = is_server_ref(v)
                 if not maybe_server_ref:
                     continue
 
-                assert maybe_server_ref in server_refs, (
-                    f"Could not find {maybe_server_ref} in the list of available servers: {server_refs}"
-                )
+                if maybe_server_ref not in server_refs:
+                    same_type_names = [ref.name for ref in server_refs if ref.type == maybe_server_ref.type]
+                    suggestions = get_close_matches(maybe_server_ref.name, same_type_names, n=3, cutoff=0.6)
+                    if suggestions:
+                        hint = "Did you mean: " + ", ".join(repr(s) for s in suggestions) + "?"
+                    else:
+                        available = ", ".join(repr(n) for n in sorted(same_type_names)) or "(none)"
+                        hint = f"Available {maybe_server_ref.type}: {available}"
+                    raise ServerRefNotFoundError(
+                        f"""In server instance '{server_instance_config.name}', field '{field_name}' references {maybe_server_ref.type}/'{maybe_server_ref.name}', which is not defined in the merged config.
+{hint}"""
+                    )
 
             # Populate the host and port values if they are not present in the config.
             with open_dict(run_server_config_dict):

tests/unit_tests/test_global_config.py

@@ -23,6 +23,7 @@
 import nemo_gym.global_config
 import nemo_gym.server_utils
 from nemo_gym import CACHE_DIR, WORKING_DIR
+from nemo_gym.config_types import ServerRefNotFoundError
 from nemo_gym.global_config import (
     DEFAULT_HEAD_SERVER_PORT,
     NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME,
@@ -369,9 +370,15 @@ def hydra_main_wrapper(fn):
         hydra_main_mock.return_value = hydra_main_wrapper
         monkeypatch.setattr(nemo_gym.global_config.hydra, "main", hydra_main_mock)
 
-        with raises(AssertionError):
+        with raises(ServerRefNotFoundError) as exc_info:
             get_global_config_dict()
 
+        # The error should name the offending instance, the field, and the missing ref.
+        message = str(exc_info.value)
+        assert "agent_name" in message
+        assert "'d'" in message
+        assert "resources_servers/'resources_name'" in message
+
     def test_get_global_config_dict_server_refs_errors_on_wrong_type(self, monkeypatch: MonkeyPatch) -> None:
         # Clear any lingering env vars.
         monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False)
@@ -420,9 +427,62 @@ def hydra_main_wrapper(fn):
         hydra_main_mock.return_value = hydra_main_wrapper
         monkeypatch.setattr(nemo_gym.global_config.hydra, "main", hydra_main_mock)
 
-        with raises(AssertionError):
+        with raises(ServerRefNotFoundError):
+            get_global_config_dict()
+
+    def test_get_global_config_dict_server_refs_suggests_close_match(self, monkeypatch: MonkeyPatch) -> None:
+        # Clear any lingering env vars.
+        monkeypatch.delenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, raising=False)
+        monkeypatch.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", None)
+
+        exists_mock = MagicMock()
+        exists_mock.return_value = False
+        monkeypatch.setattr(nemo_gym.global_config.Path, "exists", exists_mock)
+
+        find_open_port_mock = MagicMock()
+        find_open_port_mock.return_value = 12345
+        monkeypatch.setattr(nemo_gym.global_config, "find_open_port", find_open_port_mock)
+
+        hydra_main_mock = MagicMock()
+
+        # The agent references "resource" but the defined resources server is "resources" — a typo.
+        def hydra_main_wrapper(fn):
+            config_dict = DictConfig(
+                {
+                    "agent_name": {
+                        "responses_api_agents": {
+                            "agent_type": {
+                                "entrypoint": "app.py",
+                                "resources_server": {
+                                    "type": "resources_servers",
+                                    "name": "resource",
+                                },
+                            }
+                        }
+                    },
+                    "resources": {
+                        "resources_servers": {
+                            "c": {
+                                "entrypoint": "app.py",
+                                "domain": "other",
+                            }
+                        }
+                    },
+                }
+            )
+            return lambda: fn(config_dict)
+
+        hydra_main_mock.return_value = hydra_main_wrapper
+        monkeypatch.setattr(nemo_gym.global_config.hydra, "main", hydra_main_mock)
+
+        with raises(ServerRefNotFoundError) as exc_info:
             get_global_config_dict()
 
+        message = str(exc_info.value)
+        # Fuzzy match should suggest the correctly-spelled resources server, scoped to the same type.
+        assert "Did you mean" in message
+        assert "'resources'" in message
+
     def test_get_first_server_config_dict(self) -> None:
         global_config_dict = DictConfig(
             {