Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions adapters/claude/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ by the SDK and is not persisted as a NeMo Fabric artifact.

## Relay Observability

Relay requires a separately installed NeMo Relay 0.6.x CLI on `PATH`; the Python
`nemo-relay` package does not provide the executable. Follow the
Relay requires a separately installed NeMo Relay 0.7.x CLI on `PATH`; the
Python `nemo-relay` package does not provide the executable. Follow the
[NeMo Relay installation instructions](https://docs.nvidia.com/nemo/fabric/getting-started/install#install-nemo-relay).

Enable Relay through the normalized NeMo Fabric configuration:
Expand Down
4 changes: 2 additions & 2 deletions adapters/codex/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,8 @@ Codex state variables, the selected model's `api_key_env`, and explicit

## Relay Integration

Relay requires a separately installed NeMo Relay 0.6.x CLI on `PATH`; the Python
`nemo-relay` package does not provide the executable. Follow the
Relay requires a separately installed NeMo Relay 0.7.x CLI on `PATH`; the
Python `nemo-relay` package does not provide the executable. Follow the
[NeMo Relay installation instructions](https://docs.nvidia.com/nemo/fabric/getting-started/install#install-nemo-relay).

Enable Relay with `FabricConfig.enable_relay(...)`. The adapter starts the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
RELAY_HEALTH_TIMEOUT_SECONDS = 10.0
RELAY_STOP_TIMEOUT_SECONDS = 5.0
RELAY_VERSION_TIMEOUT_SECONDS = 5.0
RELAY_MINIMUM_VERSION = (0, 6, 0)
RELAY_MAXIMUM_VERSION = (0, 7, 0)
RELAY_MINIMUM_VERSION = (0, 7, 0)
RELAY_MAXIMUM_VERSION = (0, 8, 0)


class RelayGatewayError(RuntimeError):
Expand Down Expand Up @@ -96,9 +96,9 @@ def relay_cli_contract(executable: Path) -> RelayCliContract:
raise RelayGatewayError(
"unsupported NeMo Relay CLI version "
f"{'.'.join(str(value) for value in version)}; "
"NeMo Fabric requires >=0.6.0,<0.7.0"
"NeMo Fabric requires >=0.7.0,<0.8.0"
)
return RelayCliContract(version=version, observability_version=2)
return RelayCliContract(version=version, observability_version=3)


def wait_for_relay_gateway(
Expand Down
113 changes: 107 additions & 6 deletions adapters/common/src/nemo_fabric_adapters/common/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from __future__ import annotations

import copy
import glob
import json
import os
Expand Down Expand Up @@ -383,11 +384,112 @@ def collect_relay_artifacts(plugin_config: dict[str, Any]) -> list[dict[str, str
return artifacts


_RELAY_V2_OTLP_ONLY_FIELDS = {
"attribute_mappings",
"capture_content",
"mark_exclude_names",
"mark_projection",
"semantic_selector",
}


def _relay_v3_otlp_endpoint(
section: dict[str, Any], *, endpoint_type: str, section_name: str
) -> dict[str, Any] | None:
if not section.get("enabled"):
return None

endpoint = section.get("endpoint")
if not isinstance(endpoint, str) or not endpoint.strip():
raise ValueError(
f"NeMo Relay observability config version 3 requires an endpoint "
f"for enabled {section_name} export"
)

unsupported = sorted(_RELAY_V2_OTLP_ONLY_FIELDS.intersection(section))
if unsupported:
fields = ", ".join(unsupported)
raise ValueError(
f"NeMo Relay observability config version 3 cannot preserve "
f"{section_name} fields: {fields}"
)

return {
"type": endpoint_type,
**{
key: value
for key, value in section.items()
if key not in {"enabled", "type"}
},
}


def _relay_plugin_config_for_version(
plugin_config: dict[str, Any], observability_version: int
) -> dict[str, Any]:
"""Render Fabric's public Relay config for one CLI without mutating it."""

if observability_version != 3:
raise ValueError(
f"unsupported NeMo Relay observability config version "
f"{observability_version}"
)

rendered = copy.deepcopy(plugin_config)
for component in rendered.get("components", []):
if component.get("kind") != "observability":
continue
config = component.get("config")
if not isinstance(config, dict):
continue

config_version = config.get("version", 2)
if config_version == observability_version:
continue
if config_version != 2:
raise ValueError(
f"cannot render NeMo Relay observability config version "
f"{config_version} for version {observability_version}"
)

legacy_sections = (
("opentelemetry", "full"),
("openinference", "openinference"),
)
endpoints = []
section_present = False
for section_name, endpoint_type in legacy_sections:
section = config.pop(section_name, None)
if section is None:
continue
section_present = True
if not isinstance(section, dict):
raise ValueError(
f"NeMo Relay {section_name} config must be an object"
)
migrated = _relay_v3_otlp_endpoint(
section,
endpoint_type=endpoint_type,
section_name=section_name,
)
if migrated is not None:
endpoints.append(migrated)

config["version"] = 3
if section_present:
config["opentelemetry"] = {
"enabled": bool(endpoints),
"endpoints": endpoints,
}

return rendered


def write_relay_configs(
*,
relay_config: dict[str, Any] | None = None,
plugin_config: dict[str, Any] | None = None,
observability_version: int = 2,
observability_version: int = 3,
) -> tuple[Path | None, Path | None]:
try:
import tomli_w
Expand All @@ -409,13 +511,12 @@ def write_relay_configs(
relay_config_path.write_text(tomli_w.dumps(relay_config), encoding="utf-8")

if plugin_config is not None:
if observability_version != 2:
raise ValueError(
f"unsupported NeMo Relay observability config version {observability_version}"
)
rendered_plugin_config = _relay_plugin_config_for_version(
plugin_config, observability_version
)
plugin_config_path = config_dir / "plugins.toml"
plugin_config_path.write_text(
tomli_w.dumps(plugin_config),
tomli_w.dumps(rendered_plugin_config),
encoding="utf-8",
)

Expand Down
4 changes: 2 additions & 2 deletions docs/integrations/harness/claude.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ gateway is scoped to that single invocation. Claude still resolves its credentia
through the selected mode, and NeMo Fabric does not write authentication values
to Relay configuration or artifacts.

NeMo Fabric supports the external NeMo Relay CLI from `0.6.0` up to, but not
including, `0.7.0`. The Python package named `nemo-relay` does not install this
NeMo Fabric supports the external NeMo Relay CLI from `0.7.0` up to, but not
including, `0.8.0`. The Python package named `nemo-relay` does not install this
CLI. NeMo Fabric owns sidecar supervision, Claude configuration, and upstream
selection. Relay owns the gateway transport and semantic observability pipeline.
4 changes: 2 additions & 2 deletions docs/integrations/harness/codex.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,8 @@ NeMo Fabric supplies runtime-scoped Relay configuration to the SDK. It does not
copy the Codex credential store into Relay configuration or persist credentials
in Relay artifacts.

NeMo Fabric supports the external NeMo Relay CLI from `0.6.0` up to, but not
including, `0.7.0`. The Python package named `nemo-relay` is a separate library
NeMo Fabric supports the external NeMo Relay CLI from `0.7.0` up to, but not
including, `0.8.0`. The Python package named `nemo-relay` is a separate library
dependency and does not install the CLI. NeMo Fabric owns sidecar supervision and
runtime-scoped SDK configuration. Relay owns gateway transport behavior,
including decoding `Content-Encoding` before it constructs managed LLM events.
Expand Down
5 changes: 3 additions & 2 deletions docs/sdk/python.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ variant = review_agent_config(config, github_mcp=True, relay=True)
```

NeMo Relay observability is represented directly in the SDK config's top-level
`relay` block. ATOF uses NeMo Relay 0.6 file and stream sinks:
`relay` block. ATOF uses NeMo Relay file and stream sinks:

```python
from nemo_fabric import (
Expand Down Expand Up @@ -505,7 +505,8 @@ Streaming has the following v0.1 constraints:
warning check.

Claude and Codex streaming use the NeMo Relay `nemo-relay` gateway CLI and require a
stream-sink-capable release from `0.6.0` up to, but not including, `0.7.0`. Follow
release from `0.7.0` up to, but not including, `0.8.0`. At launch, NeMo Fabric
translates its public observability model to Relay's version 3 configuration. Follow

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🟡 Minor | ⚡ Quick win

Use NeMo Relay instead of Relay's, and replace the vague At launch qualifier.

The rest of this page uses NeMo Relay consistently. At launch does not identify which launch.

📝 Proposed wording
-release from `0.7.0` up to, but not including, `0.8.0`. At launch, NeMo Fabric
-translates its public observability model to Relay's version 3 configuration. Follow
+release from `0.7.0` up to, but not including, `0.8.0`. When it starts the gateway,
+NeMo Fabric translates its public observability model to the NeMo Relay version 3
+configuration. Follow

As per coding guidelines: "Use the same term consistently for the same concept."

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@docs/sdk/python.mdx` at line 509, Update the sentence near “translates its
public observability model” to use “NeMo Relay” consistently instead of
“Relay’s,” and replace the vague “At launch” qualifier with wording that
identifies the specific launch or timeframe.

Source: Coding guidelines

the [NeMo Relay CLI installation instructions](../getting-started/install.mdx#install-nemo-relay)
to provision it. Hermes Agent and Deep Agents use their in-process NeMo Relay
integrations.
Comment on lines 507 to 512

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Description: Find remaining NeMo Relay 0.6 references and check the install page range.
set -euo pipefail

# Locate the install page and print its Relay section.
fd -t f 'install.mdx' docs | while IFS= read -r f; do
  echo "== $f =="
  rg -n -C 5 'nemo-relay|NEMO_RELAY_VERSION|0\.6|0\.7' "$f" || true
done

# Any lingering 0.6.x Relay CLI references across docs, examples, code, and manifests.
rg -n -C 2 '0\.6\.0|0\.6\.x|<0\.7' --glob '!**/*.lock' || true

Repository: NVIDIA/NeMo-Fabric

Length of output: 2029


🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "== docs/getting-started/install.mdx: Relay section =="
sed -n '126,186p' docs/getting-started/install.mdx

echo "== References to the install-page anchor =="
rg -n -C 2 'install\.mdx#install-nemo-relay|0\.7\.0|0\.8\.0|0\.6\.[0-9]+' \
  docs examples README.md --glob '!**/*.lock' || true

Repository: NVIDIA/NeMo-Fabric

Length of output: 6660


Add the NeMo Relay CLI version range to the installation guide.

docs/getting-started/install.mdx#install-nemo-relay does not state the required 0.7.0 to <0.8.0 range. Add it so the linked installation guidance matches the adapter documentation.

🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@docs/sdk/python.mdx` around lines 507 - 512, Update the NeMo Relay CLI
installation section in the install guide to explicitly require versions from
0.7.0 inclusive through below 0.8.0, matching the range documented near the
Claude and Codex streaming requirements in the SDK guide.

Expand Down
2 changes: 1 addition & 1 deletion examples/code_review_agent/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ The entrypoint exposes complete harness configs defined in
Add `--relay` to any variant to enable the Relay ATOF and ATIF configuration:

Relay requirements depend on the selected adapter. The Codex and Claude
adapters require an external `nemo-relay` CLI in the supported `0.6.x` range;
adapters require an external `nemo-relay` CLI in the supported 0.7.x range;
the Python package named `nemo-relay` does not install the `nemo-relay` CLI
tool. Hermes Agent and Deep Agents require the Relay Python package in their
selected adapter environment. Refer to the
Expand Down
2 changes: 1 addition & 1 deletion examples/harbor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ version `0.1.0`.
| --- | --- | --- |
| Harbor host | `nemo-fabric[harbor]==0.1.0` | Harbor CLI, `FabricAgent`, and typed `FabricConfig` construction |
| Claude task without Relay | `nemo-fabric[claude]==0.1.0` | NeMo Fabric runner, Claude adapter, and supported Claude harness |
| Claude task with Relay | `nemo-fabric[claude]==0.1.0` plus a NeMo Relay 0.6.x CLI on `PATH` | NeMo Fabric runner, Claude adapter and harness, and the adapter-managed Relay gateway and hooks |
| Claude task with Relay | `nemo-fabric[claude]==0.1.0` plus a NeMo Relay 0.7.x CLI on `PATH` | NeMo Fabric runner, Claude adapter and harness, and the adapter-managed Relay gateway and hooks |
| Hermes Agent task with Relay | `nemo-fabric[hermes-agent,relay]==0.1.0` | NeMo Fabric runner, Hermes Agent adapter and harness, and the NeMo Relay Python package |

The `nemo-fabric` package installs the runtime. The `relay` extra installs the
Expand Down
4 changes: 2 additions & 2 deletions examples/harbor/swebench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ export FABRIC_PACKAGE='nemo-fabric[claude,hermes-agent,relay]==0.1.0'
export RUNS_DIR="$PWD/.tmp/harbor/fabric-swebench"

curl -fsSL https://raw.githubusercontent.com/NVIDIA/NeMo-Relay/main/install.sh |
NEMO_RELAY_VERSION=0.6.0 sh -s -- \
NEMO_RELAY_VERSION=0.7.0 sh -s -- \
--install-dir "$FABRIC_BUNDLE/.relay/bin"
```

The curl command downloads and installs the standalone NeMo Relay 0.6.0 CLI tool
The curl command downloads and installs the standalone NeMo Relay 0.7.0 CLI tool
and verifies its checksum. For other installation methods, refer to the
[NeMo Relay installation instructions](../../../docs/getting-started/install.mdx#install-nemo-relay).

Expand Down
103 changes: 81 additions & 22 deletions tests/adapters/test_adapaters_common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,9 @@ def test_write_relay_configs(
assert tomllib.load(stream) == config


def test_write_relay_configs_preserves_current_cli_contract(tmp_path: Path):
def test_write_relay_configs_migrates_otlp_and_preserves_sinks_for_relay_0_7(
tmp_path: Path,
):
os.environ["FABRIC_RELAY_CONFIG_PATH"] = str(tmp_path / "relay.json")
plugin_config = {
"version": 1,
Expand All @@ -763,8 +765,6 @@ def test_write_relay_configs_preserves_current_cli_contract(tmp_path: Path):
{
"type": "file",
"output_directory": "/tmp/atof",
"filename": "events.jsonl",
"mode": "overwrite",
},
{
"type": "stream",
Expand All @@ -778,43 +778,102 @@ def test_write_relay_configs_preserves_current_cli_contract(tmp_path: Path):
],
},
"atif": {"enabled": True, "output_directory": "/tmp/atif"},
"opentelemetry": {
"enabled": True,
"endpoint": "http://localhost:4318/v1/traces",
"transport": "http_binary",
"headers": {"x-tenant-id": "demo"},
"resource_attributes": {"deployment.environment": "test"},
"service_name": "fabric",
"timeout_millis": 1000,
},
"openinference": {
"enabled": True,
"endpoint": "http://localhost:6006/v1/traces",
"transport": "http_binary",
"service_name": "fabric",
},
},
}
],
}

_, plugin_path = common_utils.write_relay_configs(
plugin_config=plugin_config,
observability_version=2,
observability_version=3,
)

assert plugin_path is not None
with plugin_path.open("rb") as stream:
rendered = tomllib.load(stream)
observability = rendered["components"][0]["config"]
assert observability["version"] == 2
assert observability["atof"] == {
assert observability["version"] == 3
assert observability["atof"] == plugin_config["components"][0]["config"]["atof"]
assert observability["atif"] == plugin_config["components"][0]["config"]["atif"]
assert "openinference" not in observability
assert observability["opentelemetry"] == {
"enabled": True,
"sinks": [
"endpoints": [
{
"type": "file",
"output_directory": "/tmp/atof",
"filename": "events.jsonl",
"mode": "overwrite",
"type": "full",
"endpoint": "http://localhost:4318/v1/traces",
"transport": "http_binary",
"headers": {"x-tenant-id": "demo"},
"resource_attributes": {"deployment.environment": "test"},
"service_name": "fabric",
"timeout_millis": 1000,
},
{
"type": "stream",
"url": "https://example.test/events",
"transport": "http_post",
"headers": {"x-test": "value"},
"header_env": {"authorization": "TOKEN"},
"timeout_millis": 1000,
"field_name_policy": "replace_dots",
"type": "openinference",
"endpoint": "http://localhost:6006/v1/traces",
"transport": "http_binary",
"service_name": "fabric",
},
],
}
assert observability["atif"] == {
"enabled": True,
"output_directory": "/tmp/atif",
assert plugin_config["components"][0]["config"]["version"] == 2
assert "openinference" in plugin_config["components"][0]["config"]


def test_relay_0_7_config_requires_enabled_otlp_endpoint():
plugin_config = {
"components": [
{
"kind": "observability",
"config": {
"version": 2,
"openinference": {"enabled": True},
},
}
]
}
assert rendered == plugin_config

with pytest.raises(
ValueError,
match="version 3 requires an endpoint for enabled openinference export",
):
common_utils._relay_plugin_config_for_version(plugin_config, 3)


def test_relay_0_7_config_rejects_removed_otlp_controls():
plugin_config = {
"components": [
{
"kind": "observability",
"config": {
"version": 2,
"opentelemetry": {
"enabled": True,
"endpoint": "http://localhost:4318/v1/traces",
"mark_projection": "tool",
},
},
}
]
}

with pytest.raises(
ValueError,
match="cannot preserve opentelemetry fields: mark_projection",
):
common_utils._relay_plugin_config_for_version(plugin_config, 3)
Comment on lines +838 to +879

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🔵 Trivial | ⚡ Quick win

Parametrize the two rejection tests and add the missing unsupported-version case.

Both tests share one shape: a plugin configuration and an expected ValueError message. The observability_version != 3 branch at adapters/common/src/nemo_fabric_adapters/common/utils.py Line 430 has no coverage, and neither does the new default of 3 in write_relay_configs.

♻️ Proposed parametrized rewrite
-def test_relay_0_7_config_requires_enabled_otlp_endpoint():
-    plugin_config = {
-        "components": [
-            {
-                "kind": "observability",
-                "config": {
-                    "version": 2,
-                    "openinference": {"enabled": True},
-                },
-            }
-        ]
-    }
-
-    with pytest.raises(
-        ValueError,
-        match="version 3 requires an endpoint for enabled openinference export",
-    ):
-        common_utils._relay_plugin_config_for_version(plugin_config, 3)
-
-
-def test_relay_0_7_config_rejects_removed_otlp_controls():
-    plugin_config = {
-        "components": [
-            {
-                "kind": "observability",
-                "config": {
-                    "version": 2,
-                    "opentelemetry": {
-                        "enabled": True,
-                        "endpoint": "http://localhost:4318/v1/traces",
-                        "mark_projection": "tool",
-                    },
-                },
-            }
-        ]
-    }
-
-    with pytest.raises(
-        ValueError,
-        match="cannot preserve opentelemetry fields: mark_projection",
-    ):
-        common_utils._relay_plugin_config_for_version(plugin_config, 3)
+@pytest.mark.parametrize(
+    ("observability_config", "target_version", "expected_message"),
+    [
+        (
+            {"version": 2, "openinference": {"enabled": True}},
+            3,
+            "version 3 requires an endpoint for enabled openinference export",
+        ),
+        (
+            {
+                "version": 2,
+                "opentelemetry": {
+                    "enabled": True,
+                    "endpoint": "http://localhost:4318/v1/traces",
+                    "mark_projection": "tool",
+                },
+            },
+            3,
+            "cannot preserve opentelemetry fields: mark_projection",
+        ),
+        (
+            {"version": 2},
+            2,
+            "unsupported NeMo Relay observability config version 2",
+        ),
+    ],
+)
+def test_relay_0_7_config_rejects_invalid_observability_config(
+    observability_config, target_version, expected_message
+):
+    plugin_config = {
+        "components": [
+            {"kind": "observability", "config": observability_config}
+        ]
+    }
+
+    with pytest.raises(ValueError, match=expected_message):
+        common_utils._relay_plugin_config_for_version(
+            plugin_config, target_version
+        )

As per coding guidelines: "Prefer pytest.mark.parametrize over separate tests for different input types."

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
def test_relay_0_7_config_requires_enabled_otlp_endpoint():
plugin_config = {
"components": [
{
"kind": "observability",
"config": {
"version": 2,
"openinference": {"enabled": True},
},
}
]
}
assert rendered == plugin_config
with pytest.raises(
ValueError,
match="version 3 requires an endpoint for enabled openinference export",
):
common_utils._relay_plugin_config_for_version(plugin_config, 3)
def test_relay_0_7_config_rejects_removed_otlp_controls():
plugin_config = {
"components": [
{
"kind": "observability",
"config": {
"version": 2,
"opentelemetry": {
"enabled": True,
"endpoint": "http://localhost:4318/v1/traces",
"mark_projection": "tool",
},
},
}
]
}
with pytest.raises(
ValueError,
match="cannot preserve opentelemetry fields: mark_projection",
):
common_utils._relay_plugin_config_for_version(plugin_config, 3)
`@pytest.mark.parametrize`(
("observability_config", "target_version", "expected_message"),
[
(
{"version": 2, "openinference": {"enabled": True}},
3,
"version 3 requires an endpoint for enabled openinference export",
),
(
{
"version": 2,
"opentelemetry": {
"enabled": True,
"endpoint": "http://localhost:4318/v1/traces",
"mark_projection": "tool",
},
},
3,
"cannot preserve opentelemetry fields: mark_projection",
),
(
{"version": 2},
2,
"unsupported NeMo Relay observability config version 2",
),
],
)
def test_relay_0_7_config_rejects_invalid_observability_config(
observability_config, target_version, expected_message
):
plugin_config = {
"components": [
{"kind": "observability", "config": observability_config}
]
}
with pytest.raises(ValueError, match=expected_message):
common_utils._relay_plugin_config_for_version(
plugin_config, target_version
)
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@tests/adapters/test_adapaters_common_utils.py` around lines 827 - 868,
Parametrize the two rejection cases in the test module using shared plugin
configuration and expected ValueError-message inputs, while preserving each
case’s distinct error match. Add a third parametrized case covering an
unsupported observability version (a value other than 3) through
_relay_plugin_config_for_version, and add coverage for write_relay_configs using
its default observability version of 3.

Source: Coding guidelines

Loading
Loading