Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion adapters/claude/src/nemo_fabric_adapters/claude/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from claude_agent_sdk import HookMatcher
from claude_agent_sdk._errors import MessageParseError
from nemo_fabric_adapters.common import lifecycle
from nemo_fabric_adapters.common import relay_artifacts
from nemo_fabric_adapters.common import relay_gateway
from nemo_fabric_adapters.common import relay_hooks
from nemo_fabric_adapters.common import utils as common_utils
Expand Down Expand Up @@ -882,7 +883,7 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
if self._unusable:
return _failure(
"claude_runtime_unavailable",
"Claude runtime cannot accept another invocation after an SDK failure",
"Claude runtime cannot accept another invocation after a runtime failure",
)

try:
Expand All @@ -891,12 +892,38 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
except ClaudeAdapterError as error:
output = adapter_failure(error)
else:
relay = self._relay
atif_before = (
relay_artifacts.snapshot_atif_paths(relay.plugin_config)
if relay is not None
and relay_artifacts.expects_local_atif(relay.plugin_config)
else None
)
output = await self._run_query(
payload,
client,
prompt,
invocation_timeout,
)
if (
output.get("completed")
and relay is not None
and atif_before is not None
):
finalized = await relay_artifacts.wait_for_finalized_atif(
relay.plugin_config, atif_before
)
if finalized is None:
self._unusable = True
output = adapter_failure(
AdapterRelayError(
"claude_relay_atif_timeout",
"NeMo Relay did not finalize an ATIF artifact before the deadline",
metadata={
"timeout_seconds": relay_artifacts.ATIF_FINALIZATION_TIMEOUT_SECONDS,
},
)
)

if self._relay is not None:
output = _relay_output(output, self._relay)
Expand Down
29 changes: 28 additions & 1 deletion adapters/codex/src/nemo_fabric_adapters/codex/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import nemo_fabric_adapters.common.relay_gateway as relay_gateway
import nemo_fabric_adapters.common.relay_hooks as relay_hooks
import nemo_fabric_adapters.common.relay_artifacts as relay_artifacts
import nemo_fabric_adapters.common.utils as common_utils
from nemo_fabric_adapters.common import lifecycle

Expand Down Expand Up @@ -1066,7 +1067,7 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
if self._unusable:
output = _failure(
"codex_runtime_unavailable",
"Codex runtime cannot accept another invocation after an SDK failure",
"Codex runtime cannot accept another invocation after a runtime failure",
)
return _relay_output(output, self._relay) if self._relay else output
Comment thread
AjayThorve marked this conversation as resolved.
Resolved

Expand All @@ -1075,7 +1076,33 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
timeout_seconds(payload)
_reasoning_effort(payload)
_output_schema(payload)
relay = self._relay
atif_before = (
relay_artifacts.snapshot_atif_paths(relay.plugin_config)
if relay is not None
and relay_artifacts.expects_local_atif(relay.plugin_config)
else None
)
output, usable = await _invoke_thread(payload, self._thread)
if (
output.get("completed")
and relay is not None
and atif_before is not None
):
finalized = await relay_artifacts.wait_for_finalized_atif(
relay.plugin_config, atif_before
)
if finalized is None:
raise AdapterRelayError(
"codex_relay_atif_timeout",
"NeMo Relay did not finalize an ATIF artifact before the deadline",
metadata={
"timeout_seconds": relay_artifacts.ATIF_FINALIZATION_TIMEOUT_SECONDS,
},
)
except AdapterRelayError as error:
output = adapter_failure(error)
usable = False
except CodexAdapterError as error:
output = adapter_failure(error)
usable = True
Expand Down
93 changes: 93 additions & 0 deletions adapters/common/src/nemo_fabric_adapters/common/relay_artifacts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Readiness checks for artifacts written asynchronously by NeMo Relay."""

from __future__ import annotations

import asyncio
import json
from collections.abc import Collection
from pathlib import Path
from typing import Any

from nemo_fabric_adapters.common import utils as common_utils

ATIF_FINALIZATION_TIMEOUT_SECONDS = 5.0
ATIF_POLL_INTERVAL_SECONDS = 0.05


def expects_local_atif(plugin_config: dict[str, Any]) -> bool:
"""Return whether Relay is configured to write ATIF to the local runtime.

Relay treats a non-empty storage list as remote-only, so there is no local
artifact for an adapter to await in that configuration.
"""

for component in plugin_config.get("components", []):
if (
not isinstance(component, dict)
or component.get("kind") != "observability"
or component.get("enabled", True) is False
):
continue
config = component.get("config")
if not isinstance(config, dict):
continue
atif = config.get("atif")
if isinstance(atif, dict) and atif.get("enabled") and not atif.get("storage"):
return True
return False


def snapshot_atif_paths(plugin_config: dict[str, Any]) -> frozenset[Path]:
"""Capture ATIF paths that existed before an adapter invocation.

Relay requires ``{session_id}`` in the ATIF filename template and creates a
new session scope for each turn. Existing paths therefore cannot satisfy a
later invocation, even if another process modifies them.
"""

return frozenset(
Path(artifact["path"])
for artifact in common_utils.collect_relay_artifacts(plugin_config)
if artifact.get("kind") == "atif"
)


def _finalized_atif_path(
plugin_config: dict[str, Any], before: Collection[Path]
) -> Path | None:
"""Find a newly created ATIF path containing a complete JSON object."""

current = snapshot_atif_paths(plugin_config)
for path in sorted(current.difference(before)):
try:
# Relay writes directly to the final path, so existence alone does
Comment thread
AjayThorve marked this conversation as resolved.
Resolved
# not prove that the JSON payload has been written completely.
document = json.loads(path.read_bytes())
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
continue
if isinstance(document, dict):
return path
return None
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved


async def wait_for_finalized_atif(
plugin_config: dict[str, Any],
before: Collection[Path],
*,
timeout_seconds: float = ATIF_FINALIZATION_TIMEOUT_SECONDS,
poll_interval_seconds: float = ATIF_POLL_INTERVAL_SECONDS,
) -> Path | None:
"""Wait for one new, complete ATIF file until a monotonic deadline."""

loop = asyncio.get_running_loop()
deadline = loop.time() + timeout_seconds
while True:
if path := _finalized_atif_path(plugin_config, before):
return path
remaining = deadline - loop.time()
if remaining <= 0:
return None
await asyncio.sleep(min(poll_interval_seconds, remaining))
148 changes: 148 additions & 0 deletions tests/adapters/test_adapters_common_relay_artifacts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

import asyncio
import json
from pathlib import Path
from typing import Any

import pytest
from nemo_fabric_adapters.common import relay_artifacts


def atif_plugin_config(
output_directory: Path,
*,
component_enabled: bool = True,
storage: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
atif: dict[str, Any] = {
"enabled": True,
"output_directory": str(output_directory),
"filename_template": "trajectory-{session_id}.atif.json",
}
if storage is not None:
atif["storage"] = storage
return {
"version": 1,
"components": [
{
"kind": "observability",
"enabled": component_enabled,
"config": {"atif": atif},
}
],
}


def test_expects_local_atif_requires_enabled_local_output(tmp_path):
local = atif_plugin_config(tmp_path / "local")
disabled = atif_plugin_config(tmp_path / "disabled", component_enabled=False)
remote = atif_plugin_config(
tmp_path / "remote",
storage=[{"type": "http", "endpoint": "https://example.test/atif"}],
)

assert relay_artifacts.expects_local_atif(local) is True
assert relay_artifacts.expects_local_atif(disabled) is False
assert relay_artifacts.expects_local_atif(remote) is False
assert relay_artifacts.expects_local_atif({"components": []}) is False


async def test_wait_for_finalized_atif_ignores_existing_and_partial_files(
tmp_path, monkeypatch
):
atif_dir = tmp_path / "atif"
atif_dir.mkdir()
existing = atif_dir / "trajectory-existing.atif.json"
candidate = atif_dir / "trajectory-current.atif.json"
existing.write_text('{"schema_version":"ATIF-v1.7"}', encoding="utf-8")
plugin_config = atif_plugin_config(atif_dir)
before = relay_artifacts.snapshot_atif_paths(plugin_config)
reads: list[Path] = []
read_bytes = Path.read_bytes

def record_read(path: Path) -> bytes:
reads.append(path)
return read_bytes(path)

monkeypatch.setattr(Path, "read_bytes", record_read)

async def finish_candidate():
existing.write_text('{"schema_version":"changed"}', encoding="utf-8")
candidate.write_text("{", encoding="utf-8")
await asyncio.sleep(0.02)
candidate.write_text(
json.dumps({"schema_version": "ATIF-v1.7", "steps": []}),
encoding="utf-8",
)

writer = asyncio.create_task(finish_candidate())
finalized = await relay_artifacts.wait_for_finalized_atif(
plugin_config,
before,
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)
await writer

assert before == frozenset({existing.resolve()})
assert finalized == candidate.resolve()
assert set(reads) == {candidate.resolve()}


async def test_wait_for_finalized_atif_has_a_hard_deadline(tmp_path):
atif_dir = tmp_path / "atif"
atif_dir.mkdir()
plugin_config = atif_plugin_config(atif_dir)

finalized = await relay_artifacts.wait_for_finalized_atif(
plugin_config,
relay_artifacts.snapshot_atif_paths(plugin_config),
timeout_seconds=0.01,
poll_interval_seconds=0.001,
)

assert finalized is None


async def test_wait_for_finalized_atif_isolated_by_runtime_directory(tmp_path):
first_dir = tmp_path / "runtime-1"
second_dir = tmp_path / "runtime-2"
first_dir.mkdir()
second_dir.mkdir()
first_config = atif_plugin_config(first_dir)
second_config = atif_plugin_config(second_dir)

async def write_atif(directory: Path, session_id: str, delay: float):
await asyncio.sleep(delay)
path = directory / f"trajectory-{session_id}.atif.json"
path.write_text(
json.dumps({"schema_version": "ATIF-v1.7", "steps": []}),
encoding="utf-8",
)
return path.resolve()

first_writer = asyncio.create_task(write_atif(first_dir, "first", 0.02))
second_writer = asyncio.create_task(write_atif(second_dir, "second", 0.01))
first_wait = relay_artifacts.wait_for_finalized_atif(
first_config,
relay_artifacts.snapshot_atif_paths(first_config),
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)
second_wait = relay_artifacts.wait_for_finalized_atif(
second_config,
relay_artifacts.snapshot_atif_paths(second_config),
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)

first_finalized, second_finalized, first_path, second_path = await asyncio.gather(
first_wait, second_wait, first_writer, second_writer
)

assert first_finalized == first_path
assert second_finalized == second_path
Loading
Loading