Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion adapters/claude/src/nemo_fabric_adapters/claude/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from claude_agent_sdk import HookMatcher
from claude_agent_sdk._errors import MessageParseError
from nemo_fabric_adapters.common import lifecycle
from nemo_fabric_adapters.common import relay_artifacts
from nemo_fabric_adapters.common import relay_gateway
from nemo_fabric_adapters.common import relay_hooks
from nemo_fabric_adapters.common import utils as common_utils
Expand Down Expand Up @@ -882,7 +883,7 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
if self._unusable:
return _failure(
"claude_runtime_unavailable",
"Claude runtime cannot accept another invocation after an SDK failure",
"Claude runtime cannot accept another invocation after a runtime failure",
)

try:
Expand All @@ -891,12 +892,38 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
except ClaudeAdapterError as error:
output = adapter_failure(error)
else:
relay = self._relay
atif_before = (
relay_artifacts.snapshot_atif_files(relay.plugin_config)
if relay is not None
and relay_artifacts.expects_local_atif(relay.plugin_config)
else None
)
output = await self._run_query(
payload,
client,
prompt,
invocation_timeout,
)
if (
output.get("completed")
and relay is not None
and atif_before is not None
):
finalized = await relay_artifacts.wait_for_finalized_atif(
relay.plugin_config, atif_before
)
if finalized is None:
self._unusable = True
output = adapter_failure(
AdapterRelayError(
"claude_relay_atif_timeout",
"NeMo Relay did not finalize an ATIF artifact before the deadline",
metadata={
"timeout_seconds": relay_artifacts.ATIF_FINALIZATION_TIMEOUT_SECONDS,
},
)
)

if self._relay is not None:
output = _relay_output(output, self._relay)
Expand Down
32 changes: 29 additions & 3 deletions adapters/codex/src/nemo_fabric_adapters/codex/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

import nemo_fabric_adapters.common.relay_gateway as relay_gateway
import nemo_fabric_adapters.common.relay_hooks as relay_hooks
import nemo_fabric_adapters.common.relay_artifacts as relay_artifacts
import nemo_fabric_adapters.common.utils as common_utils
from nemo_fabric_adapters.common import lifecycle

Expand Down Expand Up @@ -1064,18 +1065,43 @@ async def invoke(self, invocation: dict[str, Any]) -> dict[str, Any]:
"request": invocation.get("request"),
}
if self._unusable:
output = _failure(
return _failure(
"codex_runtime_unavailable",
"Codex runtime cannot accept another invocation after an SDK failure",
"Codex runtime cannot accept another invocation after a runtime failure",
)
return _relay_output(output, self._relay) if self._relay else output

try:
request_prompt(payload)
timeout_seconds(payload)
_reasoning_effort(payload)
_output_schema(payload)
relay = self._relay
atif_before = (
relay_artifacts.snapshot_atif_files(relay.plugin_config)
if relay is not None
and relay_artifacts.expects_local_atif(relay.plugin_config)
else None
)
output, usable = await _invoke_thread(payload, self._thread)
if (
output.get("completed")
and relay is not None
and atif_before is not None
):
finalized = await relay_artifacts.wait_for_finalized_atif(
relay.plugin_config, atif_before
)
if finalized is None:
raise AdapterRelayError(
"codex_relay_atif_timeout",
"NeMo Relay did not finalize an ATIF artifact before the deadline",
metadata={
"timeout_seconds": relay_artifacts.ATIF_FINALIZATION_TIMEOUT_SECONDS,
},
)
except AdapterRelayError as error:
output = adapter_failure(error)
usable = False
except CodexAdapterError as error:
output = adapter_failure(error)
usable = True
Expand Down
109 changes: 109 additions & 0 deletions adapters/common/src/nemo_fabric_adapters/common/relay_artifacts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Readiness checks for artifacts written asynchronously by NeMo Relay."""

from __future__ import annotations

import asyncio
import json
from pathlib import Path
from typing import Any

from nemo_fabric_adapters.common import utils as common_utils

ATIF_FINALIZATION_TIMEOUT_SECONDS = 5.0
ATIF_POLL_INTERVAL_SECONDS = 0.05
AtifFileFingerprint = tuple[int, int, int, int]
AtifSnapshot = dict[Path, AtifFileFingerprint]


def expects_local_atif(plugin_config: dict[str, Any]) -> bool:
"""Return whether Relay is configured to write ATIF to the local runtime.

Relay treats a non-empty storage list as remote-only, so there is no local
artifact for an adapter to await in that configuration.
"""

for component in plugin_config.get("components", []):
if (
not isinstance(component, dict)
or component.get("kind") != "observability"
or component.get("enabled", True) is False
):
continue
config = component.get("config")
if not isinstance(config, dict):
continue
atif = config.get("atif")
if isinstance(atif, dict) and atif.get("enabled") and not atif.get("storage"):
return True
return False


def _atif_fingerprint(path: Path) -> AtifFileFingerprint | None:
try:
status = path.stat()
except (OSError, RuntimeError, ValueError):
return None
return (status.st_dev, status.st_ino, status.st_size, status.st_mtime_ns)


def snapshot_atif_files(plugin_config: dict[str, Any]) -> AtifSnapshot:
"""Capture ATIF file metadata before an adapter invocation.

Relay requires ``{session_id}`` in the ATIF filename template and creates a
new session scope for each turn, so a new path is the normal case. Metadata
fingerprints also detect a writer that rewrites an existing path without
reading or hashing artifacts from prior turns.
"""

snapshot: AtifSnapshot = {}
for artifact in common_utils.collect_relay_artifacts(plugin_config):
if artifact.get("kind") != "atif":
continue
path = Path(artifact["path"])
fingerprint = _atif_fingerprint(path)
if fingerprint is not None:
snapshot[path] = fingerprint
return snapshot


def _finalized_atif_path(
plugin_config: dict[str, Any], before: AtifSnapshot
) -> Path | None:
"""Find a new or changed ATIF path containing a complete JSON object."""

current = snapshot_atif_files(plugin_config)
for path in sorted(current):
if before.get(path) == current[path]:
continue
try:
# Relay writes directly to the final path, so existence alone does
Comment thread
AjayThorve marked this conversation as resolved.
Resolved
# not prove that the JSON payload has been written completely.
document = json.loads(path.read_bytes())
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
continue
if isinstance(document, dict):
return path
return None


async def wait_for_finalized_atif(
plugin_config: dict[str, Any],
before: AtifSnapshot,
*,
timeout_seconds: float = ATIF_FINALIZATION_TIMEOUT_SECONDS,
poll_interval_seconds: float = ATIF_POLL_INTERVAL_SECONDS,
) -> Path | None:
"""Wait for one new or changed, complete ATIF file until a deadline."""

loop = asyncio.get_running_loop()
deadline = loop.time() + timeout_seconds
while True:
if path := _finalized_atif_path(plugin_config, before):
return path
remaining = deadline - loop.time()
if remaining <= 0:
return None
await asyncio.sleep(min(poll_interval_seconds, remaining))
169 changes: 169 additions & 0 deletions tests/adapters/test_adapters_common_relay_artifacts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

import asyncio
import json
from pathlib import Path
from typing import Any

import pytest
from nemo_fabric_adapters.common import relay_artifacts


def atif_plugin_config(
output_directory: Path,
*,
component_enabled: bool = True,
storage: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
atif: dict[str, Any] = {
"enabled": True,
"output_directory": str(output_directory),
"filename_template": "trajectory-{session_id}.atif.json",
}
if storage is not None:
atif["storage"] = storage
return {
"version": 1,
"components": [
{
"kind": "observability",
"enabled": component_enabled,
"config": {"atif": atif},
}
],
}


def test_expects_local_atif_requires_enabled_local_output(tmp_path):
local = atif_plugin_config(tmp_path / "local")
disabled = atif_plugin_config(tmp_path / "disabled", component_enabled=False)
remote = atif_plugin_config(
tmp_path / "remote",
storage=[{"type": "http", "endpoint": "https://example.test/atif"}],
)

assert relay_artifacts.expects_local_atif(local) is True
assert relay_artifacts.expects_local_atif(disabled) is False
assert relay_artifacts.expects_local_atif(remote) is False
assert relay_artifacts.expects_local_atif({"components": []}) is False


async def test_wait_for_finalized_atif_ignores_unchanged_and_partial_files(
tmp_path, monkeypatch
):
atif_dir = tmp_path / "atif"
atif_dir.mkdir()
existing = atif_dir / "trajectory-existing.atif.json"
candidate = atif_dir / "trajectory-current.atif.json"
existing.write_text('{"schema_version":"ATIF-v1.7"}', encoding="utf-8")
plugin_config = atif_plugin_config(atif_dir)
before = relay_artifacts.snapshot_atif_files(plugin_config)
reads: list[Path] = []
read_bytes = Path.read_bytes

def record_read(path: Path) -> bytes:
reads.append(path)
return read_bytes(path)

monkeypatch.setattr(Path, "read_bytes", record_read)

async def finish_candidate():
candidate.write_text("{", encoding="utf-8")
await asyncio.sleep(0.02)
candidate.write_text(
json.dumps({"schema_version": "ATIF-v1.7", "steps": []}),
encoding="utf-8",
)

writer = asyncio.create_task(finish_candidate())
finalized = await relay_artifacts.wait_for_finalized_atif(
plugin_config,
before,
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)
await writer

assert set(before) == {existing.resolve()}
assert finalized == candidate.resolve()
assert set(reads) == {candidate.resolve()}


async def test_wait_for_finalized_atif_accepts_modified_existing_path(tmp_path):
atif_dir = tmp_path / "atif"
atif_dir.mkdir()
existing = atif_dir / "trajectory-existing.atif.json"
existing.write_text('{"schema_version":"ATIF-v1.7"}', encoding="utf-8")
plugin_config = atif_plugin_config(atif_dir)
before = relay_artifacts.snapshot_atif_files(plugin_config)
existing.write_text(
json.dumps({"schema_version": "ATIF-v1.7", "steps": [{"step_id": 1}]}),
encoding="utf-8",
)

finalized = await relay_artifacts.wait_for_finalized_atif(
plugin_config,
before,
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)

assert finalized == existing.resolve()


async def test_wait_for_finalized_atif_has_a_hard_deadline(tmp_path):
atif_dir = tmp_path / "atif"
atif_dir.mkdir()
plugin_config = atif_plugin_config(atif_dir)

finalized = await relay_artifacts.wait_for_finalized_atif(
plugin_config,
relay_artifacts.snapshot_atif_files(plugin_config),
timeout_seconds=0.01,
poll_interval_seconds=0.001,
)

assert finalized is None


async def test_wait_for_finalized_atif_isolated_by_runtime_directory(tmp_path):
first_dir = tmp_path / "runtime-1"
second_dir = tmp_path / "runtime-2"
first_dir.mkdir()
second_dir.mkdir()
first_config = atif_plugin_config(first_dir)
second_config = atif_plugin_config(second_dir)

async def write_atif(directory: Path, session_id: str, delay: float):
await asyncio.sleep(delay)
path = directory / f"trajectory-{session_id}.atif.json"
path.write_text(
json.dumps({"schema_version": "ATIF-v1.7", "steps": []}),
encoding="utf-8",
)
return path.resolve()

first_writer = asyncio.create_task(write_atif(first_dir, "first", 0.02))
second_writer = asyncio.create_task(write_atif(second_dir, "second", 0.01))
first_wait = relay_artifacts.wait_for_finalized_atif(
first_config,
relay_artifacts.snapshot_atif_files(first_config),
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)
second_wait = relay_artifacts.wait_for_finalized_atif(
second_config,
relay_artifacts.snapshot_atif_files(second_config),
timeout_seconds=0.5,
poll_interval_seconds=0.001,
)

first_finalized, second_finalized, first_path, second_path = await asyncio.gather(
first_wait, second_wait, first_writer, second_writer
)

assert first_finalized == first_path
assert second_finalized == second_path
Loading
Loading