(
const Icon = action.Icon;
const key = `${action.toolCallId}-${index}`;
- if (action.details?.length) {
+ const invocations = action.invocations ?? [action.invocation];
+
+ if (action.details?.length || invocations.length) {
return (
(
data-testid="claude-code-tool-call-subtle-details"
>
(
/>
{action.message}
-
- {action.details.map((detail, detailIndex) => (
- -
- {detail}
-
- ))}
-
+ {action.details?.length ? (
+
+ {action.details.map((detail, detailIndex) => {
+ const invocation = invocations[detailIndex];
+ if (!invocation) return null;
+
+ return (
+
+
+ {detail}
+
+
+
+
+
+
+ );
+ })}
+
+ ) : (
+
+
+
+ )}
);
}
diff --git a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/helpers.ts b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/helpers.ts
index 286c56b32f..ff8e2cf607 100644
--- a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/helpers.ts
+++ b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/helpers.ts
@@ -355,6 +355,15 @@ export const getSubtleToolDetail = (
return compactSubtleDetail(getToolSummary(toolName, args)) ?? message;
};
+export const getToolInvocation = (toolName: string, args: ClaudeCodeToolArgs): string => {
+ if (toolName === 'Bash') {
+ const command = args.command;
+ if (typeof command === 'string' && command) return command;
+ }
+
+ return JSON.stringify(args, null, 2);
+};
+
export const getSubtleToolGroupActions = (
args: ClaudeCodeToolArgs
): readonly SubtleToolAction[] => {
@@ -374,6 +383,7 @@ export const getSubtleToolGroupActions = (
return {
detail: getSubtleToolDetail(toolName, actionArgs, message),
Icon: getSubtleToolIcon(toolName),
+ invocation: getToolInvocation(toolName, actionArgs),
message,
toolCallId: getRawStringArg(action, ['toolCallId'])?.trim() ?? `${toolName}-${index}`,
toolName,
@@ -403,6 +413,7 @@ export const summarizeRepeatedSubtleToolActions = (
return {
...firstAction,
details: group.map((action) => action.detail),
+ invocations: group.map((action) => action.invocation),
message: getRepeatedSubtleToolMessage(firstAction.toolName, group.length),
title: group.map((action) => action.message).join(' | '),
toolCallId: `${firstAction.toolCallId}-${group.length}`,
diff --git a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/types.ts b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/types.ts
index dc61fabedd..cce3a28d7f 100644
--- a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/types.ts
+++ b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/types.ts
@@ -7,6 +7,8 @@ export interface SubtleToolAction {
readonly detail: string;
readonly details?: readonly string[];
readonly Icon: LucideIcon;
+ readonly invocation: string;
+ readonly invocations?: readonly string[];
readonly message: string;
readonly title?: string;
readonly toolCallId: string;
From ad23c759dd42f79651fb3a55bea74d7a8bd42fbd Mon Sep 17 00:00:00 2001
From: Danielle Ali <44468613+dmariali@users.noreply.github.com>
Date: Mon, 13 Jul 2026 13:17:44 -0400
Subject: [PATCH 2/4] cleanup
Signed-off-by: Danielle Ali <44468613+dmariali@users.noreply.github.com>
---
.../ClaudeCodeToolCallPart.test.tsx | 41 +++++--
.../toolCall/SubtleToolCallRow.tsx | 111 ++++++++++--------
2 files changed, 94 insertions(+), 58 deletions(-)
diff --git a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/ClaudeCodeToolCallPart.test.tsx b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/ClaudeCodeToolCallPart.test.tsx
index bdf3d3530a..55d5d9d74e 100644
--- a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/ClaudeCodeToolCallPart.test.tsx
+++ b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/ClaudeCodeToolCallPart.test.tsx
@@ -68,7 +68,9 @@ const expectSubtleToolBlock = (subtleBlock: HTMLElement) => {
expect(subtleBlock).toHaveClass(
'my-density-xs',
'flex',
+ 'w-full',
'max-w-full',
+ 'overflow-hidden',
'flex-wrap',
'items-center',
'gap-x-density-sm',
@@ -313,6 +315,13 @@ describe('ClaudeCodeToolCallPart', () => {
expect(screen.getByTestId('claude-code-tool-call-invocation')).toHaveTextContent(
'pnpm --filter nemo-studio-ui test'
);
+ expect(screen.getByTestId('claude-code-tool-call-invocation-surface')).toHaveClass(
+ 'w-full',
+ 'min-w-0',
+ 'max-w-full',
+ 'overflow-auto',
+ 'break-words'
+ );
await user.click(screen.getByRole('button', { name: 'Copy invocation' }));
@@ -429,12 +438,27 @@ describe('ClaudeCodeToolCallPart', () => {
expect(
within(nestedInvocations[0]!).getByTestId('claude-code-tool-call-invocation')
).not.toBeVisible();
+ const readDetails = screen.getAllByTestId('claude-code-tool-call-subtle-details')[1]!;
+ expect(readDetails).not.toHaveAttribute('open');
+
+ await user.click(screen.getByText('Read 2 files'));
+
+ expect(readDetails).toHaveAttribute('open');
+ expect(
+ within(readDetails)
+ .getAllByTestId('claude-code-tool-call-subtle-detail-item')
+ .map((item) => item.textContent)
+ ).toEqual(['README.md', 'package.json']);
+ expect(
+ within(readDetails).queryByTestId('claude-code-tool-call-nested-invocation')
+ ).not.toBeInTheDocument();
+ expect(
+ within(readDetails).queryByTestId('claude-code-tool-call-invocation')
+ ).not.toBeInTheDocument();
expect(screen.getAllByTestId('claude-code-tool-call-subtle-detail-item')).toHaveLength(7);
});
- it('renders Read as subtle text with the full invocation behind a disclosure', async () => {
- const user = userEvent.setup();
-
+ it('renders Read as summary-only subtle text', () => {
render(
{
expect(readBlock).toHaveTextContent('Read App.tsx');
expect(readBlock.tagName).toBe('DIV');
expectSubtleToolBlock(readBlock);
- const disclosure = screen.getByTestId('claude-code-tool-call-subtle-details');
- expect(disclosure).not.toHaveAttribute('open');
-
- await user.click(screen.getByText('Read App.tsx'));
-
- expect(disclosure).toHaveAttribute('open');
- expect(screen.getByTestId('claude-code-tool-call-invocation')).toHaveTextContent(
- '"file_path": "web/packages/studio/src/App.tsx"'
- );
+ expect(screen.queryByTestId('claude-code-tool-call-subtle-details')).not.toBeInTheDocument();
+ expect(screen.queryByTestId('claude-code-tool-call-invocation')).not.toBeInTheDocument();
});
it('renders running subtle tool text without special color or animation', () => {
diff --git a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx
index b24030c535..19cf7b6d3e 100644
--- a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx
+++ b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-import { Text } from '@nvidia/foundations-react-core';
+import { Flex, Text } from '@nvidia/foundations-react-core';
import { CODE_BLOCK_SURFACE_CLASS } from '@studio/routes/agents/ClaudeCodeChatRoute/toolCall/constants';
import { summarizeRepeatedSubtleToolActions } from '@studio/routes/agents/ClaudeCodeChatRoute/toolCall/helpers';
import type { SubtleToolAction } from '@studio/routes/agents/ClaudeCodeChatRoute/toolCall/types';
@@ -29,12 +29,13 @@ const InvocationPanel = ({
invocationIndex,
toolCallId,
}: InvocationPanelProps) => (
-
+
+
);
export const SubtleToolCallRow = ({ actions }: SubtleToolCallRowProps) => (
action.title ?? action.message).join(' | ')}
>
@@ -65,16 +67,18 @@ export const SubtleToolCallRow = ({ actions }: SubtleToolCallRowProps) => (
const key = `${action.toolCallId}-${index}`;
const invocations = action.invocations ?? [action.invocation];
+ const isExpandable =
+ Boolean(action.details?.length) || (action.toolName !== 'Read' && invocations.length > 0);
- if (action.details?.length || invocations.length) {
+ if (isExpandable) {
return (
(
{action.message}
{action.details?.length ? (
-
- {action.details.map((detail, detailIndex) => {
- const invocation = invocations[detailIndex];
- if (!invocation) return null;
-
- return (
-
- (
+
- {detail}
-
-
-
-
-
-
- );
- })}
-
+ {detail}
+
+ ))
+ : action.details.map((detail, detailIndex) => {
+ const invocation = invocations[detailIndex];
+ if (!invocation) return null;
+
+ return (
+
+
+ {detail}
+
+
+
+
+
+
+ );
+ })}
+
) : (
-
+
-
+
)}
);
@@ -143,7 +162,7 @@ export const SubtleToolCallRow = ({ actions }: SubtleToolCallRowProps) => (
return (
Date: Tue, 14 Jul 2026 09:09:34 -0400
Subject: [PATCH 3/4] change flex dir='row' to stack
Signed-off-by: Danielle Ali <44468613+dmariali@users.noreply.github.com>
---
.../toolCall/SubtleToolCallRow.tsx | 24 +++++++------------
1 file changed, 8 insertions(+), 16 deletions(-)
diff --git a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx
index 19cf7b6d3e..7ebf2f3f9e 100644
--- a/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx
+++ b/web/packages/studio/src/routes/agents/ClaudeCodeChatRoute/toolCall/SubtleToolCallRow.tsx
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-import { Flex, Text } from '@nvidia/foundations-react-core';
+import { Stack, Text } from '@nvidia/foundations-react-core';
import { CODE_BLOCK_SURFACE_CLASS } from '@studio/routes/agents/ClaudeCodeChatRoute/toolCall/constants';
import { summarizeRepeatedSubtleToolActions } from '@studio/routes/agents/ClaudeCodeChatRoute/toolCall/helpers';
import type { SubtleToolAction } from '@studio/routes/agents/ClaudeCodeChatRoute/toolCall/types';
@@ -29,13 +29,12 @@ const InvocationPanel = ({
invocationIndex,
toolCallId,
}: InvocationPanelProps) => (
-
-
+
-
+
+
);
export const SubtleToolCallRow = ({ actions }: SubtleToolCallRowProps) => (
@@ -93,8 +92,7 @@ export const SubtleToolCallRow = ({ actions }: SubtleToolCallRowProps) => (
{action.message}
{action.details?.length ? (
-
@@ -130,30 +128,24 @@ export const SubtleToolCallRow = ({ actions }: SubtleToolCallRowProps) => (
className="size-3 shrink-0 transition-transform group-open/invocation:rotate-90"
/>
-
+
-
);
})}
-
+
) : (
-
-
)}
);
From c3b7d16daa998a1f3a42196c5e3c21aa861606df Mon Sep 17 00:00:00 2001
From: Danielle Ali <44468613+dmariali@users.noreply.github.com>
Date: Tue, 14 Jul 2026 09:44:11 -0400
Subject: [PATCH 4/4] Revert "Merge branch 'main' into
coding_agent_show_tool_details"
This reverts commit 31dc4ee0b42716d883321139a1296230251983f1, reversing
changes made to 8c88e071a6837440413cc6f3658394d9a17bff08.
Signed-off-by: Danielle Ali <44468613+dmariali@users.noreply.github.com>
---
.github/actions/build-policy-wasm/action.yaml | 4 +-
.../actions/setup-kind-cluster/action.yaml | 1 +
.github/assets/ngc/containers/nmp-core.md | 11 +
.../scripts/write_release_bundle_metadata.py | 259 ++++
.../nemo-platform-services.txt | 2 +-
.github/workflows/ngc-metadata.yaml | 3 -
.github/workflows/release-bundle.yaml | 674 ++++++++++
.github/workflows/release-nightly.yaml | 59 +
.github/workflows/release-rc.yaml | 59 +
.github/workflows/release-stable.yaml | 116 ++
.github/workflows/release.yaml | 1166 -----------------
.gitignore | 5 -
RELEASING.md | 241 ++--
conftest.py | 17 -
docker-bake.hcl | 4 +
docker/Dockerfile.auditor-tasks | 38 +-
docker/Dockerfile.nmp-unsloth-training | 9 +-
docker/Dockerfile.safe-synthesizer-tasks | 27 +-
docker/base/Dockerfile.nmp-workspace | 1 +
docs/set-up/config-reference.mdx | 4 +-
e2e/k8s/scripts/local_build_and_upgrade.sh | 34 +-
e2e/test_safe_synthesizer.py | 143 --
.../examples/profbench/profbench.py | 4 +-
.../agent_eval/evaluator.py | 4 +-
.../agent_eval/runtimes/codex/runtime.py | 38 +-
.../agent_eval/runtimes/docker_sandbox.py | 14 +-
.../agent_eval/runtimes/fabric/runtime.py | 254 ++--
.../nemo_evaluator_sdk/agent_eval/tasks.py | 17 -
.../tests/agent_eval/test_codex_runtime.py | 64 +-
.../agent_eval/test_docker_sandbox_runtime.py | 36 +-
.../agent_eval/test_fabric_integration.py | 50 +-
.../tests/agent_eval/test_fabric_runtime.py | 211 +--
packages/nemo_platform/pyproject.toml | 2 +-
plugins/nemo-safe-synthesizer/constraints.txt | 19 +-
.../openapi/openapi.yaml | 99 +-
plugins/nemo-safe-synthesizer/pyproject.toml | 2 +-
.../nemo_safe_synthesizer_plugin/config.py | 2 +-
.../nemo_safe_synthesizer_plugin/runtime.py | 23 +-
.../tests/unit/test_runtime.py | 6 +-
pytest.ini | 1 -
release/assets.yaml | 15 +
script/dev-install-fabric.sh | 2 +-
.../beta/evaluator/agent_eval/evaluator.py | 4 +-
.../agent_eval/runtimes/codex/runtime.py | 38 +-
.../agent_eval/runtimes/docker_sandbox.py | 14 +-
.../agent_eval/runtimes/fabric/runtime.py | 254 ++--
.../beta/evaluator/agent_eval/tasks.py | 17 -
.../studio/src/nmp/studio/env_mappings.py | 2 +-
.../test_write_release_bundle_metadata.py | 441 +++++++
uv.lock | 15 +-
.../schema/AgentsGetDeploymentLogsParams.ts | 16 +
.../agents/schema/DeploymentLogsResponse.ts | 20 +
.../sdk/generated/agents/schema/LogLine.ts | 24 +
web/packages/studio/env/.env.dev.local.sample | 2 +-
.../constants/featureFlags/featureFlags.ts | 2 +-
.../routes/DashboardLandingRoute/index.tsx | 1 -
56 files changed, 2282 insertions(+), 2308 deletions(-)
create mode 100644 .github/assets/ngc/containers/nmp-core.md
create mode 100644 .github/scripts/write_release_bundle_metadata.py
create mode 100644 .github/workflows/release-bundle.yaml
create mode 100644 .github/workflows/release-nightly.yaml
create mode 100644 .github/workflows/release-rc.yaml
create mode 100644 .github/workflows/release-stable.yaml
delete mode 100644 .github/workflows/release.yaml
delete mode 100644 e2e/test_safe_synthesizer.py
create mode 100644 release/assets.yaml
create mode 100644 tests/unit/release/test_write_release_bundle_metadata.py
create mode 100644 web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts
create mode 100644 web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts
create mode 100644 web/packages/sdk/generated/agents/schema/LogLine.ts
diff --git a/.github/actions/build-policy-wasm/action.yaml b/.github/actions/build-policy-wasm/action.yaml
index 863406d262..98d3219447 100644
--- a/.github/actions/build-policy-wasm/action.yaml
+++ b/.github/actions/build-policy-wasm/action.yaml
@@ -10,6 +10,4 @@ runs:
steps:
- name: Build policy WASM
shell: bash
- env:
- REPO_ROOT: ${{ github.action_path }}/../../..
- run: "${REPO_ROOT}/script/build_policy_wasm.sh"
+ run: ./script/build_policy_wasm.sh
diff --git a/.github/actions/setup-kind-cluster/action.yaml b/.github/actions/setup-kind-cluster/action.yaml
index 57d591c75f..961d6a8b09 100644
--- a/.github/actions/setup-kind-cluster/action.yaml
+++ b/.github/actions/setup-kind-cluster/action.yaml
@@ -141,6 +141,7 @@ runs:
run: |
e2e/k8s/scripts/prepull_kind_images.sh \
"${NMP_E2E_REGISTRY}/nmp-api:${NMP_E2E_TAG}" \
+ "${NMP_E2E_REGISTRY}/nmp-core:${NMP_E2E_TAG}" \
"${NMP_E2E_REGISTRY}/nmp-cpu-tasks:${NMP_E2E_TAG}"
- name: Install NeMo Platform
diff --git a/.github/assets/ngc/containers/nmp-core.md b/.github/assets/ngc/containers/nmp-core.md
new file mode 100644
index 0000000000..04b7a72176
--- /dev/null
+++ b/.github/assets/ngc/containers/nmp-core.md
@@ -0,0 +1,11 @@
+## NeMo Platform Core Container
+
+This container image provides required services for a deployed version of the NeMo Platform.
+
+### Resources
+
+[Documentation](https://docs.nvidia.com/nemo-platform)
+
+### License
+
+This container is licensed under the [Apache License 2.0](https://github.com/NVIDIA-NeMo/nemo-platform/blob/main/LICENSE).
diff --git a/.github/scripts/write_release_bundle_metadata.py b/.github/scripts/write_release_bundle_metadata.py
new file mode 100644
index 0000000000..4eff45ad1d
--- /dev/null
+++ b/.github/scripts/write_release_bundle_metadata.py
@@ -0,0 +1,259 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Write release bundle metadata for downloaded SDK wheel artifacts.
+
+Container artifacts are metadata-only manifest entries: the image bits are
+built and staged by the release consumer from its dev registry at the bundle's
+source SHA, so container entries carry no path or checksum.
+"""
+
+import argparse
+import hashlib
+import json
+import re
+import shutil
+import sys
+import zipfile
+from email.parser import BytesParser
+from email.policy import default
+from pathlib import Path
+from typing import Literal
+
+Cadence = Literal["nightly", "rc", "release"]
+
+
+class BundleMetadataError(Exception):
+ """Raised when the release bundle metadata cannot be written safely."""
+
+
+def safe_artifact_id(artifact_type: str, artifact_id: str) -> str:
+ if not re.fullmatch(r"[A-Za-z0-9._-]+", artifact_id) or artifact_id in {".", ".."}:
+ raise BundleMetadataError(f"selected {artifact_type} id must be a safe single path segment: {artifact_id}")
+ return artifact_id
+
+
+def parse_release_date_json(value: str) -> str | None:
+ try:
+ parsed = json.loads(value)
+ except json.JSONDecodeError as error:
+ raise BundleMetadataError(f"release_date_json must be valid JSON: {error.msg}") from error
+
+ if parsed is not None and not isinstance(parsed, str):
+ raise BundleMetadataError("release_date_json must be a JSON string or null")
+ return parsed
+
+
+def artifact_ref(artifact_type: object, artifact_id: object) -> str:
+ return f"{artifact_type}:{artifact_id}"
+
+
+def parse_selected_artifact_ids(value: str) -> dict[str, list[str]]:
+ """Parse selected_artifacts_json into ids grouped by artifact type."""
+ try:
+ parsed = json.loads(value)
+ except json.JSONDecodeError as error:
+ raise BundleMetadataError(f"selected_artifacts_json must be valid JSON: {error.msg}") from error
+
+ if not isinstance(parsed, list) or not parsed:
+ raise BundleMetadataError("selected_artifacts_json must be a non-empty list")
+
+ ids_by_type: dict[str, list[str]] = {"sdk": [], "container": []}
+ seen: dict[str, set[str]] = {artifact_type: set() for artifact_type in ids_by_type}
+ for artifact in parsed:
+ if not isinstance(artifact, dict):
+ raise BundleMetadataError("selected_artifacts_json entries must be objects")
+
+ artifact_type = artifact.get("type")
+ artifact_id = artifact.get("id")
+ if artifact_type not in ids_by_type:
+ raise BundleMetadataError(
+ f"unsupported artifact type in bundle selection: {artifact_ref(artifact_type, artifact_id)}"
+ )
+ if not isinstance(artifact_id, str) or not artifact_id:
+ raise BundleMetadataError(f"selected {artifact_type} artifact id must be a non-empty string")
+
+ checked_id = safe_artifact_id(artifact_type, artifact_id)
+ if checked_id in seen[artifact_type]:
+ raise BundleMetadataError(f"selected_artifacts_json contains duplicate {artifact_type} id: {checked_id}")
+
+ seen[artifact_type].add(checked_id)
+ ids_by_type[artifact_type].append(checked_id)
+
+ # A bundle may be SDK-only, container-only, or mixed; the non-empty-list
+ # check above already guarantees at least one artifact of some type.
+ return ids_by_type
+
+
+def find_sdk_wheel(sdk_artifacts_dir: Path, sdk_id: str, *, single_sdk_artifact: bool) -> Path:
+ artifact_dir = sdk_artifacts_dir / f"release-sdk-{sdk_id}"
+ if not artifact_dir.is_dir():
+ if single_sdk_artifact:
+ # download-artifact extracts one pattern match directly into the target path.
+ wheels = sorted(sdk_artifacts_dir.glob("*.whl"))
+ if wheels:
+ if len(wheels) != 1:
+ raise BundleMetadataError(f"expected exactly one wheel in {sdk_artifacts_dir}, found {len(wheels)}")
+ return wheels[0]
+ raise BundleMetadataError(f"missing downloaded SDK artifact directory: {artifact_dir}")
+
+ wheels = sorted(artifact_dir.glob("*.whl"))
+ if len(wheels) != 1:
+ raise BundleMetadataError(f"expected exactly one wheel in {artifact_dir}, found {len(wheels)}")
+ return wheels[0]
+
+
+def read_wheel_version(path: Path) -> str:
+ with zipfile.ZipFile(path) as wheel:
+ metadata_files = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")]
+ if len(metadata_files) != 1:
+ raise BundleMetadataError(
+ f"expected exactly one METADATA file in wheel {path.name}, found {len(metadata_files)}"
+ )
+
+ metadata = BytesParser(policy=default).parsebytes(wheel.read(metadata_files[0]))
+
+ version = metadata["Version"]
+ if not version:
+ raise BundleMetadataError(f"wheel metadata must include Version: {path.name}")
+ return str(version)
+
+
+def file_sha256(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as file:
+ for chunk in iter(lambda: file.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def bundle_relative_path(bundle_dir: Path, path: Path) -> str:
+ return path.relative_to(bundle_dir).as_posix()
+
+
+def write_checksums(bundle_dir: Path) -> Path:
+ checksums_path = bundle_dir / "checksums.txt"
+ files = [
+ bundle_dir / "release-manifest.json",
+ *sorted(path for path in (bundle_dir / "wheels").rglob("*") if path.is_file()),
+ ]
+
+ with checksums_path.open("w", encoding="utf-8") as checksums:
+ for path in files:
+ checksums.write(f"{file_sha256(path)} {bundle_relative_path(bundle_dir, path)}\n")
+
+ return checksums_path
+
+
+def prepare_bundle_dir(bundle_dir: Path) -> Path:
+ wheels_dir = bundle_dir / "wheels"
+ if wheels_dir.exists():
+ shutil.rmtree(wheels_dir)
+
+ bundle_dir.mkdir(parents=True, exist_ok=True)
+ for filename in ("release-manifest.json", "checksums.txt"):
+ path = bundle_dir / filename
+ if path.exists():
+ path.unlink()
+
+ wheels_dir.mkdir()
+ return wheels_dir
+
+
+def write_release_bundle_metadata(
+ *,
+ sdk_artifacts_dir: Path,
+ bundle_dir: Path,
+ selected_artifacts_json: str,
+ cadence: Cadence,
+ release_label: str,
+ release_date_json: str,
+ source_sha: str,
+) -> dict[str, object]:
+ if not release_label:
+ raise BundleMetadataError("release_label is required")
+ if not source_sha:
+ raise BundleMetadataError("source_sha is required")
+
+ ids_by_type = parse_selected_artifact_ids(selected_artifacts_json)
+ sdk_ids = ids_by_type["sdk"]
+ release_date = parse_release_date_json(release_date_json)
+ wheels_dir = prepare_bundle_dir(bundle_dir)
+
+ artifacts: list[dict[str, str]] = []
+ for sdk_id in sdk_ids:
+ source_wheel = find_sdk_wheel(sdk_artifacts_dir, sdk_id, single_sdk_artifact=len(sdk_ids) == 1)
+ wheel_version = read_wheel_version(source_wheel)
+ wheel_path = wheels_dir / source_wheel.name
+ if wheel_path.exists():
+ raise BundleMetadataError(f"duplicate wheel filename in bundle: {source_wheel.name}")
+
+ shutil.copy2(source_wheel, wheel_path)
+ artifacts.append(
+ {
+ "type": "sdk",
+ "id": sdk_id,
+ "version": wheel_version,
+ "path": bundle_relative_path(bundle_dir, wheel_path),
+ }
+ )
+
+ # Container artifacts are metadata-only: the consumer stages the images
+ # from its dev registry by source_sha and tags them with release_label.
+ for container_id in ids_by_type["container"]:
+ artifacts.append(
+ {
+ "type": "container",
+ "id": container_id,
+ "version": release_label,
+ }
+ )
+
+ manifest: dict[str, object] = {
+ "cadence": cadence,
+ "release_label": release_label,
+ "release_date": release_date,
+ "source_sha": source_sha,
+ "artifacts": artifacts,
+ }
+
+ manifest_path = bundle_dir / "release-manifest.json"
+ manifest_path.write_text(f"{json.dumps(manifest, indent=2)}\n", encoding="utf-8")
+ write_checksums(bundle_dir)
+ return manifest
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--sdk-artifacts-dir", required=True, type=Path)
+ parser.add_argument("--bundle-dir", required=True, type=Path)
+ parser.add_argument("--selected-artifacts-json", required=True)
+ parser.add_argument("--cadence", required=True, choices=["nightly", "rc", "release"])
+ parser.add_argument("--release-label", required=True)
+ parser.add_argument("--release-date-json", required=True)
+ parser.add_argument("--source-sha", required=True)
+ return parser.parse_args(argv)
+
+
+def main(argv: list[str]) -> int:
+ args = parse_args(argv)
+ try:
+ manifest = write_release_bundle_metadata(
+ sdk_artifacts_dir=args.sdk_artifacts_dir,
+ bundle_dir=args.bundle_dir,
+ selected_artifacts_json=args.selected_artifacts_json,
+ cadence=args.cadence,
+ release_label=args.release_label,
+ release_date_json=args.release_date_json,
+ source_sha=args.source_sha,
+ )
+ except (BundleMetadataError, OSError, zipfile.BadZipFile) as error:
+ print(f"::error::{error}", file=sys.stderr)
+ return 1
+
+ print(f"Wrote release bundle metadata for {len(manifest['artifacts'])} artifact(s).")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main(sys.argv[1:]))
diff --git a/.github/wheel-constraints/nemo-platform-services.txt b/.github/wheel-constraints/nemo-platform-services.txt
index a779d6db9d..888cbc4b88 100644
--- a/.github/wheel-constraints/nemo-platform-services.txt
+++ b/.github/wheel-constraints/nemo-platform-services.txt
@@ -43,7 +43,7 @@ langchain==1.3.13
lark==1.3.1
litellm<1.92 # 1.92.0 native build has no py3.14 wheel
nemo-anonymizer==0.2.1
-nemo-safe-synthesizer==0.1.7
+nemo-safe-synthesizer==0.1.2
nemoguardrails==0.23.0
ngcsdk==4.20.1
nvidia-ml-py==13.610.43
diff --git a/.github/workflows/ngc-metadata.yaml b/.github/workflows/ngc-metadata.yaml
index 5da4fc41fe..c25fbfb970 100644
--- a/.github/workflows/ngc-metadata.yaml
+++ b/.github/workflows/ngc-metadata.yaml
@@ -2,9 +2,6 @@ name: NGC Metadata Sync
on:
workflow_call:
- secrets:
- AIRE_NGC_GITHUB_PLATFORM_RW:
- required: true
workflow_dispatch:
permissions:
diff --git a/.github/workflows/release-bundle.yaml b/.github/workflows/release-bundle.yaml
new file mode 100644
index 0000000000..a91f2e4a77
--- /dev/null
+++ b/.github/workflows/release-bundle.yaml
@@ -0,0 +1,674 @@
+name: Release Bundle
+
+on:
+ workflow_call:
+ inputs:
+ cadence:
+ description: "Release cadence: nightly, rc, or release."
+ required: true
+ type: string
+ source_sha:
+ description: "Expected source commit SHA for RC/stable releases."
+ required: false
+ type: string
+ default: ""
+ version:
+ description: "SemVer core version for RC or stable releases, for example 1.0.0."
+ required: false
+ type: string
+ default: ""
+ release_date:
+ description: "Optional. Provide only for first-time registration of this release version. Leave blank for reruns or when this release version is already registered. Format: YYYY-MM-DD."
+ required: false
+ type: string
+ default: ""
+ release_scope:
+ description: "Release artifact scope: all, sdks, containers, or custom."
+ required: false
+ type: string
+ default: "all"
+ sdk_ids:
+ description: "Comma-separated SDK IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+ container_ids:
+ description: "Comma-separated container IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+ send_notifications:
+ description: "Whether downstream release jobs should send notifications."
+ required: false
+ type: boolean
+ default: true
+ secrets:
+ CI_DISPATCH_TOKEN:
+ description: "Token with access to send repository dispatch events to the dispatch repo."
+ required: true
+ CI_DISPATCH_REPO:
+ description: "Repository that receives release bundle handoff dispatch events."
+ required: true
+permissions:
+ contents: read
+
+jobs:
+ plan-release:
+ name: Plan release
+ runs-on: ubuntu-latest
+ # Normalize nightly and RC/stable planner results into one contract for downstream jobs.
+ # Later jobs should read needs.plan-release.outputs.* without caring which cadence produced them.
+ outputs:
+ source_repo: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_repo || github.repository }}
+ source_ref: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_ref || steps.resolve-release-metadata.outputs.source_ref }}
+ release_label: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.release_label || steps.resolve-release-metadata.outputs.release_label }}
+ nightly_timestamp: ${{ steps.resolve-nightly-source.outputs.nightly_timestamp }}
+ selected_artifacts_json: ${{ steps.plan-assets.outputs.selected_artifacts_json }}
+ sdk_count: ${{ steps.plan-assets.outputs.sdk_count }}
+ container_count: ${{ steps.plan-assets.outputs.container_count }}
+ sdk_matrix: ${{ steps.plan-assets.outputs.sdk_matrix }}
+ release_tag: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.release_tag || steps.resolve-release-metadata.outputs.release_tag }}
+ release_date_json: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.release_date_json || steps.resolve-release-metadata.outputs.release_date_json }}
+ source_sha: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_sha || inputs.source_sha }}
+ # Previous stable tag that bounds the generated release notes range (RC/stable only; empty for nightly).
+ notes_start_tag: ${{ steps.resolve-release-metadata.outputs.notes_start_tag }}
+ steps:
+ # Fetch tags so RC auto-increment can use plain local Git.
+ - name: Checkout workflow code
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ fetch-depth: 0
+ fetch-tags: true
+
+ # Fail fast on cadence-specific inputs before source lookup or asset planning:
+ # nightly may pin an exact source SHA, while RC/stable require source SHA and version.
+ - name: Validate cadence contract
+ shell: bash
+ env:
+ CADENCE: ${{ inputs.cadence }}
+ SOURCE_SHA: ${{ inputs.source_sha }}
+ VERSION: ${{ inputs.version }}
+ RELEASE_DATE: ${{ inputs.release_date }}
+ run: |
+ set -euo pipefail
+
+ case "${CADENCE}" in
+ nightly)
+ if [[ -n "${VERSION}" || -n "${RELEASE_DATE}" ]]; then
+ echo "::error::nightly releases must not provide version or release_date inputs"
+ exit 1
+ fi
+ ;;
+ rc | release)
+ if [[ -z "${SOURCE_SHA}" || -z "${VERSION}" ]]; then
+ echo "::error::rc and stable releases require source_sha and version inputs"
+ exit 1
+ fi
+ ;;
+ *)
+ echo "::error::Unknown release cadence: ${CADENCE}"
+ exit 1
+ ;;
+ esac
+
+ if [[ -n "${SOURCE_SHA}" && ! "${SOURCE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then
+ echo "::error::source_sha must be a full 40-character commit SHA"
+ exit 1
+ fi
+
+ # Nightly defaults to the repository default branch, but may pin an exact source SHA.
+ - name: Resolve nightly source
+ id: resolve-nightly-source
+ if: inputs.cadence == 'nightly'
+ shell: bash
+ env:
+ DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ SOURCE_SHA: ${{ inputs.source_sha }}
+ run: |
+ set -euo pipefail
+
+ timestamp="$(date -u +%Y%m%d%H%M%S)"
+
+ if [[ -n "${SOURCE_SHA}" ]]; then
+ source_ref="${SOURCE_SHA}"
+ source_sha="${SOURCE_SHA}"
+ else
+ [[ -n "${DEFAULT_BRANCH}" ]] || { echo "::error::repository default branch is empty"; exit 1; }
+ source_ref="refs/heads/${DEFAULT_BRANCH}"
+ git fetch --no-tags --depth=1 origin "${source_ref}"
+ source_sha="$(git rev-parse --verify "FETCH_HEAD^{commit}")"
+ fi
+
+ {
+ printf 'source_repo=%s\n' "${GITHUB_REPOSITORY}"
+ printf 'source_ref=%s\n' "${source_ref}"
+ printf 'source_sha=%s\n' "${source_sha}"
+ printf 'nightly_timestamp=%s\n' "${timestamp}"
+ printf 'release_label=nightly-%s\n' "${timestamp}"
+ printf 'release_tag=\n'
+ printf 'release_date_json=null\n'
+ } >>"${GITHUB_OUTPUT}"
+
+ # RC/stable use the selected workflow branch plus manual version inputs:
+ # compute the release label/tag, auto-increment RC tags, and format release_date.
+ - name: Resolve RC/stable metadata
+ id: resolve-release-metadata
+ if: inputs.cadence != 'nightly'
+ shell: bash
+ env:
+ CADENCE: ${{ inputs.cadence }}
+ SOURCE_SHA: ${{ inputs.source_sha }}
+ VERSION: ${{ inputs.version }}
+ RELEASE_DATE: ${{ inputs.release_date }}
+ run: |
+ set -euo pipefail
+
+ if [[ "${GITHUB_REF}" != refs/heads/* ]]; then
+ echo "::error::rc and stable releases must run from a branch ref"
+ exit 1
+ fi
+
+ if ! git cat-file -e "${SOURCE_SHA}^{commit}" 2>/dev/null; then
+ echo "::error::source_sha is not a commit in this repository: ${SOURCE_SHA}"
+ exit 1
+ fi
+
+ if ! git merge-base --is-ancestor "${SOURCE_SHA}" "${GITHUB_SHA}"; then
+ echo "::error::source_sha must be reachable from the selected workflow branch"
+ exit 1
+ fi
+
+ if [[ ! "${VERSION}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
+ echo "::error::version must be SemVer core MAJOR.MINOR.PATCH, for example 1.0.0"
+ exit 1
+ fi
+
+ if [[ "${CADENCE}" == "rc" ]]; then
+ rc_prefix="${VERSION}-rc"
+ existing_rcs="$(git tag -l "${rc_prefix}*" | grep -E "^${rc_prefix}[0-9]+$" | sort -V || true)"
+
+ if [[ -z "${existing_rcs}" ]]; then
+ next_rc=0
+ else
+ last_rc="$(echo "${existing_rcs}" | tail -n 1)"
+ last_rc_number="${last_rc#"${rc_prefix}"}"
+ next_rc=$((10#${last_rc_number} + 1))
+ fi
+
+ release_label="${rc_prefix}${next_rc}"
+ else
+ release_label="${VERSION}"
+ fi
+
+ release_date_json="null"
+ if [[ -n "${RELEASE_DATE}" ]]; then
+ if [[ ! "${RELEASE_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then
+ echo "::error::release_date must use YYYY-MM-DD format"
+ exit 1
+ fi
+ if [[ "$(date -u -d "${RELEASE_DATE}" +%Y-%m-%d 2>/dev/null)" != "${RELEASE_DATE}" ]]; then
+ echo "::error::release_date must be a valid calendar date"
+ exit 1
+ fi
+ release_date_json="\"${RELEASE_DATE}\""
+ fi
+
+ # Generated release notes span from the previous stable release so RC and stable both
+ # show the full delta since the last stable. Empty when no prior stable tag exists.
+ # Hardened for `set -euo pipefail`: `|| true` covers the no-stable-tags case, and awk
+ # reads to EOF (no early `exit`/`head`) so `sort` is never killed by SIGPIPE.
+ notes_start_tag=""
+ prev_stable="$(
+ { git tag -l | grep -E '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' || true; printf '%s\n' "${VERSION}"; } \
+ | sort -V -u \
+ | awk -v v="${VERSION}" '$0==v{print prev} {prev=$0}'
+ )"
+ if [[ -n "${prev_stable}" && "${prev_stable}" != "${VERSION}" ]]; then
+ notes_start_tag="${prev_stable}"
+ fi
+
+ {
+ printf 'source_ref=%s\n' "${GITHUB_REF}"
+ printf 'release_label=%s\n' "${release_label}"
+ printf 'release_tag=%s\n' "${release_label}"
+ printf 'release_date_json=%s\n' "${release_date_json}"
+ printf 'notes_start_tag=%s\n' "${notes_start_tag}"
+ } >>"${GITHUB_OUTPUT}"
+
+ - name: Checkout selected source
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ repository: ${{ github.repository }}
+ ref: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_sha || inputs.source_sha }}
+ path: source
+
+ # Plan release artifacts before any release side effects. release_scope:
+ # all -> every catalog SDK + every catalog container (default)
+ # sdks -> every catalog SDK, no containers
+ # containers -> every catalog container, no SDKs
+ # custom -> exactly sdk_ids + container_ids (either may be empty)
+ # Container image bits are not built here; container entries are recorded
+ # in the manifest as typed artifacts and the release consumer stages them
+ # from the dev registry by this release's source SHA.
+ - name: Plan release assets
+ id: plan-assets
+ shell: bash
+ env:
+ RELEASE_SCOPE: ${{ inputs.release_scope }}
+ SDK_IDS: ${{ inputs.sdk_ids }}
+ CONTAINER_IDS: ${{ inputs.container_ids }}
+ run: |
+ set -euo pipefail
+
+ catalog="source/release/assets.yaml"
+ [[ -f "${catalog}" ]] || { echo "::error::release asset catalog is missing: ${catalog}"; exit 1; }
+
+ # Read and validate a catalog section ('.sdk' / '.container') into a
+ # JSON id array: non-empty string ids, no duplicates.
+ read_catalog() {
+ local section="$1" kind="$2" ids
+ ids="$(yq -o=json "${section} // []" "${catalog}" | jq -c 'map(.id)')"
+ if ! jq -e 'type == "array" and all(.[]; type == "string" and length > 0)' <<<"${ids}" >/dev/null; then
+ echo "::error::release/assets.yaml ${kind} must contain non-empty string id values"
+ exit 1
+ fi
+ if ! jq -e 'length == (unique | length)' <<<"${ids}" >/dev/null; then
+ echo "::error::release/assets.yaml ${kind} contains duplicate ids"
+ exit 1
+ fi
+ printf '%s' "${ids}"
+ }
+
+ # Parse a comma-separated custom id list and validate it against its
+ # catalog: no empty entries, no duplicates, no unknown ids.
+ parse_custom_ids() {
+ local raw="$1" kind="$2" catalog_json="$3" parsed unknown
+ parsed="$(jq -Rnc --arg value "${raw}" '$value | split(",") | map(gsub("^\\s+|\\s+$"; ""))')"
+ if jq -e 'any(.[]; length == 0)' <<<"${parsed}" >/dev/null; then
+ echo "::error::${kind}_ids contains an empty entry"
+ exit 1
+ fi
+ if ! jq -e 'length == (unique | length)' <<<"${parsed}" >/dev/null; then
+ echo "::error::${kind}_ids contains duplicate entries"
+ exit 1
+ fi
+ unknown="$(jq -nc --argjson selected "${parsed}" --argjson catalog "${catalog_json}" '$selected - $catalog')"
+ if [[ "${unknown}" != "[]" ]]; then
+ echo "::error::unknown ${kind} id(s): $(jq -r 'join(", ")' <<<"${unknown}")"
+ exit 1
+ fi
+ printf '%s' "${parsed}"
+ }
+
+ sdk_catalog="$(read_catalog '.sdk' sdk)"
+ container_catalog="$(read_catalog '.container' container)"
+
+ case "${RELEASE_SCOPE}" in
+ all | sdks | containers | custom)
+ ;;
+ *)
+ echo "::error::Unknown release_scope: ${RELEASE_SCOPE} (expected all, sdks, containers, or custom)"
+ exit 1
+ ;;
+ esac
+
+ if [[ "${RELEASE_SCOPE}" != "custom" && ( -n "${SDK_IDS}" || -n "${CONTAINER_IDS}" ) ]]; then
+ echo "::error::sdk_ids/container_ids can only be used when release_scope is custom"
+ exit 1
+ fi
+
+ case "${RELEASE_SCOPE}" in
+ all)
+ selected_sdk_ids="${sdk_catalog}"
+ selected_container_ids="${container_catalog}"
+ ;;
+ sdks)
+ selected_sdk_ids="${sdk_catalog}"
+ selected_container_ids='[]'
+ ;;
+ containers)
+ selected_sdk_ids='[]'
+ selected_container_ids="${container_catalog}"
+ ;;
+ custom)
+ selected_sdk_ids='[]'
+ selected_container_ids='[]'
+ [[ -n "${SDK_IDS}" ]] && selected_sdk_ids="$(parse_custom_ids "${SDK_IDS}" sdk "${sdk_catalog}")"
+ [[ -n "${CONTAINER_IDS}" ]] && selected_container_ids="$(parse_custom_ids "${CONTAINER_IDS}" container "${container_catalog}")"
+ ;;
+ esac
+
+ total="$(jq -n --argjson s "${selected_sdk_ids}" --argjson c "${selected_container_ids}" '($s | length) + ($c | length)')"
+ if [[ "${total}" == "0" ]]; then
+ echo "::error::release selection is empty; choose a non-empty release_scope or provide sdk_ids/container_ids"
+ exit 1
+ fi
+
+ sdk_artifacts_json="$(jq -nc --argjson ids "${selected_sdk_ids}" '$ids | map({type: "sdk", id: .})')"
+ container_artifacts_json="$(jq -nc --argjson ids "${selected_container_ids}" '$ids | map({type: "container", id: .})')"
+ selected_artifacts_json="$(jq -nc --argjson sdks "${sdk_artifacts_json}" --argjson containers "${container_artifacts_json}" '$sdks + $containers')"
+ sdk_matrix="$(jq -nc --argjson artifacts "${sdk_artifacts_json}" '{include: $artifacts}')"
+ sdk_count="$(jq -r 'length' <<<"${selected_sdk_ids}")"
+ container_count="$(jq -r 'length' <<<"${selected_container_ids}")"
+
+ {
+ printf 'selected_artifacts_json=%s\n' "${selected_artifacts_json}"
+ printf 'sdk_count=%s\n' "${sdk_count}"
+ printf 'container_count=%s\n' "${container_count}"
+ printf 'sdk_matrix=%s\n' "${sdk_matrix}"
+ } >>"${GITHUB_OUTPUT}"
+
+ echo "Planned SDK release artifacts: $(jq -r 'if length == 0 then "(none)" else join(", ") end' <<<"${selected_sdk_ids}")"
+ echo "Planned container release artifacts: $(jq -r 'if length == 0 then "(none)" else join(", ") end' <<<"${selected_container_ids}")"
+
+ reserve-release-tag:
+ name: Reserve release tag
+ needs: plan-release
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - name: Checkout source at release SHA
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ ref: ${{ needs.plan-release.outputs.source_sha }}
+ fetch-tags: true
+
+ # Reserve the tag before build jobs so a tagged RC attempt consumes its RC number.
+ - name: Create and push tag
+ shell: bash
+ env:
+ CADENCE: ${{ inputs.cadence }}
+ RELEASE_TAG: ${{ needs.plan-release.outputs.release_tag }}
+ SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
+ run: |
+ set -euo pipefail
+
+ if [[ "${CADENCE}" == "nightly" ]]; then
+ echo "Nightly cadence: no release tag to reserve."
+ exit 0
+ fi
+
+ [[ -n "${RELEASE_TAG}" ]] || { echo "::error::release_tag is required"; exit 1; }
+ [[ -n "${SOURCE_SHA}" ]] || { echo "::error::source_sha is required"; exit 1; }
+
+ if git show-ref --tags --verify --quiet "refs/tags/${RELEASE_TAG}"; then
+ existing_sha="$(git rev-list -n 1 "refs/tags/${RELEASE_TAG}")"
+ if [[ "${existing_sha}" == "${SOURCE_SHA}" ]]; then
+ echo "Release tag '${RELEASE_TAG}' already points at '${SOURCE_SHA}'."
+ exit 0
+ fi
+
+ echo "::error::release tag '${RELEASE_TAG}' already points at '${existing_sha}', not '${SOURCE_SHA}'"
+ exit 1
+ fi
+
+ git tag "${RELEASE_TAG}"
+ git push origin "refs/tags/${RELEASE_TAG}"
+ echo "Created release tag '${RELEASE_TAG}' at '${SOURCE_SHA}'."
+
+ build-sdks:
+ name: Build SDK (${{ matrix.id }})
+ needs: [plan-release, reserve-release-tag]
+ # Skip cleanly for container-only releases (sdk_matrix would be empty).
+ if: ${{ needs.plan-release.outputs.sdk_count != '0' }}
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix: ${{ fromJson(needs.plan-release.outputs.sdk_matrix) }}
+ steps:
+ - name: Checkout workflow code
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ path: workflow
+
+ - name: Checkout source
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ repository: ${{ needs.plan-release.outputs.source_repo }}
+ ref: ${{ needs.plan-release.outputs.source_sha }}
+ path: source
+ fetch-depth: 0
+ fetch-tags: true
+
+ # Toolchain setup + version stamp + `uv build` are factored into the
+ # composite action, which ci.yaml's wheel-test job also calls so the
+ # test wheel and the published wheel come out of one code path. The
+ # action runs out of the workflow checkout above, not the
+ # release-target source checkout.
+ - name: Build SDK wheel
+ id: build-sdk-wheel
+ uses: ./workflow/.github/actions/build-nemo-platform-wheel
+ with:
+ package: ${{ matrix.id }}
+ source-root: source
+ out-dir: ${{ github.workspace }}/release-sdk-wheel
+ cadence: ${{ inputs.cadence }}
+ release-label: ${{ needs.plan-release.outputs.release_label }}
+ nightly-timestamp: ${{ needs.plan-release.outputs.nightly_timestamp }}
+
+ - name: Upload SDK wheel
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: release-sdk-${{ matrix.id }}
+ path: ${{ steps.build-sdk-wheel.outputs.wheel-path }}
+ if-no-files-found: error
+ overwrite: true
+ retention-days: 1
+
+ assemble-release-bundle:
+ name: Assemble release bundle
+ needs: [plan-release, reserve-release-tag, build-sdks]
+ # build-sdks is skipped for container-only releases; still assemble as long as
+ # planning and tag reservation succeeded and the SDK builds did not fail. The
+ # reserve-release-tag gate is required because build-sdks is also skipped when
+ # tag creation fails (its dependency failed), so its result alone would let a
+ # failed tag reservation still build and dispatch the bundle.
+ if: ${{ !cancelled() && needs.plan-release.result == 'success' && needs.reserve-release-tag.result == 'success' && (needs.build-sdks.result == 'success' || needs.build-sdks.result == 'skipped') }}
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ outputs:
+ # Selects the consumer's fetch path: nightly/RC = Actions artifact, stable = GitHub Release asset.
+ source: ${{ inputs.cadence != 'release' && 'artifact' || 'release' }}
+ bundle_artifact_id: ${{ steps.upload-release-bundle.outputs.artifact-id }}
+ bundle_artifact_name: release-bundle-${{ needs.plan-release.outputs.release_label }}
+ bundle_artifact_digest: ${{ steps.upload-release-bundle.outputs.artifact-digest }}
+ release_tag: ${{ steps.upload-release.outputs.release_tag }}
+ release_checksums_digest: ${{ steps.upload-release.outputs.checksums_digest }}
+ steps:
+ - name: Checkout workflow code
+ uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+
+ # No SDK wheels exist for container-only releases; skip the download
+ # (the metadata writer handles an empty/absent artifacts dir).
+ - name: Download SDK wheels
+ if: ${{ needs.plan-release.outputs.sdk_count != '0' }}
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: release-sdk-*
+ path: downloaded-sdk-artifacts
+ merge-multiple: false
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+ with:
+ python-version: "3.11"
+
+ - name: Write release bundle metadata
+ shell: bash
+ env:
+ CADENCE: ${{ inputs.cadence }}
+ RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
+ RELEASE_DATE_JSON: ${{ needs.plan-release.outputs.release_date_json }}
+ SELECTED_ARTIFACTS_JSON: ${{ needs.plan-release.outputs.selected_artifacts_json }}
+ SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
+ run: |
+ set -euo pipefail
+
+ uv run --no-project python .github/scripts/write_release_bundle_metadata.py \
+ --sdk-artifacts-dir downloaded-sdk-artifacts \
+ --bundle-dir release-bundle \
+ --selected-artifacts-json "${SELECTED_ARTIFACTS_JSON}" \
+ --cadence "${CADENCE}" \
+ --release-label "${RELEASE_LABEL}" \
+ --release-date-json "${RELEASE_DATE_JSON}" \
+ --source-sha "${SOURCE_SHA}"
+
+ - name: Verify release bundle checksums
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ cd release-bundle
+ sha256sum -c checksums.txt
+
+ # Nightly/RC: hand off via an Actions artifact. RCs still reserve a git tag, but do
+ # not create a GitHub Release page until the RC release-page flow is settled.
+ - name: Upload release bundle artifact
+ id: upload-release-bundle
+ if: inputs.cadence != 'release'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: release-bundle-${{ needs.plan-release.outputs.release_label }}
+ path: release-bundle/
+ if-no-files-found: error
+ overwrite: true
+
+ # Stable: attach each artifact as its own asset on the tag reserve-release-tag pushed, so
+ # users can grab a single file instead of one opaque blob.
+ - name: Upload release bundle to release
+ id: upload-release
+ if: inputs.cadence == 'release'
+ shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RELEASE_TAG: ${{ needs.plan-release.outputs.release_tag }}
+ CADENCE: ${{ inputs.cadence }}
+ NOTES_START_TAG: ${{ needs.plan-release.outputs.notes_start_tag }}
+ run: |
+ set -euo pipefail
+
+ [[ -n "${RELEASE_TAG}" ]] || { echo "::error::release_tag is required for ${CADENCE} releases"; exit 1; }
+
+ # checksums.txt covers release-manifest.json + every wheel, so its digest anchors the
+ # whole release. The dispatch carries it; the consumer re-verifies with sha256sum -c.
+ checksums_digest="$(sha256sum release-bundle/checksums.txt | cut -d' ' -f1)"
+
+ # Categorized notes are generated by GitHub from .github/release.yml (merged PRs grouped
+ # by their conventional-commit label). NOTES_START_TAG pins the range to the previous
+ # stable tag; when empty (no prior stable) GitHub falls back to its default previous-tag
+ # detection.
+ notes_range=()
+ if [[ -n "${NOTES_START_TAG:-}" ]]; then
+ notes_range=(--notes-start-tag "${NOTES_START_TAG}")
+ fi
+
+ # Wheels are absent for container-only releases; nullglob keeps the
+ # unmatched glob from being passed to gh literally.
+ shopt -s nullglob
+ wheels=(release-bundle/wheels/*.whl)
+ shopt -u nullglob
+
+ # Upload each artifact as its own asset. GitHub flattens asset names to basenames; the
+ # consumer restores wheels/ from the manifest paths. Re-run model: if a release already
+ # exists for this tag the command fails ("a release with the same tag name already
+ # exists"); delete it (gh release delete "${RELEASE_TAG}") to redo.
+ # Body is GitHub's generated, categorized notes (.github/release.yml); --title keeps the
+ # tag as the release name so no auto-title is generated.
+ gh release create "${RELEASE_TAG}" \
+ release-bundle/release-manifest.json \
+ release-bundle/checksums.txt \
+ "${wheels[@]}" \
+ --title "${RELEASE_TAG}" \
+ --generate-notes \
+ "${notes_range[@]}"
+
+ {
+ printf 'release_tag=%s\n' "${RELEASE_TAG}"
+ printf 'checksums_digest=%s\n' "${checksums_digest}"
+ } >>"${GITHUB_OUTPUT}"
+
+ dispatch-release-bundle:
+ name: Dispatch release bundle
+ needs: assemble-release-bundle
+ runs-on: ubuntu-latest
+ steps:
+ - name: Send release bundle pointer
+ shell: bash
+ env:
+ GH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
+ DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
+ BUNDLE_REPO: ${{ github.repository }}
+ SOURCE: ${{ needs.assemble-release-bundle.outputs.source }}
+ BUNDLE_WORKFLOW_RUN_ID: ${{ github.run_id }}
+ BUNDLE_WORKFLOW_RUN_ATTEMPT: ${{ github.run_attempt }}
+ BUNDLE_ARTIFACT_ID: ${{ needs.assemble-release-bundle.outputs.bundle_artifact_id }}
+ BUNDLE_ARTIFACT_NAME: ${{ needs.assemble-release-bundle.outputs.bundle_artifact_name }}
+ BUNDLE_ARTIFACT_DIGEST: ${{ needs.assemble-release-bundle.outputs.bundle_artifact_digest }}
+ RELEASE_TAG: ${{ needs.assemble-release-bundle.outputs.release_tag }}
+ RELEASE_CHECKSUMS_DIGEST: ${{ needs.assemble-release-bundle.outputs.release_checksums_digest }}
+ SEND_NOTIFICATIONS: ${{ inputs.send_notifications }}
+ run: |
+ set -euo pipefail
+
+ [[ -n "${GH_TOKEN:-}" ]] || { echo "::error::CI_DISPATCH_TOKEN must be set"; exit 1; }
+ [[ -n "${DISPATCH_REPO:-}" ]] || { echo "::error::CI_DISPATCH_REPO must be set"; exit 1; }
+ [[ -n "${BUNDLE_REPO:-}" ]] || { echo "::error::bundle repo is required"; exit 1; }
+
+ # assemble-release-bundle picks the storage backend by cadence (nightly/RC -> Actions
+ # artifact, stable -> GitHub Release asset) and reports it as `source`. The consumer uses
+ # this same field to fail-closed select its fetch path.
+ case "${SOURCE}" in
+ artifact)
+ [[ -n "${BUNDLE_ARTIFACT_ID:-}" ]] || { echo "::error::bundle artifact id is required"; exit 1; }
+ [[ -n "${BUNDLE_ARTIFACT_NAME:-}" ]] || { echo "::error::bundle artifact name is required"; exit 1; }
+ [[ -n "${BUNDLE_ARTIFACT_DIGEST:-}" ]] || { echo "::error::bundle artifact digest is required"; exit 1; }
+ bundle="$(jq -n \
+ --arg repo "${BUNDLE_REPO}" \
+ --arg workflow_run_id "${BUNDLE_WORKFLOW_RUN_ID}" \
+ --arg workflow_run_attempt "${BUNDLE_WORKFLOW_RUN_ATTEMPT}" \
+ --arg artifact_id "${BUNDLE_ARTIFACT_ID}" \
+ --arg artifact_name "${BUNDLE_ARTIFACT_NAME}" \
+ --arg artifact_digest "${BUNDLE_ARTIFACT_DIGEST}" \
+ --argjson send_notifications "${SEND_NOTIFICATIONS}" \
+ '{
+ source: "artifact",
+ repo: $repo,
+ workflow_run_id: $workflow_run_id,
+ workflow_run_attempt: $workflow_run_attempt,
+ artifact_id: $artifact_id,
+ artifact_name: $artifact_name,
+ artifact_digest: $artifact_digest,
+ send_notifications: $send_notifications
+ }')"
+ ;;
+ release)
+ [[ -n "${RELEASE_TAG:-}" ]] || { echo "::error::release tag is required"; exit 1; }
+ [[ -n "${RELEASE_CHECKSUMS_DIGEST:-}" ]] || { echo "::error::release checksums digest is required"; exit 1; }
+ bundle="$(jq -n \
+ --arg repo "${BUNDLE_REPO}" \
+ --arg workflow_run_id "${BUNDLE_WORKFLOW_RUN_ID}" \
+ --arg tag "${RELEASE_TAG}" \
+ --arg checksums_digest "${RELEASE_CHECKSUMS_DIGEST}" \
+ --argjson send_notifications "${SEND_NOTIFICATIONS}" \
+ '{
+ source: "release",
+ repo: $repo,
+ workflow_run_id: $workflow_run_id,
+ tag: $tag,
+ checksums_digest: $checksums_digest,
+ send_notifications: $send_notifications
+ }')"
+ ;;
+ *)
+ echo "::error::unknown bundle source: '${SOURCE}'"
+ exit 1
+ ;;
+ esac
+
+ jq -n --argjson bundle "${bundle}" \
+ '{event_type: "release-bundle-produced", client_payload: {bundle: $bundle}}' \
+ | gh api -X POST "/repos/${DISPATCH_REPO}/dispatches" --input -
diff --git a/.github/workflows/release-nightly.yaml b/.github/workflows/release-nightly.yaml
new file mode 100644
index 0000000000..c47c35a77c
--- /dev/null
+++ b/.github/workflows/release-nightly.yaml
@@ -0,0 +1,59 @@
+name: "Release: Nightly"
+
+on:
+ schedule:
+ # Runs Monday through Friday at 8:00 PM America/Los_Angeles.
+ - cron: "0 20 * * 1-5"
+ timezone: "America/Los_Angeles"
+ workflow_dispatch:
+ inputs:
+ source_sha:
+ description: "Optional exact source commit SHA. Defaults to the repository default branch."
+ required: false
+ type: string
+ default: ""
+ release_scope:
+ description: "Release artifact scope."
+ required: false
+ type: choice
+ default: all
+ options:
+ - all
+ - sdks
+ - containers
+ - custom
+ sdk_ids:
+ description: "Comma-separated SDK IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+ container_ids:
+ description: "Comma-separated container IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+ send_notifications:
+ description: "Send downstream release notifications."
+ required: false
+ type: boolean
+ default: true
+
+permissions:
+ contents: read
+
+jobs:
+ release-bundle:
+ name: Produce release bundle
+ permissions:
+ contents: write
+ uses: ./.github/workflows/release-bundle.yaml
+ with:
+ cadence: nightly
+ source_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.source_sha || '' }}
+ release_scope: ${{ github.event_name == 'workflow_dispatch' && inputs.release_scope || 'all' }}
+ sdk_ids: ${{ github.event_name == 'workflow_dispatch' && inputs.sdk_ids || '' }}
+ container_ids: ${{ github.event_name == 'workflow_dispatch' && inputs.container_ids || '' }}
+ send_notifications: ${{ github.event_name != 'workflow_dispatch' || inputs.send_notifications }}
+ secrets:
+ CI_DISPATCH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
+ CI_DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
diff --git a/.github/workflows/release-rc.yaml b/.github/workflows/release-rc.yaml
new file mode 100644
index 0000000000..759d1749d2
--- /dev/null
+++ b/.github/workflows/release-rc.yaml
@@ -0,0 +1,59 @@
+name: "Release: RC"
+
+on:
+ workflow_dispatch:
+ inputs:
+ source_sha:
+ description: "Exact source commit SHA to release from the selected workflow branch."
+ required: true
+ type: string
+ base_version:
+ description: "Base SemVer core release version, for example 1.0.0."
+ required: true
+ type: string
+ release_date:
+ description: "Optional. Provide only for first-time registration of this release version. Leave blank for reruns or when this release version is already registered. Format: YYYY-MM-DD."
+ required: false
+ type: string
+ default: ""
+ release_scope:
+ description: "Release artifact scope."
+ required: false
+ type: choice
+ default: all
+ options:
+ - all
+ - sdks
+ - containers
+ - custom
+ sdk_ids:
+ description: "Comma-separated SDK IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+ container_ids:
+ description: "Comma-separated container IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+
+permissions:
+ contents: read
+
+jobs:
+ release-bundle:
+ name: Produce release bundle
+ uses: ./.github/workflows/release-bundle.yaml
+ permissions:
+ contents: write
+ with:
+ cadence: rc
+ source_sha: ${{ inputs.source_sha }}
+ version: ${{ inputs.base_version }}
+ release_date: ${{ inputs.release_date }}
+ release_scope: ${{ inputs.release_scope }}
+ sdk_ids: ${{ inputs.sdk_ids }}
+ container_ids: ${{ inputs.container_ids }}
+ secrets:
+ CI_DISPATCH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
+ CI_DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
diff --git a/.github/workflows/release-stable.yaml b/.github/workflows/release-stable.yaml
new file mode 100644
index 0000000000..93b2c27e2c
--- /dev/null
+++ b/.github/workflows/release-stable.yaml
@@ -0,0 +1,116 @@
+name: "Release: Stable"
+run-name: Stable release ${{ inputs.version }} from ${{ inputs.source_sha }} by @${{ github.actor }}
+
+on:
+ workflow_dispatch:
+ inputs:
+ source_sha:
+ description: "Exact source commit SHA to release from the selected workflow branch."
+ required: true
+ type: string
+ version:
+ description: "Stable SemVer core release version, for example 1.0.0."
+ required: true
+ type: string
+ release_date:
+ description: "Optional. Provide only for first-time registration of this release version. Leave blank for reruns or when this release version is already registered. Format: YYYY-MM-DD."
+ required: false
+ type: string
+ default: ""
+ release_scope:
+ description: "Release artifact scope."
+ required: false
+ type: choice
+ default: all
+ options:
+ - all
+ - sdks
+ - containers
+ - custom
+ sdk_ids:
+ description: "Comma-separated SDK IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+ container_ids:
+ description: "Comma-separated container IDs for custom releases."
+ required: false
+ type: string
+ default: ""
+
+concurrency:
+ group: release-stable
+ cancel-in-progress: false
+
+permissions:
+ contents: read
+
+jobs:
+ preview-stable-release:
+ name: Preview stable release ${{ inputs.version }} from ${{ inputs.source_sha }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Summarize stable release request
+ shell: bash
+ env:
+ VERSION: ${{ inputs.version }}
+ SOURCE_SHA: ${{ inputs.source_sha }}
+ RELEASE_DATE: ${{ inputs.release_date }}
+ RELEASE_SCOPE: ${{ inputs.release_scope }}
+ SDK_IDS: ${{ inputs.sdk_ids }}
+ CONTAINER_IDS: ${{ inputs.container_ids }}
+ run: |
+ set -euo pipefail
+
+ source_link="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${SOURCE_SHA}"
+
+ {
+ echo "## Stable release approval preview"
+ echo
+ echo "| Field | Value |"
+ echo "| --- | --- |"
+ echo "| Version | ${VERSION} |"
+ echo "| Stable tag | ${VERSION} |"
+ echo "| Source SHA | [${SOURCE_SHA}](${source_link}) |"
+ echo "| Workflow branch | ${GITHUB_REF_NAME} |"
+ echo "| Release date | ${RELEASE_DATE:-not provided} |"
+ echo "| Release scope | ${RELEASE_SCOPE} |"
+ if [[ "${RELEASE_SCOPE}" == "custom" ]]; then
+ echo "| SDK IDs | ${SDK_IDS:-not provided} |"
+ echo "| Container IDs | ${CONTAINER_IDS:-not provided} |"
+ fi
+ echo "| Destination | Stable git tag, release bundle artifact, downstream \`release-bundle-produced\` dispatch |"
+ echo
+ echo "This preview is informational only. \`release-bundle.yaml\` still performs the authoritative validation before creating tags, building bundles, or dispatching downstream."
+ } >>"${GITHUB_STEP_SUMMARY}"
+
+ approve-stable-release:
+ name: Approve public stable ${{ inputs.version }} from ${{ inputs.source_sha }}
+ needs: preview-stable-release
+ runs-on: ubuntu-latest
+ environment: release-stable
+ steps:
+ - name: Record stable release approval
+ run: |
+ echo "release-stable approval passed for ${VERSION} from ${SOURCE_SHA}."
+ env:
+ VERSION: ${{ inputs.version }}
+ SOURCE_SHA: ${{ inputs.source_sha }}
+
+ release-bundle:
+ name: Produce release bundle
+ needs: approve-stable-release
+ uses: ./.github/workflows/release-bundle.yaml
+ permissions:
+ contents: write
+ with:
+ cadence: release
+ source_sha: ${{ inputs.source_sha }}
+ version: ${{ inputs.version }}
+ release_date: ${{ inputs.release_date }}
+ release_scope: ${{ inputs.release_scope }}
+ sdk_ids: ${{ inputs.sdk_ids }}
+ container_ids: ${{ inputs.container_ids }}
+ secrets:
+ CI_DISPATCH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }}
+ CI_DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
deleted file mode 100644
index 350a37e5f5..0000000000
--- a/.github/workflows/release.yaml
+++ /dev/null
@@ -1,1166 +0,0 @@
-name: Release
-run-name: >-
- Release ${{ github.event_name == 'schedule' && 'nightly' || inputs['release-type'] }}
- by @${{ github.actor }}
-
-on:
- schedule:
- # Runs Monday through Friday at 8:00 PM America/Los_Angeles.
- - cron: "0 20 * * 1-5"
- timezone: "America/Los_Angeles"
- workflow_dispatch:
- inputs:
- release-type:
- description: "Release type."
- required: false
- type: choice
- default: nightly
- options:
- - nightly
- - stable
- source-sha:
- description: "Exact source commit SHA. Required for stable releases; optional for nightlies."
- required: false
- type: string
- default: ""
- version:
- description: "Stable SemVer core release version, for example 1.0.0."
- required: false
- type: string
- default: ""
- release-scope:
- description: "Release artifact scope."
- required: false
- type: choice
- default: all
- options:
- - all
- - wheels
- - containers
- - helm
- - custom
- wheel-ids:
- description: >-
- Custom only. Comma-separated wheel IDs.
- Allowed: nemo-platform, nemo-platform-plugin.
- required: false
- type: string
- default: ""
- container-ids:
- description: >-
- Custom only. Comma-separated container IDs.
- Allowed: nmp-api, nmp-cpu-tasks, nmp-automodel-tasks,
- nmp-automodel-training, nmp-unsloth-training, auditor-tasks,
- safe-synthesizer-tasks.
- required: false
- type: string
- default: ""
- include-helm:
- description: "Include the Helm chart if defining a strict list of release artifacts instead of the default (release everything)."
- required: false
- type: boolean
- default: false
- update-ngc-metadata:
- description: "Synchronize NGC metadata from the this branch."
- required: false
- type: boolean
- default: false
- send-notifications:
- description: "Send release start and end notifications."
- required: false
- type: boolean
- default: true
- dry-run:
- description: >-
- Validate and package selected artifacts without publishing, dispatching,
- polling, deploying, or notifying.
- required: false
- type: boolean
- default: false
-
-concurrency:
- group: release-${{ inputs['dry-run'] && github.run_id || 'live' }}
- cancel-in-progress: false
-
-permissions:
- contents: read
-
-env:
- # Releasable artifact catalog.
- #
- # Keep this in the workflow so contributors can see release intent without
- # chasing a second config file. When this changes, update the matching
- # workflow_dispatch input descriptions above; GitHub does not support dynamic
- # descriptions in the Run workflow form.
- #
- # Wheel fields:
- # id - value users type in wheel-ids and the release artifact id
- # package - package passed to uv build --package
- # path - package directory checked before any later release work starts
- RELEASE_WHEELS_JSON: >-
- [
- {"id":"nemo-platform","package":"nemo-platform","path":"packages/nemo_platform"},
- {"id":"nemo-platform-plugin","package":"nemo-platform-plugin","path":"packages/nemo_platform_plugin"}
- ]
- # Container fields:
- # id - value users type in container-ids and the published image name
- # target - docker buildx bake target that must exist in docker-bake.hcl
- #
- # Every container id must also have .github/assets/ngc/containers/.md so
- # the published image has matching NGC catalog metadata.
- RELEASE_CONTAINERS_JSON: >-
- [
- {"id":"nmp-api","target":"nmp-api-docker"},
- {"id":"nmp-cpu-tasks","target":"nmp-cpu-tasks-docker"},
- {"id":"nmp-automodel-tasks","target":"nmp-automodel-tasks-docker"},
- {"id":"nmp-automodel-training","target":"nmp-automodel-training-docker"},
- {"id":"nmp-unsloth-training","target":"nmp-unsloth-training"},
- {"id":"auditor-tasks","target":"auditor-tasks-docker"},
- {"id":"safe-synthesizer-tasks","target":"safe-synthesizer-tasks-docker"}
- ]
- RELEASE_HELM_ID: nemo-platform
- RELEASE_HELM_PATH: k8s/helm
- RELEASE_HELM_REGISTRY: nvcr.io/0921617854601259/nemo-platform
- RELEASE_NIGHTLY_WHEEL_INDEX: https://pypi.nvidia.com
- RELEASE_STABLE_WHEEL_INDEX: https://pypi.org/simple
- RELEASE_NIGHTLY_CONTAINER_REGISTRY: ghcr.io/nvidia-nemo/nemo-platform
- RELEASE_STABLE_CONTAINER_REGISTRY: nvcr.io/nvidia/nemo-platform
- # Make this package public in GitHub Packages after its first push.
- RELEASE_NIGHTLY_HELM_OCI_REGISTRY: oci://ghcr.io/nvidia-nemo/nemo-platform
- # Stable charts must be promoted from RELEASE_HELM_REGISTRY before polling.
- RELEASE_STABLE_HELM_REPOSITORY: https://helm.ngc.nvidia.com/nvidia/nemo-platform
- RELEASE_NGC_CATALOG_BASE: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-platform
-
-jobs:
- plan-release:
- name: Plan and validate release inputs
- runs-on: ubuntu-latest
- outputs:
- release_type: ${{ steps.plan.outputs.release_type }}
- release_scope: ${{ steps.plan.outputs.release_scope }}
- source_sha: ${{ steps.plan.outputs.source_sha }}
- version: ${{ steps.plan.outputs.version }}
- release_label: ${{ steps.plan.outputs.release_label }}
- nightly_timestamp: ${{ steps.plan.outputs.nightly_timestamp }}
- wheel_version: ${{ steps.wheel-version.outputs.wheel_version }}
- wheel_ids: ${{ steps.plan.outputs.wheel_ids }}
- container_ids: ${{ steps.plan.outputs.container_ids }}
- has_wheels: ${{ steps.plan.outputs.has_wheels }}
- has_containers: ${{ steps.plan.outputs.has_containers }}
- include_helm: ${{ steps.plan.outputs.include_helm }}
- update_ngc_metadata: ${{ steps.plan.outputs.update_ngc_metadata }}
- send_notifications: ${{ steps.plan.outputs.send_notifications }}
- dry_run: ${{ steps.plan.outputs.dry_run }}
- steps:
- # This job is intentionally the first gate. It resolves the user's
- # requested artifacts, checks custom ids against the inline catalog, then
- # validates the selected source tree has the wheel package paths, bake
- # targets, and NGC container overview files needed by later jobs.
- - name: Resolve release plan
- id: plan
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- with:
- github-token: ${{ github.token }}
- script: |
- const allWheels = JSON.parse(process.env.RELEASE_WHEELS_JSON);
- const allContainers = JSON.parse(process.env.RELEASE_CONTAINERS_JSON);
- const allWheelIds = allWheels.map((wheel) => wheel.id);
- const allContainerIds = allContainers.map((container) => container.id);
- const inputs = context.payload.inputs ?? {};
- const isManual = context.eventName === "workflow_dispatch";
- const releaseType = isManual ? inputs["release-type"] : "nightly";
- const releaseScope = isManual ? (inputs["release-scope"] || "all") : "all";
- const updateNgcMetadata = isManual && inputs["update-ngc-metadata"] === "true";
- const sendNotifications = inputs["send-notifications"] !== "false";
- const dryRun = isManual && inputs["dry-run"] === "true";
-
- let sourceSha = isManual ? (inputs["source-sha"] ?? "").trim() : context.sha;
- const version = releaseType === "stable" ? (inputs.version ?? "").trim() : "";
-
- if (releaseType === "stable") {
- if (!/^[0-9a-f]{40}$/i.test(sourceSha)) {
- throw new Error("Stable releases require an exact 40-character source SHA.");
- }
- if (!/^\d+\.\d+\.\d+$/.test(version)) {
- throw new Error("Stable releases require a MAJOR.MINOR.PATCH version.");
- }
- } else {
- if (sourceSha && !/^[0-9a-f]{40}$/i.test(sourceSha)) {
- throw new Error("A pinned nightly source must be an exact 40-character SHA.");
- }
- if (!sourceSha && dryRun) {
- sourceSha = context.sha;
- } else if (!sourceSha) {
- const defaultBranch = context.payload.repository.default_branch;
- const { data: commit } = await github.rest.repos.getCommit({
- ...context.repo,
- ref: defaultBranch,
- });
- sourceSha = commit.sha;
- }
- }
-
- sourceSha = sourceSha.toLowerCase();
-
- const presets = {
- all: { wheels: allWheels, containers: allContainers, includeHelm: true },
- wheels: { wheels: allWheels, containers: [], includeHelm: false },
- containers: { wheels: [], containers: allContainers, includeHelm: false },
- helm: { wheels: [], containers: [], includeHelm: true },
- };
- let selection = presets[releaseScope];
-
- if (releaseScope === "custom") {
- const selectArtifacts = (value, allowedArtifacts, allowedIds, label, inputName) => {
- if (!value.trim()) {
- return [];
- }
- const requestedList = value.split(",").map((id) => id.trim());
- const emptyEntry = requestedList.some((id) => id.length === 0);
- if (emptyEntry) {
- throw new Error(`${inputName} contains an empty entry.`);
- }
- const requestedIds = new Set(requestedList);
- if (requestedIds.size !== requestedList.length) {
- throw new Error(`${inputName} contains duplicate entries.`);
- }
- const unknownIds = [...requestedIds].filter(
- (id) => !allowedIds.includes(id),
- );
- if (unknownIds.length > 0) {
- throw new Error(
- `Unknown ${label} IDs: ${unknownIds.join(", ")}. `
- + `Allowed ${label} IDs: ${allowedIds.join(", ")}.`,
- );
- }
- return allowedArtifacts.filter((artifact) => requestedIds.has(artifact.id));
- };
-
- selection = {
- wheels: selectArtifacts(
- inputs["wheel-ids"] || "",
- allWheels,
- allWheelIds,
- "wheel",
- "wheel-ids",
- ),
- containers: selectArtifacts(
- inputs["container-ids"] || "",
- allContainers,
- allContainerIds,
- "container",
- "container-ids",
- ),
- includeHelm: inputs["include-helm"] === "true",
- };
-
- if (
- selection.wheels.length === 0
- && selection.containers.length === 0
- && !selection.includeHelm
- ) {
- throw new Error("A custom release must select at least one artifact.");
- }
- }
-
- if (!selection) {
- throw new Error(`Unknown release scope: ${releaseScope}.`);
- }
-
- const { wheels, containers, includeHelm } = selection;
- const wheelIds = wheels.map((wheel) => wheel.id);
- const containerIds = containers.map((container) => container.id);
-
- const nightlyTimestamp = releaseType === "nightly"
- ? new Date().toISOString().replace(/\D/g, "").slice(0, 14)
- : "";
- const releaseLabel = releaseType === "nightly"
- ? `nightly-${nightlyTimestamp}`
- : version;
-
- core.setOutput("release_type", releaseType);
- core.setOutput("release_scope", releaseScope);
- core.setOutput("source_sha", sourceSha);
- core.setOutput("version", version);
- core.setOutput("release_label", releaseLabel);
- core.setOutput("nightly_timestamp", nightlyTimestamp);
- core.setOutput("wheel_ids", JSON.stringify(wheelIds));
- core.setOutput("container_ids", JSON.stringify(containerIds));
- core.setOutput("wheel_artifacts", JSON.stringify(wheels));
- core.setOutput("container_artifacts", JSON.stringify(containers));
- core.setOutput("has_wheels", String(wheelIds.length > 0));
- core.setOutput("has_containers", String(containerIds.length > 0));
- core.setOutput("include_helm", String(includeHelm));
- core.setOutput("update_ngc_metadata", String(updateNgcMetadata));
- core.setOutput("send_notifications", String(sendNotifications));
- core.setOutput("dry_run", String(dryRun));
-
- core.info(
- `Planned ${releaseType} release ${releaseLabel} from ${sourceSha}`,
- );
-
- - name: Checkout selected source
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- ref: ${{ steps.plan.outputs.source_sha }}
- fetch-depth: 0
- fetch-tags: true
- persist-credentials: false
-
- - name: Ensure selected source is an ancestor of this workflow revision
- shell: bash
- env:
- SOURCE_SHA: ${{ steps.plan.outputs.source_sha }}
- run: git merge-base --is-ancestor "${SOURCE_SHA}" "${GITHUB_SHA}"
-
- - name: Validate selected release artifacts
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- WHEEL_ARTIFACTS: ${{ steps.plan.outputs.wheel_artifacts }}
- CONTAINER_ARTIFACTS: ${{ steps.plan.outputs.container_artifacts }}
- with:
- script: |
- const fs = require("fs");
- const path = require("path");
-
- const parseArtifacts = (envName) => JSON.parse(process.env[envName] || "[]");
- const readText = (filePath) => fs.readFileSync(filePath, "utf8");
- const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
- const requireFile = (filePath, message) => {
- if (!fs.existsSync(filePath)) {
- throw new Error(`${message}: ${filePath}`);
- }
- };
-
- const artifactList = (artifacts) => {
- if (artifacts.length === 0) {
- return "(none)";
- }
- return artifacts.map((artifact) => artifact.id).join(", ");
- };
-
- const validateWheel = (wheel) => {
- const pyprojectPath = path.join(wheel.path, "pyproject.toml");
- requireFile(
- pyprojectPath,
- `Wheel ${wheel.id} cannot be built because its package config is missing`,
- );
- const pyproject = readText(pyprojectPath);
- const namePattern = new RegExp(
- `^\\s*name\\s*=\\s*["']${escapeRegExp(wheel.package)}["']\\s*$`,
- "m",
- );
-
- if (!namePattern.test(pyproject)) {
- throw new Error(
- `Wheel ${wheel.id} expects package ${wheel.package}, `
- + `but ${pyprojectPath} does not declare that project name.`,
- );
- }
- };
-
- const readBakeTargets = (bakePath) => {
- requireFile(bakePath, "Container validation needs docker-bake.hcl");
- return new Set(
- [...readText(bakePath).matchAll(/^target\s+"([^"]+)"/gm)]
- .map((match) => match[1]),
- );
- };
-
- const ngcMetadataPath = (container) => path.join(
- ".github",
- "assets",
- "ngc",
- "containers",
- `${container.id}.md`,
- );
-
- const validateContainer = (container, bakeTargets) => {
- const bakePath = "docker-bake.hcl";
- if (!bakeTargets.has(container.target)) {
- throw new Error(
- `Container ${container.id} cannot be built because bake target `
- + `${container.target} is missing from ${bakePath}.`,
- );
- }
- requireFile(
- ngcMetadataPath(container),
- `Container ${container.id} is missing matching NGC metadata`,
- );
- };
-
- const writeSummary = async (wheels, containers) => {
- const rows = [
- [
- {data: "Type", header: true},
- {data: "Selected", header: true},
- ],
- ["Wheels", artifactList(wheels)],
- ["Containers", artifactList(containers)],
- ];
-
- await core.summary
- .addHeading("Release input validation")
- .addTable(rows)
- .write();
- };
-
- const wheels = parseArtifacts("WHEEL_ARTIFACTS");
- const containers = parseArtifacts("CONTAINER_ARTIFACTS");
- const bakeTargets = readBakeTargets("docker-bake.hcl");
-
- wheels.forEach(validateWheel);
- containers.forEach((container) => validateContainer(container, bakeTargets));
-
- await writeSummary(wheels, containers);
-
- core.info("Selected release artifacts are valid for the checked-out source.");
-
- - name: Resolve wheel version
- id: wheel-version
- if: steps.plan.outputs.has_wheels == 'true'
- shell: bash
- env:
- WHEEL_IDS: ${{ steps.plan.outputs.wheel_ids }}
- RELEASE_TYPE: ${{ steps.plan.outputs.release_type }}
- RELEASE_VERSION: ${{ steps.plan.outputs.version }}
- NIGHTLY_TIMESTAMP: ${{ steps.plan.outputs.nightly_timestamp }}
- run: |
- set -euo pipefail
-
- if [[ "${RELEASE_TYPE}" == "nightly" ]]; then
- cadence="nightly"
- else
- cadence="release"
- fi
-
- args=(
- --source-root .
- --sdk-id "$(jq -r '.[0]' <<< "${WHEEL_IDS}")"
- --cadence "${cadence}"
- --nightly-timestamp "${NIGHTLY_TIMESTAMP}"
- --print-version
- )
- if [[ "${RELEASE_TYPE}" == "stable" ]]; then
- args+=(--release-label "${RELEASE_VERSION}")
- fi
- wheel_version="$(python3 .github/scripts/stamp_sdk_version.py "${args[@]}")"
- echo "wheel_version=${wheel_version}" >> "${GITHUB_OUTPUT}"
-
- notify-start:
- # Start alerts intentionally run for dry runs so the webhook can be tested.
- name: Notify release start
- needs: plan-release
- if: needs.plan-release.outputs.send_notifications == 'true'
- runs-on: ubuntu-latest
- steps:
- - name: Send Slack alert
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- SLACK_ALERTS_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }}
- CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }}
- INCLUDE_HELM: ${{ needs.plan-release.outputs.include_helm }}
- DRY_RUN: ${{ needs.plan-release.outputs.dry_run }}
- COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.plan-release.outputs.source_sha }}
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- RUN_NUMBER: ${{ github.run_number }}
- with:
- script: |
- const wheels = JSON.parse(process.env.WHEEL_IDS);
- const containers = JSON.parse(process.env.CONTAINER_IDS);
- const releaseType = process.env.RELEASE_TYPE;
- const title = releaseType === "stable"
- ? "*:ship: Release started*"
- : "*:crescent_moon: Nightly release started*";
- const lines = [
- title,
- `Release: ${process.env.RELEASE_LABEL}`,
- `Commit: <${process.env.COMMIT_URL}|${process.env.SOURCE_SHA.slice(0, 7)}>`,
- "",
- "*Artifacts:*",
- ];
- if (wheels.length > 0) {
- lines.push("*:python: Wheels to publish:*");
- wheels.forEach((wheel) => lines.push(`- ${wheel}`));
- }
- if (containers.length > 0) {
- lines.push("*:docker_: Containers to publish:*");
- containers.forEach((container) => lines.push(`- ${container}`));
- }
- if (process.env.INCLUDE_HELM === "true") {
- lines.push("*:helm: Helm chart to publish:*", "- nemo-platform");
- }
- lines.push(
- process.env.DRY_RUN === "true" && "Mode: dry run (no publishing)",
- `:link: <${process.env.RUN_URL}|Release run #${process.env.RUN_NUMBER}>`,
- );
- const response = await fetch(process.env.SLACK_ALERTS_WEBHOOK, {
- method: "POST",
- headers: {"Content-Type": "application/json"},
- body: JSON.stringify({text: lines.filter((line) => line !== false).join("\n")}),
- });
- if (!response.ok) {
- core.setFailed(`Slack webhook returned ${response.status}.`);
- }
-
- sync-ngc-metadata:
- name: Synchronize NGC metadata
- needs: plan-release
- if: >-
- needs.plan-release.outputs.update_ngc_metadata == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- uses: ./.github/workflows/ngc-metadata.yaml
- secrets:
- AIRE_NGC_GITHUB_PLATFORM_RW: ${{ secrets.AIRE_NGC_GITHUB_PLATFORM_RW }}
-
- dispatch-release-registration:
- name: Dispatch release registration
- needs: plan-release
- if: >-
- needs.plan-release.outputs.release_type == 'stable' &&
- needs.plan-release.outputs.dry_run != 'true'
- runs-on: ubuntu-latest
- steps:
- - name: Dispatch registration workflow
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
- RELEASE_VERSION: ${{ needs.plan-release.outputs.version }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- SOURCE_RUN_URL: >-
- ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }}
- CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }}
- INCLUDE_HELM: ${{ needs.plan-release.outputs.include_helm }}
- with:
- github-token: ${{ secrets.CI_DISPATCH_TOKEN }}
- script: |
- const [owner, repo] = process.env.DISPATCH_REPO.split("/");
-
- await github.rest.repos.createDispatchEvent({
- owner,
- repo,
- event_type: "register-release-artifacts",
- client_payload: {
- version: process.env.RELEASE_VERSION,
- source_sha: process.env.SOURCE_SHA,
- source_run_url: process.env.SOURCE_RUN_URL,
- wheel_ids: JSON.parse(process.env.WHEEL_IDS),
- container_ids: JSON.parse(process.env.CONTAINER_IDS),
- helm_id: process.env.INCLUDE_HELM === "true"
- ? process.env.RELEASE_HELM_ID
- : null,
- },
- });
-
- dispatch-wheel-stage:
- name: Dispatch wheel builds
- needs: plan-release
- if: >-
- needs.plan-release.outputs.has_wheels == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- runs-on: ubuntu-latest
- steps:
- - name: Dispatch wheel builds
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- NIGHTLY_TIMESTAMP: ${{ needs.plan-release.outputs.nightly_timestamp }}
- WHEEL_VERSION: ${{ needs.plan-release.outputs.wheel_version }}
- WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }}
- with:
- github-token: ${{ secrets.CI_DISPATCH_TOKEN }}
- script: |
- const [owner, repo] = process.env.DISPATCH_REPO.split("/");
-
- await github.rest.repos.createDispatchEvent({
- owner,
- repo,
- event_type: "stage-wheels",
- client_payload: {
- ref: process.env.SOURCE_SHA,
- cadence: process.env.RELEASE_TYPE === "stable" ? "release" : "nightly",
- release_label: process.env.RELEASE_LABEL,
- nightly_timestamp: process.env.NIGHTLY_TIMESTAMP,
- wheel_version: process.env.WHEEL_VERSION,
- wheels: JSON.parse(process.env.WHEEL_IDS),
- },
- });
-
- dispatch-container-stage:
- name: Dispatch container builds
- needs: plan-release
- if: >-
- needs.plan-release.outputs.has_containers == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- runs-on: ubuntu-latest
- steps:
- - name: Dispatch container builds
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }}
- with:
- github-token: ${{ secrets.CI_DISPATCH_TOKEN }}
- script: |
- const [owner, repo] = process.env.DISPATCH_REPO.split("/");
-
- await github.rest.repos.createDispatchEvent({
- owner,
- repo,
- event_type: "release",
- client_payload: {
- ref: process.env.SOURCE_SHA,
- cadence: process.env.RELEASE_TYPE,
- version: process.env.RELEASE_LABEL,
- containers: JSON.parse(process.env.CONTAINER_IDS),
- },
- });
-
- stage-helm:
- name: Stage Helm chart
- needs: plan-release
- if: needs.plan-release.outputs.include_helm == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 30
- outputs:
- chart_version: ${{ steps.package.outputs.chart_version }}
- permissions:
- contents: read
- packages: write
- steps:
- - name: Check out selected source
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- ref: ${{ needs.plan-release.outputs.source_sha }}
- persist-credentials: false
-
- - name: Set up Helm
- uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
- with:
- version: v4.2.1
-
- - name: Install Helm push plugin
- if: >-
- needs.plan-release.outputs.release_type == 'stable' &&
- needs.plan-release.outputs.dry_run != 'true'
- run: >-
- helm plugin install https://github.com/chartmuseum/helm-push.git
- --version v0.11.1 --verify=false
-
- - name: Package Helm chart
- id: package
- shell: bash
- env:
- HELM_CHART: ${{ env.RELEASE_HELM_PATH }}
- NIGHTLY_IMAGE_REGISTRY: ${{ env.RELEASE_NIGHTLY_CONTAINER_REGISTRY }}
- STABLE_IMAGE_REGISTRY: ${{ env.RELEASE_HELM_REGISTRY }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_VERSION: ${{ needs.plan-release.outputs.version }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- NIGHTLY_TIMESTAMP: ${{ needs.plan-release.outputs.nightly_timestamp }}
- run: |
- set -euo pipefail
-
- chart_dir="$(mktemp -d)"
- package_dir="${RUNNER_TEMP}/helm-package"
- trap 'rm -rf "${chart_dir}"' EXIT
- cp -R "${HELM_CHART}/." "${chart_dir}/"
-
- helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
- helm dependency build "${chart_dir}"
-
- chart_version="$(yq -r '.version' "${chart_dir}/Chart.yaml")"
- if [[ "${RELEASE_TYPE}" == "nightly" ]]; then
- chart_version="${chart_version}-nightly-${NIGHTLY_TIMESTAMP}"
- release_registry="${NIGHTLY_IMAGE_REGISTRY}"
- else
- # TODO: Decide whether stable chart versions should track the platform
- # release version or remain independently managed in Chart.yaml.
- chart_version="${RELEASE_VERSION}"
- release_registry="${STABLE_IMAGE_REGISTRY}"
- fi
-
- yq -i ".platformConfig.platform.image_registry = \"${release_registry}\"" "${chart_dir}/values.yaml"
- yq -i ".api.image.repository = \"${release_registry}/nmp-api\"" "${chart_dir}/values.yaml"
- yq -i ".core.image.repository = \"${release_registry}/nmp-api\"" "${chart_dir}/values.yaml"
-
- mkdir -p "${package_dir}"
- helm package "${chart_dir}" \
- --version "${chart_version}" \
- --app-version "${RELEASE_LABEL}" \
- --destination "${package_dir}"
-
- echo "chart_version=${chart_version}" >> "${GITHUB_OUTPUT}"
- echo "chart_package=${package_dir}/nemo-platform-${chart_version}.tgz" >> "${GITHUB_OUTPUT}"
-
- - name: Report packaged Helm chart
- if: needs.plan-release.outputs.dry_run == 'true'
- env:
- CHART_PACKAGE: ${{ steps.package.outputs.chart_package }}
- run: |
- echo "::notice::Dry run: packaged ${CHART_PACKAGE}"
-
- - name: Push nightly Helm chart to GHCR
- if: >-
- needs.plan-release.outputs.release_type == 'nightly' &&
- needs.plan-release.outputs.dry_run != 'true'
- shell: bash
- env:
- GHCR_TOKEN: ${{ github.token }}
- HELM_OCI_REGISTRY: ${{ env.RELEASE_NIGHTLY_HELM_OCI_REGISTRY }}
- CHART_PACKAGE: ${{ steps.package.outputs.chart_package }}
- run: |
- set -euo pipefail
-
- printf '%s' "${GHCR_TOKEN}" | helm registry login ghcr.io \
- --username "${GITHUB_ACTOR}" --password-stdin
- helm push "${CHART_PACKAGE}" "${HELM_OCI_REGISTRY}"
-
- - name: Push stable Helm chart to NGC
- if: >-
- needs.plan-release.outputs.release_type == 'stable' &&
- needs.plan-release.outputs.dry_run != 'true'
- shell: bash
- env:
- HELM_PASSWORD: ${{ secrets.AIRE_NVCR_GITHUB }}
- RELEASE_REGISTRY: ${{ env.RELEASE_HELM_REGISTRY }}
- CHART_VERSION: ${{ steps.package.outputs.chart_version }}
- CHART_PACKAGE: ${{ steps.package.outputs.chart_package }}
- run: |
- set -euo pipefail
-
- helm_repository="https://helm.ngc.nvidia.com/${RELEASE_REGISTRY#nvcr.io/}"
- printf '%s' "${HELM_PASSWORD}" | helm repo add nemo-platform "${helm_repository}" \
- --username "\$oauthtoken" --password-stdin
-
- if helm search repo nemo-platform/nemo-platform --devel --version "${CHART_VERSION}" \
- | grep -qv 'No results found'; then
- echo "Helm chart version ${CHART_VERSION} already exists. Skipping."
- else
- helm cm-push "${CHART_PACKAGE}" nemo-platform
- fi
-
- dispatch-release-deployment:
- name: Dispatch release deployment
- needs: [plan-release, stage-helm, poll-final-release]
- if: >-
- needs.plan-release.outputs.include_helm == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- runs-on: ubuntu-latest
- steps:
- - name: Dispatch deployment creation
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- HELM_VERSION: ${{ needs.stage-helm.outputs.chart_version }}
- with:
- github-token: ${{ secrets.CI_DISPATCH_TOKEN }}
- script: |
- const [owner, repo] = process.env.DISPATCH_REPO.split("/");
-
- await github.rest.repos.createDispatchEvent({
- owner,
- repo,
- event_type: "create-release-deployment",
- client_payload: {
- ref: process.env.SOURCE_SHA,
- cadence: process.env.RELEASE_TYPE === "stable" ? "release" : "nightly",
- release_label: process.env.RELEASE_LABEL,
- helm_version: process.env.HELM_VERSION,
- },
- });
-
- poll-final-release:
- name: Wait for published release artifacts
- needs:
- - plan-release
- - dispatch-release-registration
- - dispatch-wheel-stage
- - dispatch-container-stage
- - stage-helm
- if: >-
- !cancelled() &&
- needs.plan-release.result == 'success' &&
- !contains(needs.*.result, 'failure') &&
- !contains(needs.*.result, 'cancelled')
- runs-on: ubuntu-latest
- timeout-minutes: 240
- permissions:
- contents: read
- env:
- POLL_INTERVAL_SECONDS: "30"
- steps:
- - name: Skip final artifact polling
- if: needs.plan-release.outputs.dry_run == 'true'
- run: |
- echo "::notice::Dry run: skipped final artifact polling"
-
- - name: Set up Helm for chart polling
- if: >-
- needs.plan-release.outputs.include_helm == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
- with:
- version: v4.2.1
-
- - name: Wait for selected wheels in PyPI
- if: >-
- needs.plan-release.outputs.has_wheels == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- shell: bash
- env:
- WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }}
- WHEEL_CATALOG: ${{ env.RELEASE_WHEELS_JSON }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- WHEEL_VERSION: ${{ needs.plan-release.outputs.wheel_version }}
- run: |
- set -euo pipefail
-
- if [[ "${RELEASE_TYPE}" == "nightly" ]]; then
- wheel_index="${RELEASE_NIGHTLY_WHEEL_INDEX}"
- else
- wheel_index="${RELEASE_STABLE_WHEEL_INDEX}"
- fi
-
- while IFS= read -r wheel_id; do
- package="$(jq -r --arg id "${wheel_id}" '.[] | select(.id == $id) | .package' <<< "${WHEEL_CATALOG}")"
- filename_prefix="${package//-/_}-${WHEEL_VERSION}"
- wheel_url="${wheel_index}/${package}/"
-
- until curl --silent --location "${wheel_url}" | grep -Fq "${filename_prefix}"; do
- echo "Waiting for ${package}==${WHEEL_VERSION} at ${wheel_url}"
- sleep "${POLL_INTERVAL_SECONDS}"
- done
- echo "Found ${package}==${WHEEL_VERSION}"
- done < <(jq -r '.[]' <<< "${WHEEL_IDS}")
-
- # Both final container registries are public, so this check deliberately
- # uses docker manifest inspect without credentials.
- - name: Wait for selected containers in the final registry
- if: >-
- needs.plan-release.outputs.has_containers == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- shell: bash
- env:
- CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }}
- CONTAINER_TAG: ${{ needs.plan-release.outputs.release_label }}
- CONTAINER_REGISTRY: >-
- ${{ needs.plan-release.outputs.release_type == 'nightly' &&
- env.RELEASE_NIGHTLY_CONTAINER_REGISTRY ||
- env.RELEASE_STABLE_CONTAINER_REGISTRY }}
- run: |
- set -euo pipefail
-
- while IFS= read -r container_id; do
- ref="${CONTAINER_REGISTRY}/${container_id}:${CONTAINER_TAG}"
- until docker manifest inspect "${ref}" >/dev/null 2>&1; do
- echo "Waiting for ${ref}"
- sleep "${POLL_INTERVAL_SECONDS}"
- done
- echo "Found ${ref}"
- done < <(jq -r '.[]' <<< "${CONTAINER_IDS}")
-
- # Both final Helm chart locations are public, so this check deliberately
- # uses Helm without registry credentials.
- - name: Wait for Helm chart in the final registry
- if: >-
- needs.plan-release.outputs.include_helm == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- shell: bash
- env:
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- CHART_ID: ${{ env.RELEASE_HELM_ID }}
- CHART_VERSION: ${{ needs.stage-helm.outputs.chart_version }}
- NIGHTLY_HELM_OCI_REGISTRY: ${{ env.RELEASE_NIGHTLY_HELM_OCI_REGISTRY }}
- STABLE_HELM_REPOSITORY: ${{ env.RELEASE_STABLE_HELM_REPOSITORY }}
- run: |
- set -euo pipefail
-
- if [[ "${RELEASE_TYPE}" == "nightly" ]]; then
- chart_ref="${NIGHTLY_HELM_OCI_REGISTRY}/${CHART_ID}"
- chart_is_available() {
- helm show chart "${chart_ref}" --version "${CHART_VERSION}" >/dev/null 2>&1
- }
- else
- chart_is_available() {
- helm repo add nemo-platform "${STABLE_HELM_REPOSITORY}" --force-update >/dev/null 2>&1 \
- && helm search repo nemo-platform/nemo-platform --devel --version "${CHART_VERSION}" \
- | grep -qv 'No results found'
- }
- fi
-
- until chart_is_available; do
- echo "Waiting for ${CHART_ID}==${CHART_VERSION}"
- sleep "${POLL_INTERVAL_SECONDS}"
- done
- echo "Found ${CHART_ID}==${CHART_VERSION}"
-
- alert-poll-delay:
- name: Alert if release polling is delayed
- needs:
- - plan-release
- - dispatch-release-registration
- - dispatch-wheel-stage
- - dispatch-container-stage
- - stage-helm
- if: >-
- !cancelled() &&
- needs.plan-release.result == 'success' &&
- needs.plan-release.outputs.send_notifications == 'true' &&
- needs.plan-release.outputs.dry_run != 'true' &&
- !contains(needs.*.result, 'failure') &&
- !contains(needs.*.result, 'cancelled')
- runs-on: ubuntu-latest
- timeout-minutes: 125
- permissions:
- actions: read
- steps:
- - name: Alert if final artifact polling exceeds two hours
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- SLACK_ALERTS_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.plan-release.outputs.source_sha }}
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- RUN_NUMBER: ${{ github.run_number }}
- with:
- script: |
- // The poll job has a four-hour timeout; alert after two hours.
- const pollJobName = "Wait for published release artifacts";
- const deadline = Date.now() + (2 * 60 * 60 * 1000);
- const sleep = (milliseconds) => new Promise((resolve) => {
- setTimeout(resolve, milliseconds);
- });
- const findPollJob = async () => {
- const {data} = await github.rest.actions.listJobsForWorkflowRun({
- ...context.repo,
- run_id: context.runId,
- per_page: 100,
- });
- return data.jobs.find((job) => job.name === pollJobName);
- };
-
- while (Date.now() < deadline) {
- const pollJob = await findPollJob();
- if (pollJob?.status === "completed") {
- core.info("Final artifact polling completed before the alert threshold.");
- return;
- }
- await sleep(Math.min(60_000, deadline - Date.now()));
- }
-
- const pollJob = await findPollJob();
- if (pollJob?.status === "completed") {
- core.info("Final artifact polling completed at the alert threshold.");
- return;
- }
-
- const title = process.env.RELEASE_TYPE === "stable"
- ? "*:warning: Release artifact polling delayed*"
- : "*:warning: Nightly artifact polling delayed*";
- const lines = [
- title,
- `Release: ${process.env.RELEASE_LABEL}`,
- `Commit: <${process.env.COMMIT_URL}|${process.env.SOURCE_SHA.slice(0, 7)}>`,
- "",
- "Final artifact polling has exceeded two hours.",
- "",
- `:link: <${process.env.RUN_URL}|Release run #${process.env.RUN_NUMBER}>`,
- ];
- const response = await fetch(process.env.SLACK_ALERTS_WEBHOOK, {
- method: "POST",
- headers: {"Content-Type": "application/json"},
- body: JSON.stringify({text: lines.join("\n")}),
- });
- if (!response.ok) {
- core.setFailed(`Slack webhook returned ${response.status}.`);
- }
-
- create-github-release:
- name: Create GitHub Release
- needs: [plan-release, poll-final-release]
- if: >-
- needs.plan-release.outputs.release_type == 'stable' &&
- needs.plan-release.outputs.release_scope == 'all' &&
- needs.plan-release.outputs.dry_run != 'true' &&
- needs.poll-final-release.result == 'success'
- runs-on: ubuntu-latest
- permissions:
- contents: write
- steps:
- - name: Check out release history
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- with:
- ref: ${{ needs.plan-release.outputs.source_sha }}
- fetch-depth: 0
- fetch-tags: true
- persist-credentials: false
-
- - name: Create GitHub Release
- shell: bash
- env:
- GH_TOKEN: ${{ github.token }}
- RELEASE_TAG: ${{ needs.plan-release.outputs.version }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- run: |
- set -euo pipefail
-
- notes_start_tag="$(
- {
- git tag -l | grep -E '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' || true
- printf '%s\n' "${RELEASE_TAG}"
- } | sort -V -u | awk -v version="${RELEASE_TAG}" '$0 == version {print previous} {previous = $0}'
- )"
- notes_range=()
- if [[ -n "${notes_start_tag}" && "${notes_start_tag}" != "${RELEASE_TAG}" ]]; then
- notes_range=(--notes-start-tag "${notes_start_tag}")
- fi
-
- gh release create "${RELEASE_TAG}" \
- --target "${SOURCE_SHA}" \
- --title "${RELEASE_TAG}" \
- --generate-notes \
- "${notes_range[@]}"
-
- signal-deployment:
- name: Signal deployment
- needs: [plan-release, poll-final-release]
- if: >-
- needs.poll-final-release.result == 'success' &&
- needs.plan-release.outputs.dry_run != 'true'
- runs-on: ubuntu-latest
- steps:
- - run: echo "::notice::Placeholder - signal the completed release"
-
- notify-end:
- name: Notify release result
- needs:
- - plan-release
- - poll-final-release
- - create-github-release
- - signal-deployment
- - stage-helm
- if: >-
- !cancelled() &&
- needs.plan-release.outputs.send_notifications == 'true' &&
- needs.plan-release.outputs.dry_run != 'true'
- runs-on: ubuntu-latest
- steps:
- - name: Send Slack alert
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
- env:
- SLACK_ALERTS_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }}
- SLACK_RELEASE_WEBHOOK: ${{ secrets.SLACK_RELEASE_WEBHOOK }}
- RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }}
- RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }}
- SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }}
- COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.plan-release.outputs.source_sha }}
- WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }}
- WHEEL_CATALOG: ${{ env.RELEASE_WHEELS_JSON }}
- WHEEL_VERSION: ${{ needs.plan-release.outputs.wheel_version }}
- CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }}
- INCLUDE_HELM: ${{ needs.plan-release.outputs.include_helm }}
- CHART_VERSION: ${{ needs.stage-helm.outputs.chart_version }}
- NIGHTLY_WHEEL_INDEX: ${{ env.RELEASE_NIGHTLY_WHEEL_INDEX }}
- STABLE_WHEEL_INDEX: ${{ env.RELEASE_STABLE_WHEEL_INDEX }}
- NGC_CATALOG_BASE: ${{ env.RELEASE_NGC_CATALOG_BASE }}
- POLL_RESULT: ${{ needs.poll-final-release.result }}
- GITHUB_RELEASE_RESULT: ${{ needs.create-github-release.result }}
- DEPLOYMENT_RESULT: ${{ needs.signal-deployment.result }}
- RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
- RUN_NUMBER: ${{ github.run_number }}
- with:
- script: |
- const releaseType = process.env.RELEASE_TYPE;
- const results = [
- process.env.POLL_RESULT,
- process.env.GITHUB_RELEASE_RESULT,
- process.env.DEPLOYMENT_RESULT,
- ];
- const failed = results.some((result) => ["failure", "cancelled"].includes(result));
- const published = process.env.POLL_RESULT === "success" && !failed;
- const webhook = published
- ? process.env.SLACK_RELEASE_WEBHOOK
- : process.env.SLACK_ALERTS_WEBHOOK;
- const title = published
- ? (releaseType === "stable"
- ? "*:ship: Release publish complete*"
- : "*:crescent_moon: Nightly release publish complete*")
- : (releaseType === "stable"
- ? "*:alert: Release publish failed*"
- : "*:alert: Nightly release publish failed*");
- const lines = [
- title,
- `Release: ${process.env.RELEASE_LABEL}`,
- `Commit: <${process.env.COMMIT_URL}|${process.env.SOURCE_SHA.slice(0, 7)}>`,
- ];
-
- if (published) {
- const wheelIds = JSON.parse(process.env.WHEEL_IDS);
- const wheelCatalog = JSON.parse(process.env.WHEEL_CATALOG);
- const containerIds = JSON.parse(process.env.CONTAINER_IDS);
- lines.push("", "*Artifacts published:*");
- if (wheelIds.length > 0) {
- lines.push("*:python: Wheels published:*");
- for (const wheelId of wheelIds) {
- const wheel = wheelCatalog.find((candidate) => candidate.id === wheelId);
- const wheelIndex = releaseType === "nightly"
- ? process.env.NIGHTLY_WHEEL_INDEX
- : process.env.STABLE_WHEEL_INDEX;
- const wheelUrl = releaseType === "nightly"
- ? `${wheelIndex}/${wheel.package}/`
- : `${wheelIndex.replace(/\/simple$/, "/project")}/${wheel.package}/${process.env.WHEEL_VERSION}/`;
- lines.push(`- <${wheelUrl}|${wheel.package}: ${process.env.WHEEL_VERSION}>`);
- }
- }
- if (containerIds.length > 0) {
- lines.push("*:docker_: Containers published:*");
- for (const containerId of containerIds) {
- const container = releaseType === "stable"
- ? `<${process.env.NGC_CATALOG_BASE}/containers/${containerId}|${containerId}>`
- : containerId;
- lines.push(`- ${container}: ${process.env.RELEASE_LABEL}`);
- }
- }
- if (process.env.INCLUDE_HELM === "true") {
- const chart = releaseType === "stable"
- ? `<${process.env.NGC_CATALOG_BASE}/helm-charts/nemo-platform|nemo-platform>`
- : "nemo-platform";
- lines.push("*:helm: Helm chart published:*");
- lines.push(`- ${chart}: ${process.env.CHART_VERSION}`);
- }
- } else {
- lines.push(
- "",
- "*Final release status:*",
- `Final artifact poll: ${process.env.POLL_RESULT}`,
- `GitHub release: ${process.env.GITHUB_RELEASE_RESULT}`,
- `Deployment signal: ${process.env.DEPLOYMENT_RESULT}`,
- );
- }
-
- lines.push("", `:link: <${process.env.RUN_URL}|Release run #${process.env.RUN_NUMBER}>`);
- const response = await fetch(webhook, {
- method: "POST",
- headers: {"Content-Type": "application/json"},
- body: JSON.stringify({text: lines.join("\n")}),
- });
- if (!response.ok) {
- core.setFailed(`Slack webhook returned ${response.status}.`);
- }
diff --git a/.gitignore b/.gitignore
index 7032d23e77..9e47336c0b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -146,8 +146,3 @@ site/
# Generated Fern-only public API reference spec
docs/fern/openapi/openapi.public.yaml
-
-# nektos/act files commonly used
-.act-variables
-.act-secrets
-
diff --git a/RELEASING.md b/RELEASING.md
index 94974c82d9..74e44cd67f 100644
--- a/RELEASING.md
+++ b/RELEASING.md
@@ -1,153 +1,138 @@
# Releasing NeMo Platform
-[`release.yaml`](.github/workflows/release.yaml) is the single release workflow
-for NeMo Platform. It handles scheduled nightlies and manually dispatched
-nightly or stable releases. The release catalog is deliberately defined in that
-workflow so contributors can see and validate every releasable artifact in one
-place.
+This document describes the end-to-end process for cutting and shipping a new version of NeMo Platform.
-Anyone with permission to run repository workflows can start a release. A
-stable release requires a specific source commit and version; a nightly can use
-the default branch head.
+> **Who can release?** Any team member can trigger the workflow. The `release-stable` GitHub Actions environment requires approval from a member of the `nmp_devops` team before the workflow proceeds.
-## Before starting a stable release
+---
-Choose the exact 40-character commit SHA and the `MAJOR.MINOR.PATCH` version
-to release. The source must contain the desired generated SDKs. If the API
-surface changed since the last SDK update, update the SDKs before releasing:
+## Overview
+
+> The examples in this document use `0.1.2` as the version being released.
+
+```
+trigger release-stable.yaml with source_sha + version → nmp_devops approval → workflow tags source_sha → Platform-Deploy publishes to PyPI
+```
+
+Artifacts published on a stable release:
+- `nemo-platform` on [pypi.org](https://pypi.org/project/nemo-platform/)
+- `nemo-platform-plugin` on [pypi.org](https://pypi.org/project/nemo-platform-plugin/)
+
+Nightly builds go to `pypi.nvidia.com` (NVIDIA's internal/public PyPI mirror), **not** public PyPI.
+
+## Versioning model
+
+Release and nightly wheel versions are resolved at build time. The release workflow runs `.github/scripts/stamp_sdk_version.py`, then passes the resolved version to Hatch through `UV_DYNAMIC_VERSIONING_BYPASS`.
+
+Dynamic versioning is intentionally limited to packages that need release/nightly wheel metadata:
+- `packages/nemo_platform` (`nemo-platform`)
+- `packages/nemo_platform_plugin` (`nemo-platform-plugin`)
+- `sdk/python/nemo-platform` (`nemo-platform-sdk`, consumed by the released wrappers and SDK tooling)
+
+All other first-party workspace packages use static stub versions, normally `0.0.0`, because they are implementation packages rather than independently released artifacts. Do not add `nmp-dynamic-versioning` to another package unless that package is added to the release catalog or otherwise needs published wheel metadata.
+
+`packages/nmp_build_tools` centralizes the Hatch version source and its defaults, but that package itself is also an internal stub-version package. The OpenAPI specs are schema inputs for SDK generation and intentionally keep a fixed `info.version: 0.0.0`; package release versions should not be copied into the specs.
+
+---
+
+## Step 1 — Choose the source SHA and release version
+
+Pick the full 40-character commit SHA on `main` that should be released, plus the SemVer core version to publish, for example `0.1.2`. The stable workflow creates the release tag at `source_sha`, and the wheel build receives the package version from the workflow input.
+
+If the API surface changed since the last SDK update, regenerate the OpenAPI spec and SDKs before releasing:
```bash
make update-sdk
```
-This regenerates the OpenAPI specifications and synchronizes the SDKs. The
-specifications intentionally retain `info.version: 0.0.0`; do not copy the
-release version into them.
+This runs `make refresh-openapi` (regenerates `openapi/openapi.yaml` and plugin specs) and then syncs the Python and web SDKs via Stainless. Requires `STAINLESS_API_KEY` to be set — see `sdk/README.md` for setup instructions. The generated OpenAPI specs should keep `info.version: 0.0.0`.
-## Release catalog
+To find the right SHA:
-The catalog in [`release.yaml`](.github/workflows/release.yaml) is the source
-of truth for selection. Do not add a separate release manifest or configuration
-file. When changing the catalog, also update the matching `workflow_dispatch`
-input description in that workflow.
+```bash
+git log --oneline main | head -5
+# Pick the commit to release and copy its full 40-character SHA.
+```
-| Type | IDs |
-| --- | --- |
-| Wheels | `nemo-platform`, `nemo-platform-plugin` |
-| Containers | `nmp-api`, `nmp-cpu-tasks`, `nmp-automodel-tasks`, `nmp-automodel-training`, `nmp-unsloth-training`, `auditor-tasks`, `safe-synthesizer-tasks` |
-| Helm chart | `nemo-platform` |
+---
-For every selected wheel, the workflow checks that its package configuration
-declares the expected project name. For every selected container, it checks the
-Docker Bake target and the corresponding
-`.github/assets/ngc/containers/.md` overview file. These checks happen
-before any external release work is dispatched.
+## Step 2 — Trigger the stable release workflow
-## Starting a release
+Navigate to the [`release-stable.yaml` workflow](https://github.com/NVIDIA-NeMo/nemo-platform/actions/workflows/release-stable.yaml) and click **Run workflow**.
-Open the [Release workflow](https://github.com/NVIDIA-NeMo/nemo-platform/actions/workflows/release.yaml)
-and select **Run workflow**. The form shows the allowed custom artifact IDs.
+| Input | Required | Description |
+|---|---|---|
+| `source_sha` | Yes | The full 40-character commit SHA to release from (must be on `main`). |
+| `version` | Yes | SemVer core version string to release, e.g. `0.1.2`. This becomes the stable git tag and wheel version. |
+| `release_date` | No | `YYYY-MM-DD`. Provide only on the first run for a given version; leave blank on reruns. |
+| `release_scope` | No | `all` (default) releases every catalog SDK and container. Use `sdks`, `containers`, or `custom` for narrower releases. |
+| `sdk_ids` | No | Comma-separated SDK IDs for `release_scope: custom`; must exist in `release/assets.yaml`. |
+| `container_ids` | No | Comma-separated container IDs for `release_scope: custom`; must exist in `release/assets.yaml`. |
-| Input | Use |
-| --- | --- |
-| `release-type` | `nightly` by default. Select `stable` for a full release. |
-| `source-sha` | Required for stable releases. Optional for nightlies; a normal nightly with no SHA uses the current default-branch head. A dry-run nightly with no SHA uses the workflow commit so a branch can be validated. |
-| `version` | Required for stable releases. Enter the `MAJOR.MINOR.PATCH` release version. |
-| `release-scope` | `all` by default. Select `wheels`, `containers`, `helm`, or `custom` for a subset. |
-| `wheel-ids`, `container-ids` | Comma-separated IDs used only with `release-scope: custom`. Each ID must be in the catalog above; duplicates and empty entries fail validation. |
-| `include-helm` | Includes the Helm chart in a custom release. |
-| `update-ngc-metadata` | Also runs the reusable NGC metadata workflow for `nemo-platform` and `nemo-platform-dev`. It checks out the workflow ref, normally `main`. |
-| `send-notifications` | Sends Slack start and final-status notifications. Defaults to `true`. |
-| `dry-run` | Validates the selected source and packages the selected Helm chart, but does not publish, dispatch external work, poll, create a GitHub release, or signal deployment. The start notification intentionally still runs when notifications are enabled. |
-
-Examples:
-
-| Goal | Inputs |
-| --- | --- |
-| Scheduled-style nightly | Leave `release-type` as `nightly` and use the default `all` scope. |
-| Stable full release | `release-type: stable`, `source-sha: <40-character SHA>`, `version: `, `release-scope: all`. |
-| One container | `release-scope: custom`, `container-ids: nmp-automodel-tasks`. |
-| Helm-only validation | `release-scope: helm`, `dry-run: true`. |
-
-Nightlies also run automatically Monday through Friday at 8:00 PM
-America/Los_Angeles.
-
-## What the workflow does
-
-1. Resolves the source, release label, selected artifacts, and wheel version.
- Stable versions use the supplied release version. Nightly labels use
- `nightly-` and the wheel version is resolved by
- `.github/scripts/stamp_sdk_version.py`.
-2. Checks out the selected source and validates the selected wheel paths,
- Docker Bake targets, and NGC overview files.
-3. Optionally synchronizes NGC metadata, when requested on a non-dry-run.
-4. Dispatches wheel, container, and stable-release registration work to the
- configured internal release repository. The selected source SHA, release
- type, version, and selected IDs are passed with the dispatch.
-5. Packages the Helm chart. A nightly chart uses the `Chart.yaml` version with
- `-nightly-` appended. A stable chart currently uses the
- stable release version. Whether stable chart versions should instead remain
- independently managed in `Chart.yaml` is an open policy decision.
-6. Waits for every selected final artifact to become public before continuing.
- The polling job times out after four hours and sends a Slack alert after two
- hours if it is still waiting.
-7. For a non-dry-run stable release with `release-scope: all`, creates the
- GitHub release and tag at the selected SHA. GitHub generates the release
- notes from the previous numeric SemVer tag. Subset releases do not create a
- GitHub release or tag.
-
-The deployment-signalling job is currently a placeholder. A successful poll
-means the selected artifacts are public; it does not yet mean a deployment was
-created by this workflow.
-
-## Publication destinations
-
-| Artifact | Nightly | Stable |
-| --- | --- | --- |
-| Wheels | [`pypi.nvidia.com`](https://pypi.nvidia.com) | [PyPI](https://pypi.org) |
-| Containers | `ghcr.io/nvidia-nemo/nemo-platform/:nightly-...` | `nvcr.io/nvidia/nemo-platform/:` and the public NGC catalog |
-| Helm chart | OCI chart at `oci://ghcr.io/nvidia-nemo/nemo-platform` | Initially staged at `0921617854601259/nemo-platform`, then promoted to the public [NGC Helm repository](https://helm.ngc.nvidia.com/nvidia/nemo-platform) |
-
-The stable Helm promotion is external to this workflow. The workflow polls the
-public NGC Helm repository, not the internal staging endpoint, before it marks
-the release complete.
-
-## Notifications
-
-With `send-notifications: true`:
-
-- A start message is sent to `SLACK_ALERTS_WEBHOOK`, including the selected
- artifacts and source commit. This is also sent for dry-runs so the webhook can
- be tested.
-- If final artifact polling exceeds two hours, a delay alert is sent to
- `SLACK_ALERTS_WEBHOOK`.
-- A successful non-dry-run release sends its completion message to
- `SLACK_RELEASE_WEBHOOK`. A failed or cancelled release sends its final status
- to `SLACK_ALERTS_WEBHOOK`.
-
-Dry-runs do not poll or send the delayed or final notification.
-
-## Required secrets
-
-| Secret | Used for |
-| --- | --- |
-| `CI_DISPATCH_REPO` | `owner/repo` of the internal release repository that receives release dispatches. |
-| `CI_DISPATCH_TOKEN` | Authenticating those cross-repository dispatches. |
-| `AIRE_NVCR_GITHUB` | Staging stable Helm charts in NGC. |
-| `AIRE_NGC_GITHUB_PLATFORM_RW` | Optional NGC metadata synchronization. |
-| `SLACK_ALERTS_WEBHOOK` | Release starts, delay alerts, and failed final statuses. |
-| `SLACK_RELEASE_WEBHOOK` | Successful final release status. |
+The workflow runs from the **`main` branch** by default. The `source_sha` must be reachable from that branch.
+
+**What the workflow does:**
+1. Validates inputs and previews the release.
+2. Pauses at the `approve-stable-release` gate — a member of the **`nmp_devops` team** must approve in the GitHub environment UI.
+3. Creates and pushes a git tag (e.g. `0.1.2`) at `source_sha`.
+4. Builds Python wheels for each SDK in `release/assets.yaml` using `.github/actions/build-nemo-platform-wheel`.
+5. Assembles a release bundle with checksums and metadata.
+6. Dispatches a `release-bundle-produced` event to the **Platform-Deploy** repository (`CI_DISPATCH_REPO` secret), which handles the actual PyPI publish.
+
+> If the PyPI publishing service is returning 5xx errors, the publish step in Platform-Deploy will fail. Wait for the service to recover and re-run the workflow with the same `source_sha` and `version` — the stable tag is already reserved so re-running is safe.
-## Verifying a completed release
+---
-The workflow summary records the selected wheels and containers. After a live
-release completes, check the selected artifacts at their destination above.
-For a full stable release, also verify that the GitHub tag and generated GitHub
-release point to the requested source SHA.
+## Step 3 — Verification
-For a wheel release, a quick client check is:
+Once the workflow completes, verify the release landed correctly:
```bash
uv tool upgrade nemo-platform
nemo --version
+# Expected: nemo version
```
+
+Also check:
+- [pypi.org/project/nemo-platform](https://pypi.org/project/nemo-platform/) — version and description updated.
+- [pypi.org/project/nemo-platform-plugin](https://pypi.org/project/nemo-platform-plugin/) — version updated.
+- GitHub: a tag (e.g. `0.1.2`) exists on the release commit.
+
+---
+
+## Container image eligibility
+
+The `container:` list in `release/assets.yaml` declares which container
+images are eligible for release publishing. The bundle workflow records the
+selected containers as `container`-typed entries in `release-manifest.json`,
+and the release consumer stages those images after the SDK publish, reading
+this list from this repository at the release ref. Eligibility is therefore
+version-pinned: re-staging an old tag publishes the container set declared at
+that commit.
+
+`release_scope` controls what a release includes (default `all`):
+
+| Scope | Includes |
+| --- | --- |
+| `all` | every catalog SDK + every catalog container (default) |
+| `sdks` | every catalog SDK, no containers |
+| `containers` | every catalog container, no SDKs |
+| `custom` | exactly the comma-separated `sdk_ids` + `container_ids` (either may be empty) |
+
+`custom` enables single-artifact or arbitrary-subset releases (for example a
+patch release of one container via `release_scope: custom`,
+`container_ids: nmp-automodel-tasks`); `containers` releases the whole
+container set with no SDK wheels.
+
+Adding an image here also requires a catalog metadata entry on the consumer
+side. Images are built into the dev registry tagged with this repository's
+commit SHA on every merge to main; release SHAs that predate that build
+trigger need a manual image build first.
+
+---
+
+## Nightly builds
+
+Nightly builds run automatically at 20:00 PT and publish to `pypi.nvidia.com`. They use the HEAD of `main` and version strings like `0.1.3.dev20260101120000`. No action required from the team.
+
+To trigger a nightly manually: [`release-nightly.yaml`](https://github.com/NVIDIA-NeMo/nemo-platform/actions/workflows/release-nightly.yaml) → **Run workflow** (no inputs required). Leave `send_notifications` enabled for real reruns; disable it only for quiet smoke/ad-hoc runs.
diff --git a/conftest.py b/conftest.py
index 64b1ec532f..5587105dce 100644
--- a/conftest.py
+++ b/conftest.py
@@ -284,20 +284,6 @@ def pytest_addoption(parser):
default=True,
help="Run integration tests (enabled by default)",
)
- parser.addoption(
- "--feature",
- action="append",
- default=[],
- metavar="NAME",
- help="Enable optional e2e feature sets (repeatable), e.g. --feature gpu",
- )
-
-
-def _e2e_features_enabled(config: pytest.Config) -> set[str]:
- features = {feature.lower() for feature in config.getoption("--feature") or []}
- if os.environ.get("RUN_NSS_K8S_E2E") == "1":
- features.add("gpu")
- return features
def pytest_runtest_setup(item):
@@ -316,9 +302,6 @@ def pytest_runtest_setup(item):
if "container_only" in [marker.name for marker in item.iter_markers()]:
if not os.environ.get("NMP_BASE_URL"):
skip_test("Skipping container-only test (requires NMP_BASE_URL)")
- if "requires_gpu" in [marker.name for marker in item.iter_markers()]:
- if "gpu" not in _e2e_features_enabled(item.config):
- skip_test("Skipping GPU container e2e (pass --feature gpu)")
from xdist.scheduler.loadgroup import LoadGroupScheduling # noqa: E402
diff --git a/docker-bake.hcl b/docker-bake.hcl
index cafad318de..2c662977bd 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -291,6 +291,7 @@ group "docker" {
group "docker-cpu" {
targets = [
"nmp-api-docker",
+ "nmp-core-docker",
"nmp-cpu-tasks-docker",
]
}
@@ -931,6 +932,9 @@ target "auditor-tasks-docker" {
contexts = {
root-lib-source-artifacts = "target:root-lib-source-artifacts"
root-busybox = "target:root-busybox"
+ nmp-python-base = "target:nmp-python-base"
+ nmp-python-dev-base = "target:nmp-python-dev-base"
+ root-distroless-base-3-11 = "target:root-distroless-base-3-11"
}
dockerfile = "docker/Dockerfile.auditor-tasks"
cache-to = maybe_registry_cache_to("auditor-tasks")
diff --git a/docker/Dockerfile.auditor-tasks b/docker/Dockerfile.auditor-tasks
index ab180bd9a4..f6d7846f5c 100644
--- a/docker/Dockerfile.auditor-tasks
+++ b/docker/Dockerfile.auditor-tasks
@@ -1,32 +1,15 @@
# hadolint global ignore=DL3059
-# The final image copies BusyBox shell utilities for parity with the task runtime.
+# distroless images don't have a real shell - & and | are not enabled
-ARG AUDITOR_PYTHON_IMAGE=python:3.13.14-slim-trixie
+ARG NMP_PYTHON_BASE=nmp-python-base
ARG CACHE_HOME=/tmp/.cache
-FROM ${AUDITOR_PYTHON_IMAGE} AS py-builder
+FROM nmp-python-dev-base AS py-builder
-ARG CACHE_HOME
-RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \
- build-essential \
- ca-certificates \
- curl \
- g++ \
- gcc \
- git \
- libffi-dev \
- libpq-dev \
- libssl-dev \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv
-
-ENV UV_COMPILE_BYTECODE=1 \
- UV_LINK_MODE=copy
+ARG TARGETARCH
-WORKDIR /app
+ARG CACHE_HOME
RUN mkdir -p ${CACHE_HOME} && chmod 777 ${CACHE_HOME}
# Create the directory structure to match the relative paths in pyproject.toml
@@ -74,8 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
"protobuf>=6.33.5,<7.0.0" \
"langchain-core>=1.2.22" \
"orjson>=3.11.6" \
- "cryptography>=48.0.1,<49" \
- "nltk>=3.10.0"
+ "nltk>=3.9.3"
# CVE Remediation
RUN --mount=type=cache,target=/root/.cache/uv \
@@ -86,17 +68,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \
"pyasn1>=0.6.3" \
"wheel>=0.46.2" \
"orjson>=3.11.6" \
- "cryptography>=48.0.1,<49" \
- "nltk>=3.10.0"
+ "nltk>=3.9.3"
-FROM ${AUDITOR_PYTHON_IMAGE} AS base
+FROM ${NMP_PYTHON_BASE} AS base
ARG USERNAME=nvs
ARG USER_UID=1000
ARG USER_GID=1000
WORKDIR /app
-RUN apt-get update && apt-get upgrade -y && \
- apt-get clean && \
- rm -rf /var/lib/apt/lists/*
RUN groupadd --gid ${USER_GID} ${USERNAME} && \
useradd --uid ${USER_UID} --gid ${USER_GID} --create-home --shell /bin/bash ${USERNAME}
diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training
index 4abe4bc51d..cf3a384756 100644
--- a/docker/Dockerfile.nmp-unsloth-training
+++ b/docker/Dockerfile.nmp-unsloth-training
@@ -5,9 +5,8 @@
#
# Install steps:
# 1. `uv pip install unsloth --torch-backend=auto` plus explicit
-# `transformers==5.3.0` and `huggingface-hub==1.3.0` pins. Unsloth's
-# resolver allows transformers 5.3.0, and the explicit pins keep later
-# platform glue installs from re-solving the HF stack.
+# `transformers==4.57.6` and `huggingface-hub==0.36.2` pins (transformers
+# 4.57.x requires hub <1.0; platform glue would otherwise pull hub 1.x).
# Unsloth's resolver still pulls unsloth_zoo
# and the rest of the HF stack (trl, peft, accelerate, datasets,
# bitsandbytes, xformers, etc.). `--overrides preserve_base_torch.txt`
@@ -71,8 +70,8 @@ ARG USERNAME=ubuntu
ARG USER_UID=1000
ARG USER_GID=1000
ARG UNSLOTH_VERSION=2026.6.1
-ARG TRANSFORMERS_VERSION=5.3.0
-ARG HF_HUB_VERSION=1.3.0
+ARG TRANSFORMERS_VERSION=4.57.6
+ARG HF_HUB_VERSION=0.36.2
ARG BITSANDBYTES_VERSION=0.49.2
ARG BNB_MAX_JOBS=10
diff --git a/docker/Dockerfile.safe-synthesizer-tasks b/docker/Dockerfile.safe-synthesizer-tasks
index 3330ae4088..15443c3464 100644
--- a/docker/Dockerfile.safe-synthesizer-tasks
+++ b/docker/Dockerfile.safe-synthesizer-tasks
@@ -11,10 +11,10 @@
ARG PYTHON_VERSION=3.13
ARG PYTHON_IMAGE=python:${PYTHON_VERSION}-slim-trixie
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.9.30
-ARG SAFE_SYNTHESIZER_RUNTIME_PACKAGE="nemo-safe-synthesizer[engine,cu129]==0.1.7"
+ARG SAFE_SYNTHESIZER_RUNTIME_PACKAGE="nemo-safe-synthesizer[engine,cu129]==0.1.2"
ARG FLASHINFER_CU129_INDEX_URL="https://flashinfer.ai/whl/cu129"
ARG PYTORCH_CU129_INDEX_URL="https://download.pytorch.org/whl/cu129"
-ARG VLLM_CU129_INDEX_URL="https://wheels.vllm.ai/ee0da84ab9e04ac7610e28580af62c365e898389/cu129"
+ARG VLLM_CU129_WHEEL="vllm @ https://github.com/vllm-project/vllm/releases/download/v0.20.0/vllm-0.20.0%2Bcu129-cp38-abi3-manylinux_2_31_x86_64.whl"
# =============================================================================
# uv binary
@@ -29,17 +29,13 @@ ARG PYTHON_IMAGE
ARG SAFE_SYNTHESIZER_RUNTIME_PACKAGE
ARG FLASHINFER_CU129_INDEX_URL
ARG PYTORCH_CU129_INDEX_URL
-ARG VLLM_CU129_INDEX_URL
+ARG VLLM_CU129_WHEEL
ARG CONTAINER_VARIANT=cu129
ARG USERNAME=nemo
ARG USER_UID=1000
ARG USER_GID=1000
-# Raise uv's HTTP retry budget above the default of 3. uv retries transient
-# failures (connection errors and 429/5xx status codes such as the intermittent
-# 503s from download.pytorch.org) with exponential backoff between attempts.
-ENV UV_HTTP_TIMEOUT=120 \
- UV_HTTP_RETRIES=8
+ENV UV_HTTP_TIMEOUT=120
COPY --from=uv /uv /uvx /usr/local/bin/
@@ -114,10 +110,11 @@ RUN printf '%s\n' \
printf '%s\n' \
"${SAFE_SYNTHESIZER_RUNTIME_PACKAGE}" \
> /tmp/safe-synthesizer-runtime.txt && \
+ printf '%s\n' \
+ "${VLLM_CU129_WHEEL}" \
+ > /tmp/vllm-cu129.txt && \
printf '%s\n' \
wandb==0.27.2 \
- 'cryptography>=48.0.1,<49' \
- 'pyarrow>=23.0.1,<24' \
> /tmp/safe-synthesizer-overrides.txt
RUN --mount=type=cache,target=/root/.cache/uv \
@@ -160,12 +157,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \
UV_CACHE_DIR=/root/.cache/uv uv pip install --python /opt/venv/bin/python \
--extra-index-url "${FLASHINFER_CU129_INDEX_URL}" \
--extra-index-url "${PYTORCH_CU129_INDEX_URL}" \
- --extra-index-url "${VLLM_CU129_INDEX_URL}" \
--index-strategy unsafe-best-match \
--torch-backend cu129 \
--overrides /tmp/safe-synthesizer-overrides.txt \
--constraints /tmp/safe-synthesizer-constraints.txt \
- --requirements /tmp/safe-synthesizer-runtime.txt
+ --requirements /tmp/safe-synthesizer-runtime.txt \
+ --requirements /tmp/vllm-cu129.txt
COPY pyproject.toml uv.lock ./
COPY packages/ packages/
@@ -182,13 +179,9 @@ RUN mkdir -p docs && \
# Skip the `nemo-platform` wrapper wheel: it bundles every first-party plugin and
# service, which drags unrelated source trees into this task image. The runtime
# imports come from nemo-platform-sdk, nemo-platform-plugin, and nmp-common.
-# The runtime layer above installs CVE-patched cryptography/pyarrow; do not let
-# the workspace lock downgrade them during plugin sync.
RUN --mount=type=cache,target=/root/.cache/uv \
UV_CACHE_DIR=/root/.cache/uv uv sync --package nemo-safe-synthesizer-plugin --no-dev --no-editable --inexact \
- --no-install-package nemo-platform \
- --no-install-package cryptography \
- --no-install-package pyarrow
+ --no-install-package nemo-platform
COPY docker/scripts/cve-cleanup.sh /bin/
RUN bash /bin/cve-cleanup.sh
diff --git a/docker/base/Dockerfile.nmp-workspace b/docker/base/Dockerfile.nmp-workspace
index 2839612dda..15c7748a1d 100644
--- a/docker/base/Dockerfile.nmp-workspace
+++ b/docker/base/Dockerfile.nmp-workspace
@@ -13,3 +13,4 @@ COPY packages/nmp_platform/ entrypoint/
COPY sdk/ sdk/
COPY script/ script/
COPY tools/ tools/
+COPY release/ release/
diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx
index 3dab764ba6..e0e3244df6 100644
--- a/docs/set-up/config-reference.mdx
+++ b/docs/set-up/config-reference.mdx
@@ -896,8 +896,8 @@ safe_synthesizer:
container_image_ref:
# default: '.nemo/safe-synthesizer-runtime'
runtime_venv: .nemo/safe-synthesizer-runtime
- # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.7'
- runtime_package: nemo-safe-synthesizer[engine,cu129]==0.1.7
+ # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.2'
+ runtime_package: nemo-safe-synthesizer[engine,cu129]==0.1.2
# default: '3.11'
runtime_python_version: '3.11'
runtime_python:
diff --git a/e2e/k8s/scripts/local_build_and_upgrade.sh b/e2e/k8s/scripts/local_build_and_upgrade.sh
index 9ab830c991..87ad05d541 100755
--- a/e2e/k8s/scripts/local_build_and_upgrade.sh
+++ b/e2e/k8s/scripts/local_build_and_upgrade.sh
@@ -8,13 +8,8 @@
# MINIKUBE_PROFILE - minikube profile name (default: minikube)
# NMP_REGISTRY - image registry (default: docker.io/my-registry)
# IMAGE_TAG - image tag (default: local-)
-# BUILD_ARCH - target platform (default: auto-detected from host)
-# MINIKUBE_GPU - when 1, start minikube with GPU passthrough
-# BUILD_SAFE_SYNTHESIZER - when 1, also build safe-synthesizer-tasks (amd64;
-# set BUILD_ARCH=linux/amd64)
-# BUILD_GPU - deprecated convenience alias: sets MINIKUBE_GPU=1 and
-# BUILD_SAFE_SYNTHESIZER=1
-# HELM_VALUES - values file (default: e2e/k8s/values/minikube.yaml)
+# BUILD_ARCH - target platform (default: auto-detected from host)
+# HELM_VALUES - values file (default: e2e/k8s/values/minikube.yaml)
set -e
@@ -23,21 +18,10 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}"
-if [ "${BUILD_GPU:-0}" = "1" ]; then
- MINIKUBE_GPU="${MINIKUBE_GPU:-1}"
- BUILD_SAFE_SYNTHESIZER="${BUILD_SAFE_SYNTHESIZER:-1}"
-fi
-MINIKUBE_GPU="${MINIKUBE_GPU:-0}"
-BUILD_SAFE_SYNTHESIZER="${BUILD_SAFE_SYNTHESIZER:-0}"
-
# Check if minikube is running
if ! minikube status -p "${MINIKUBE_PROFILE}" &>/dev/null; then
echo "Minikube profile ${MINIKUBE_PROFILE} is not running. Starting..."
- if [ "${MINIKUBE_GPU}" = "1" ]; then
- MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_gpu.sh"
- else
- MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_cpu.sh"
- fi
+ MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_cpu.sh"
fi
# Wait for minikube to be ready
@@ -64,22 +48,10 @@ eval "$(minikube -p "${MINIKUBE_PROFILE}" docker-env)"
IMAGE_REGISTRY="${NMP_REGISTRY}" \
BUILD_ARCH="$BUILD_ARCH" \
docker buildx bake docker-cpu --set "*.platform=$BUILD_ARCH"
-
- if [ "${BUILD_SAFE_SYNTHESIZER}" = "1" ]; then
- echo "Building safe-synthesizer-tasks (BUILD_SAFE_SYNTHESIZER=1)..."
- CI_COMMIT_SHA="$GIT_SHA" \
- BAKE_TAG="$IMAGE_TAG" \
- IMAGE_REGISTRY="${NMP_REGISTRY}" \
- BUILD_ARCH="$BUILD_ARCH" \
- docker buildx bake safe-synthesizer-tasks-docker --set "safe-synthesizer-tasks-docker.platform=$BUILD_ARCH"
- fi
)
echo "----------------------------------------"
echo "Images built with tag: $IMAGE_TAG"
-if [ "${BUILD_SAFE_SYNTHESIZER}" = "1" ]; then
- echo "Also built: safe-synthesizer-tasks"
-fi
echo "----------------------------------------"
# Delegate helm install to install_helm_e2e.sh
diff --git a/e2e/test_safe_synthesizer.py b/e2e/test_safe_synthesizer.py
deleted file mode 100644
index 33b2bf8fc3..0000000000
--- a/e2e/test_safe_synthesizer.py
+++ /dev/null
@@ -1,143 +0,0 @@
-"""Opt-in container E2E for Safe Synthesizer GPU jobs (Docker or Kubernetes).
-
-These tests exercise the full platform path: plugin job API -> Jobs controller ->
-GPU container step -> safe-synthesizer-tasks image -> Files results.
-
-Excluded from default kind-cpu CI (no GPU, no safe-synthesizer-tasks image).
-Run manually against minikube GPU, dev-blue, or a GPU-enabled Docker backend:
-
- # After nss-k8s-deploy.sh (or MINIKUBE_GPU=1 BUILD_SAFE_SYNTHESIZER=1 local_build_and_upgrade.sh)
- NMP_BASE_URL=http://localhost:30080 \
- uv run --frozen pytest e2e/test_safe_synthesizer.py -v --run-e2e --run-slow --feature gpu
-"""
-
-from __future__ import annotations
-
-import os
-import random
-import subprocess
-from datetime import date
-from pathlib import Path
-
-import pandas as pd
-import pytest
-from nemo_platform import NeMoPlatform
-from nemo_safe_synthesizer_plugin.sdk.job import SafeSynthesizerJob
-from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder
-
-_REPO_ROOT = Path(__file__).resolve().parents[1]
-_SETUP_MODEL_FILESETS = _REPO_ROOT / "plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py"
-
-_MIN_INPUT_ROWS = 200
-_DEFAULT_INPUT_ROWS = 250
-_DEFAULT_NUM_RECORDS = _DEFAULT_INPUT_ROWS
-
-_ICE_CREAM_FLAVORS = [
- "Vanilla",
- "Chocolate",
- "Strawberry",
- "Mint Chocolate Chip",
- "Cookies and Cream",
- "Pistachio",
- "Rocky Road",
- "Butter Pecan",
- "Coffee",
- "Mango Sorbet",
- "Salted Caramel",
- "Cookie Dough",
-]
-
-pytestmark = [
- pytest.mark.e2e,
- pytest.mark.container_only,
- pytest.mark.requires_gpu,
- pytest.mark.slow,
- pytest.mark.timeout(7200),
-]
-
-
-@pytest.fixture(scope="module")
-def nss_model_filesets(sdk: NeMoPlatform, _services: str) -> None:
- """Register HuggingFace-backed model filesets required by Safe Synthesizer tasks."""
- result = subprocess.run(
- [
- "uv",
- "run",
- "python",
- str(_SETUP_MODEL_FILESETS),
- "--files-api-url",
- _services,
- ],
- cwd=_REPO_ROOT,
- check=False,
- capture_output=True,
- text=True,
- )
- if result.returncode != 0:
- pytest.fail(
- f"Failed to register Safe Synthesizer model filesets\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
- )
-
-
-def _synthesis_dataset(rows: int | None = None) -> pd.DataFrame:
- """Build a tabular dataset suitable for Safe Synthesizer training (>= 200 rows).
-
- Schema matches plugins/nemo-safe-synthesizer/tests/e2e/test_local_synthesis.py:
- names, dates, and a categorical column with realistic variation.
- """
- if rows is None:
- rows = int(os.environ.get("NSS_E2E_INPUT_ROWS", str(_DEFAULT_INPUT_ROWS)))
- if rows < _MIN_INPUT_ROWS:
- raise ValueError(f"Safe Synthesizer container E2E requires at least {_MIN_INPUT_ROWS} input rows, got {rows}")
-
- faker_mod = pytest.importorskip("faker")
- fake = faker_mod.Faker()
- faker_mod.Faker.seed(42)
- random.seed(42)
-
- records = [
- {
- "name": fake.name(),
- "signup_date": fake.date_between_dates(
- date_start=date(2020, 1, 1),
- date_end=date(2026, 5, 4),
- ).isoformat(),
- "birthdate": fake.date_between_dates(
- date_start=date(1945, 1, 1),
- date_end=date(2006, 12, 31),
- ).isoformat(),
- "favorite_ice_cream_flavor": random.choice(_ICE_CREAM_FLAVORS),
- }
- for _ in range(rows)
- ]
- return pd.DataFrame.from_records(records)
-
-
-def test_safe_synthesizer_container_job_completes(
- sdk: NeMoPlatform,
- workspace: str,
- nss_model_filesets: None,
-) -> None:
- """Submit a GPU container job and verify synthetic data is produced."""
- num_records = int(os.environ.get("NSS_E2E_NUM_RECORDS", str(_DEFAULT_NUM_RECORDS)))
- if num_records < _MIN_INPUT_ROWS:
- raise ValueError(f"NSS_E2E_NUM_RECORDS must be at least {_MIN_INPUT_ROWS}, got {num_records}")
-
- job = (
- SafeSynthesizerJobBuilder(sdk, workspace=workspace)
- .with_data_source(_synthesis_dataset())
- .synthesize()
- .with_generate(num_records=num_records)
- .with_evaluate(enabled=True)
- .create_job()
- )
-
- nss_job = SafeSynthesizerJob(job.job_name, sdk, workspace=workspace)
- nss_job.wait_for_completion(poll_interval=15, verbose=True)
-
- summary = nss_job.fetch_summary()
- assert summary.timing.training_time_sec is not None
- assert summary.timing.generation_time_sec is not None
-
- synthetic = nss_job.fetch_data()
- assert len(synthetic) == num_records
diff --git a/packages/nemo_evaluator_sdk/examples/profbench/profbench.py b/packages/nemo_evaluator_sdk/examples/profbench/profbench.py
index 888dee118d..84f71e9a07 100644
--- a/packages/nemo_evaluator_sdk/examples/profbench/profbench.py
+++ b/packages/nemo_evaluator_sdk/examples/profbench/profbench.py
@@ -334,7 +334,7 @@ async def _score(self, input: MetricInput) -> ProfBenchRubricDetails:
task = input.row.data.get("task", {})
judge_request = ProfBenchJudgeRequest(
task_id=str(task.get("id", "")) if isinstance(task, dict) else "",
- prompt=str(inputs.get("instruction", "")) if isinstance(inputs, dict) else "",
+ prompt=str(inputs.get("prompt", "")) if isinstance(inputs, dict) else "",
response=output_text,
criterion_id=criterion.id,
criterion_description=criterion.description,
@@ -447,7 +447,7 @@ def load_profbench(
task = AgentEvalTask(
id=task_id,
intent=str(row["prompt"]),
- inputs={"instruction": row["prompt"], "domain": row.get("domain")},
+ inputs={"prompt": row["prompt"], "domain": row.get("domain")},
metrics=[ProfBenchRubricMetric(criteria=criteria, judge=judge, evidence_dir=evidence_dir)],
metadata={
"benchmark": "ProfBench",
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py
index f47e160c24..037bdadd37 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py
@@ -628,12 +628,10 @@ def _default_prompt_template(target: Model | Agent) -> dict[str, Any]:
def _task_row(task: AgentEvalTask) -> dict[str, Any]:
- # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}`
- # template): a dataset `prompt` column or the agent `instruction`.
return {
**task.inputs,
"task_id": task.id,
- "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"),
+ "prompt": task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent,
}
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py
index 4e7fb6ce04..c75d402f9d 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py
@@ -68,7 +68,7 @@ def __init__(
self._work_root = Path(work_root).expanduser() if work_root is not None else None
self._codex_bin = codex_bin
self._timeout_s = timeout_s
- self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt
+ self._prompt_builder = prompt_builder or default_codex_prompt
self._process_factory = process_factory or asyncio.create_subprocess_exec
self._runtime_name = runtime_name
@@ -95,17 +95,15 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC
evidence_dir.mkdir(parents=True, exist_ok=True)
workspace_dir.mkdir(parents=True, exist_ok=True)
+ prompt = self._prompt_builder(task)
prompt_path = evidence_dir / "prompt.txt"
task_path = evidence_dir / "task.json"
stdout_path = evidence_dir / "stdout.jsonl"
stderr_path = evidence_dir / "stderr.txt"
final_output_path = evidence_dir / "final_output.txt"
- # Persist the task for debugging, but never the grader-only fields: the docker variant mounts
- # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired
- # behavior) or `reference` (held-out ground truth) here would let the agent read them back out
- # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes.
- task_path.write_text(task.model_dump_json(indent=2, exclude={"intent", "reference"}), encoding="utf-8")
+ prompt_path.write_text(prompt, encoding="utf-8")
+ task_path.write_text(task.model_dump_json(indent=2), encoding="utf-8")
command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path)
process: Any | None = None
@@ -115,10 +113,6 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC
# synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this
# runs on the event loop shared by every concurrent task, so a blocking seed would stall them all.
seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY))
- # Build the prompt after seeding and inside the guarded block: an instruction-less task
- # raises here, failing just this task instead of aborting the run (and seeding wins if both).
- prompt = self._prompt_builder(task)
- prompt_path.write_text(prompt, encoding="utf-8")
process = await self._process_factory(
*command,
stdin=subprocess.PIPE,
@@ -392,6 +386,30 @@ def print_codex_agent_models(*, codex_bin: str = "codex") -> None:
print(slug)
+def default_codex_prompt(task: AgentEvalTask) -> str:
+ """Frame a task for Codex as an agent that works in its current directory.
+
+ Task-agnostic: it states the intent and inputs and invites the agent to read/create/edit files,
+ rather than constraining the answer to a single text reply. Seed files (``inputs[SEED_FILES_INPUT_KEY]``)
+ are listed by name instead of dumped inline — the agent finds them already in its workspace. Pass a
+ custom :data:`CodexPromptBuilder` to the runtime to override this framing for a specific benchmark.
+ """
+ body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY}
+ lines = [f"Task id: {task.id}", f"Intent: {task.intent}"]
+ if body_inputs:
+ lines += ["", "Inputs:", json.dumps(body_inputs, indent=2, default=str)]
+ seeded = task.inputs.get(SEED_FILES_INPUT_KEY)
+ if isinstance(seeded, Mapping) and seeded:
+ lines += ["", "These files are already in your working directory:"]
+ lines += [f" - {path}" for path in seeded]
+ lines += [
+ "",
+ "Complete the task by working in your current directory. You may read, create, and edit files "
+ "as needed. When you are done, briefly summarize what you changed.",
+ ]
+ return "\n".join(lines) + "\n"
+
+
def _failed_codex_trial(
task: AgentEvalTask,
evidence_dir: Path,
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py
index 489db70355..43354d00b9 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py
@@ -136,15 +136,13 @@ async def _run_task(
) -> AgentEvalTrial:
evidence_dir = self._evidence_dir(index, task, config)
evidence_dir.mkdir(parents=True, exist_ok=True)
+ prompt = _task_prompt(task)
+ manifest = self._build_manifest(task, sdk)
+ agent = self._build_agent(manifest, sdk)
client = self._build_client(sdk)
sandbox = None
try:
- # Build the prompt inside the guarded block: an instruction-less task raises here and fails
- # just this task rather than aborting the whole run.
- prompt = task.agent_prompt()
- manifest = self._build_manifest(task, sdk)
- agent = self._build_agent(manifest, sdk)
sandbox = await client.create(
manifest=manifest,
options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE),
@@ -165,7 +163,7 @@ def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any:
# workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose
# grader-only fields (e.g. ``reference`` held-out ground truth) to the agent.
entries: dict[str, Any] = {
- "instruction.md": sdk.File(content=task.agent_prompt().encode("utf-8")),
+ "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")),
"output": sdk.Dir(),
}
workspace_dir = task.inputs.get("workspace_dir")
@@ -297,6 +295,10 @@ def _validated_workspace_dir(workspace_dir: Any) -> Path:
return resolved
+def _task_prompt(task: AgentEvalTask) -> str:
+ return str(task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent)
+
+
async def _maybe_await(value: Awaitable[Any] | Any) -> Any:
if inspect.isawaitable(value):
return await value
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py
index 5d21d3fc9c..cc41c5d99d 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py
@@ -7,11 +7,7 @@
NeMo Fabric Python SDK and adapts each normalized Fabric ``RunResult`` into an
:class:`AgentEvalTrial`. The harness is chosen by the supplied Fabric config's
``harness.adapter_id`` (never inferred from a model); an optional ``model`` slug
-is applied as the config's default model, mirroring Fabric's own Harbor integration.
-
-Per-task settings (workspace, model, trajectory capture) are composed directly onto
-a copy of the supplied config via the SDK's config helpers (``model_copy`` +
-``enable_relay`` + ``environment``), rather than layered as profile overlays.
+is applied as a final profile overlay, mirroring Fabric's own Harbor integration.
Every task runs in its own fresh workspace: the runtime seeds it from
``inputs['files']`` (a no-op when there are none), runs the harness in it (via
@@ -42,7 +38,6 @@
CandidateEvidence,
EvidenceDescriptor,
)
-from pydantic import JsonValue
if TYPE_CHECKING:
# Annotations use nemo_fabric's real types (single source of truth). nemo_fabric is an optional
@@ -50,10 +45,9 @@
# loaded lazily at runtime (see ``run_tasks``). Drop the ty:ignore once nemo-fabric is a
# resolvable dependency and the checker can see it.
from nemo_fabric import ( # ty: ignore[unresolved-import]
- Fabric,
+ FabricClient,
FabricConfig,
FabricProfileConfig,
- RunOutput,
RunResult,
)
@@ -72,18 +66,18 @@
# Per-task workspace: where seed files are staged and where the harness reads/writes. We
# create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence.
_WORKSPACE_SUBDIR = "workspace"
+_WORKSPACE_PROFILE_NAME = "eval_workspace"
# Evidence key + descriptor kind for the staged workspace, consumed by the
# workspace-reading metrics.
_WORKSPACE_EVIDENCE_KEY = "workspace"
_WORKSPACE_EVIDENCE_KIND = "filesystem"
-# File-exporter output names we choose for the Relay ATIF/ATOF trajectory (Relay accepts these as inputs).
+# Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as inputs).
+_TRAJECTORY_PROFILE_NAME = "eval_trajectory"
_ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json"
_ATOF_FILENAME = "events.atof.jsonl"
-# Names for the trailing overlays that re-assert the evaluator-owned per-task settings (see
-# ``_eval_lock_profiles``): Fabric applies caller profiles over the config, so these must trail them.
-_WORKSPACE_PROFILE_NAME = "eval_workspace"
-_MODEL_PROFILE_NAME = "eval_model"
-_ARTIFACTS_PROFILE_NAME = "eval_artifacts"
+# Fabric telemetry-profile selectors (file exporter, no OTLP endpoint).
+_TELEMETRY_PROVIDER = "relay"
+_TELEMETRY_MODE = "sdk"
# ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence.
_ATIF_ARTIFACT_KIND = "atif"
@@ -127,77 +121,69 @@ async def run_tasks(
try:
# nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported
# lazily here rather than at module load.
- from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import]
+ from nemo_fabric import FabricClient, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import]
except ImportError as exc:
raise RuntimeError(_MISSING_FABRIC_MSG) from exc
resolved_config = config or AgentEvalRunConfig()
agent_config = FabricConfig.from_mapping(self._config)
- # Fail fast (once) if trajectory capture is requested but the nemo-relay gateway isn't
- # importable, rather than failing every task the same way inside the per-task guard.
- if self._capture_trajectory:
- try:
- import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import]
- except ImportError as exc:
- raise RuntimeError(_MISSING_RELAY_MSG) from exc
- # Caller-supplied profile overlays pass through as-is; this runtime's per-task workspace, model,
- # and trajectory settings are composed directly onto a copy of the config (config-first), not
- # layered as profiles.
- base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles]
+ base_profiles = self._build_profiles(FabricProfileConfig)
semaphore = asyncio.Semaphore(resolved_config.parallelism)
- # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle
- # context manager — so it is created once and reused across tasks with no cleanup.
- client = Fabric()
+ async with FabricClient() as client:
- async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial:
- async with semaphore:
- return await self._run_task(client, agent_config, base_profiles, index, task, resolved_config)
+ async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial:
+ async with semaphore:
+ return await self._run_task(
+ client, agent_config, base_profiles, FabricProfileConfig, index, task, resolved_config
+ )
- return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks)))
+ return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks)))
async def _run_task(
self,
- client: Fabric,
+ client: FabricClient,
agent_config: FabricConfig,
base_profiles: list[FabricProfileConfig],
+ profile_cls: type[FabricProfileConfig],
index: int,
task: AgentEvalTask,
config: AgentEvalRunConfig,
) -> AgentEvalTrial:
- # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules
- # lookup, not a re-load, so the type is used where it's constructed instead of threaded down.
- from nemo_fabric import FabricProfileConfig, RunRequest # ty: ignore[unresolved-import]
-
evidence_dir = self._evidence_dir(index, task, config)
evidence_dir.mkdir(parents=True, exist_ok=True)
+ profiles = list(base_profiles)
+ if self._capture_trajectory:
+ # Enable Relay's ATIF file exporter, writing the trajectory under this task's durable
+ # evidence dir; Fabric promotes the resulting file into RunResult.artifacts. Both the
+ # Fabric artifact root and the relay output dir must exist and be durable.
+ relay_dir = evidence_dir / _RELAY_SUBDIR
+ artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR
+ relay_dir.mkdir(parents=True, exist_ok=True)
+ artifacts_dir.mkdir(parents=True, exist_ok=True)
+ profiles.append(self._trajectory_profile(profile_cls, relay_dir=relay_dir, artifacts_dir=artifacts_dir))
+
# Every task runs in its own fresh workspace: seed any ``inputs['files']`` into it (a no-op when
- # there are none), point the harness at it, and expose it as ``workspace`` filesystem evidence —
- # a uniform per-task dir that maps cleanly onto a per-task container volume later. Seeding runs
- # inside the guarded block so a bad seed (a path escaping the workspace, an unresolvable fileset)
- # fails just this task, not the whole run; it is synchronous and may block (a fileset handler
- # downloads), so it is offloaded off the shared event loop.
+ # there are none), point the harness at it via ``environment.workspace``, and expose it as
+ # ``workspace`` filesystem evidence — a uniform per-task dir that maps cleanly onto a per-task
+ # container volume later. Seeding runs inside the guarded block so a bad seed (a path escaping
+ # the workspace, an unresolvable fileset) fails just this task, not the whole run; it is
+ # synchronous and may block (a fileset handler downloads), so it is offloaded off the shared
+ # event loop.
workspace_dir = evidence_dir / _WORKSPACE_SUBDIR
workspace_dir.mkdir(parents=True, exist_ok=True)
try:
- # Stage seed files into the workspace for their on-disk side effect; the prompt is the task
- # instruction only, so the returned paths are unused.
- await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY))
- task_config = self._compose_config(agent_config, evidence_dir, workspace_dir)
- # Caller ``base_profiles`` are applied by Fabric over the config; the evaluator-owned
- # settings are re-asserted as trailing overlays so they win over any caller profile.
- lock_profiles = self._eval_lock_profiles(
- FabricProfileConfig, workspace_dir=workspace_dir, evidence_dir=evidence_dir
- )
+ seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY))
+ profiles.append(self._workspace_profile(profile_cls, workspace_dir=workspace_dir))
result = await asyncio.wait_for(
- # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``.
client.run(
- task_config,
- profiles=[*base_profiles, *lock_profiles],
+ agent_config,
+ profiles=profiles,
+ input=_fabric_input(task, seeded_files),
+ request_id=task.id,
base_dir=self._base_dir,
- request=RunRequest(input=task.agent_prompt(), request_id=task.id),
),
timeout=self._timeout_s,
)
@@ -228,17 +214,13 @@ def _to_trial(
if result.status != "succeeded":
return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata)
- # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52),
- # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the
- # trial's ``JsonValue``-typed response.
- output = _normalize_output(result.output)
return AgentEvalTrial(
id=f"{task.id}:fabric",
task_id=task.id,
status=AgentEvalTrialStatus.COMPLETED,
output=AgentOutput(
- output_text=_extract_output_text(output),
- response=output,
+ output_text=_extract_output_text(result.output),
+ response=result.output,
metadata={**base_metadata, "evidence_dir": str(evidence_dir)},
),
evidence=self._evidence(result, result_path, workspace_dir),
@@ -315,90 +297,31 @@ def _failed_trial(
},
)
- def _compose_config(
- self,
- agent_config: FabricConfig,
- evidence_dir: Path,
- workspace_dir: Path,
- ) -> FabricConfig:
- # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules
- # lookup, not a re-load, so the type is used where it's constructed instead of threaded down.
- from nemo_fabric import EnvironmentConfig # ty: ignore[unresolved-import]
-
- # Config-first composition (the SDK's recommended in-memory pattern): copy the base config and
- # apply this task's workspace, model, and trajectory settings directly onto it, rather than
- # layering FabricProfileConfig overlays.
- cfg = agent_config.model_copy(deep=True)
-
- # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from
- # it). ``provider="local"`` is required by the native planner. Any config-supplied
- # environment.workspace is overridden per task.
- environment = cfg.environment or EnvironmentConfig(provider="local")
- environment.provider = environment.provider or "local"
- environment.workspace = str(workspace_dir)
- cfg.environment = environment
-
- # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor).
+ def _build_profiles(self, profile_cls: type[FabricProfileConfig]) -> list[FabricProfileConfig]:
+ profiles = [profile_cls.from_mapping(profile) for profile in self._profiles]
if self._model:
+ # Apply the model as a final profile overlay (mirrors nemo_fabric.integrations.harbor).
provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai"
- cfg.models["default"] = {"provider": provider, "model": self._model}
-
- if self._capture_trajectory:
- # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the
- # Fabric artifact root so the promoted ``trajectory-*.atif.json`` persists. Requires the
- # ``nemo-relay`` gateway on PATH in the runtime.
- relay_dir = evidence_dir / _RELAY_SUBDIR
- artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR
- relay_dir.mkdir(parents=True, exist_ok=True)
- artifacts_dir.mkdir(parents=True, exist_ok=True)
- cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir))
- cfg.runtime.artifacts = str(artifacts_dir)
- cfg.environment.artifacts = str(artifacts_dir)
-
- return cfg
-
- def _eval_lock_profiles(
- self,
- profile_cls: type[FabricProfileConfig],
- *,
- workspace_dir: Path,
- evidence_dir: Path,
- ) -> list[FabricProfileConfig]:
- # ``_compose_config`` composes the evaluator's per-task settings onto the config, but Fabric
- # applies caller-supplied profiles OVER the config (last-wins), so a caller profile could
- # otherwise override them. Re-assert the evaluator-owned settings here as trailing overlays —
- # applied after the caller profiles — so the per-task workspace (isolation + ``workspace``
- # evidence integrity), the model under evaluation, and the trajectory artifact location stay
- # authoritative and non-overridable.
- overlays = [
- profile_cls.from_mapping(
- {"name": _WORKSPACE_PROFILE_NAME, "environment": {"workspace": str(workspace_dir)}}
- )
- ]
- if self._model:
- provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai"
- overlays.append(
- profile_cls.from_mapping(
- {"name": _MODEL_PROFILE_NAME, "models": {"default": {"provider": provider, "model": self._model}}}
+ profiles.append(
+ profile_cls(
+ name="eval_model",
+ models={"default": {"provider": provider, "model": self._model}},
)
)
- if self._capture_trajectory:
- artifacts_dir = str(evidence_dir / _ARTIFACTS_SUBDIR)
- overlays.append(
- profile_cls.from_mapping(
- {
- "name": _ARTIFACTS_PROFILE_NAME,
- "runtime": {"artifacts": artifacts_dir},
- "environment": {"artifacts": artifacts_dir},
- }
- )
- )
- return overlays
-
- def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
+ return profiles
+
+ def _trajectory_profile(
+ self, profile_cls: type[FabricProfileConfig], *, relay_dir: Path, artifacts_dir: Path
+ ) -> FabricProfileConfig:
+ # Relay ATIF/ATOF file exporter (mode=sdk): the harness emits its trajectory to a local
+ # nemo-relay gateway, which writes ``trajectory-*.atif.json`` under ``relay_dir``. No OTLP
+ # collector endpoint is involved. Requires the ``nemo-relay`` gateway on PATH in the runtime.
+ # The Fabric artifact root is pinned to a durable dir so the promoted trajectory persists.
+ #
# The observability component is built from nemo_relay's own typed config objects so Relay owns
# its schema (no hand-maintained dict that silently drifts when Relay changes it); imported
- # lazily since nemo-relay, like nemo-fabric, is an optional native dependency.
+ # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. ``schema_version``
+ # is omitted — ``FabricProfileConfig`` defaults it.
try:
from nemo_relay.observability import ( # ty: ignore[unresolved-import]
AtifConfig,
@@ -410,6 +333,7 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
raise RuntimeError(_MISSING_RELAY_MSG) from exc
relay_dir_str = str(relay_dir)
+ artifacts_dir_str = str(artifacts_dir)
observability = ComponentSpec(
config=ObservabilityConfig(
atif=AtifConfig(
@@ -427,7 +351,33 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
),
)
)
- return {"version": 1, "components": [observability.to_dict()]}
+ return profile_cls.from_mapping(
+ {
+ "name": _TRAJECTORY_PROFILE_NAME,
+ "description": "Capture the agent trajectory as ATIF via the NeMo Relay file exporter.",
+ "runtime": {"artifacts": artifacts_dir_str},
+ "environment": {"artifacts": artifacts_dir_str},
+ "telemetry": {
+ "enabled": True,
+ "provider": _TELEMETRY_PROVIDER,
+ "mode": _TELEMETRY_MODE,
+ "output_dir": relay_dir_str,
+ "config": {"version": 1, "components": [observability.to_dict()]},
+ },
+ }
+ )
+
+ def _workspace_profile(self, profile_cls: type[FabricProfileConfig], *, workspace_dir: Path) -> FabricProfileConfig:
+ # Point the harness at this task's staged workspace via ``environment.workspace`` (the codex-cli
+ # adapter resolves its cwd from it). Set as a final profile overlay, the same mechanism the
+ # trajectory profile uses for ``environment.artifacts``.
+ return profile_cls.from_mapping(
+ {
+ "name": _WORKSPACE_PROFILE_NAME,
+ "description": "Run the harness in the per-task evaluation workspace seeded with the task inputs.",
+ "environment": {"workspace": str(workspace_dir)},
+ }
+ )
def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path:
root = self._work_root
@@ -438,16 +388,22 @@ def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunCon
return Path(root) / task_dir
-def _normalize_output(output: RunOutput | JsonValue) -> JsonValue:
- """Unwrap a Fabric ``RunResult.output`` into the plain JSON value the trial response stores.
+def _fabric_input(task: AgentEvalTask, seeded_files: Sequence[str] = ()) -> str:
+ """Frame the task as the harness's input text.
- Newer Fabric wraps output in a ``RunOutput`` (the RunOutput response contract), which is a
- ``Mapping``; copy it into a plain dict (equivalent to its ``to_mapping()``). Raw/older JSON outputs
- are already JSON values and pass through unchanged.
+ When files were staged into the workspace, list them by name and invite the agent to work in its
+ current directory rather than dumping their contents inline; the seed-files key is dropped from
+ the echoed inputs since those files are already on disk.
"""
- if isinstance(output, Mapping):
- return dict(output)
- return output
+ body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY}
+ lines = [f"Task id: {task.id}", f"Intent: {task.intent}"]
+ if body_inputs:
+ lines += ["", f"Inputs: {body_inputs}"]
+ if seeded_files:
+ lines += ["", "These files are already in your working directory:"]
+ lines += [f" - {path}" for path in seeded_files]
+ lines += ["", "Complete the task by reading, creating, and editing files in your current directory."]
+ return "\n".join(lines) + "\n"
def _extract_output_text(output: object) -> str | None:
diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py
index 83e5d64520..7d1a23ee7d 100644
--- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py
+++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py
@@ -97,23 +97,6 @@ def _id_must_not_be_empty(cls, value: str) -> str:
raise ValueError("task id must not be empty")
return value
- def agent_prompt(self) -> str:
- """The intent-free prompt handed to the agent under evaluation.
-
- Exactly the task's natural-language instruction (``inputs["instruction"]``), with no
- runtime-added framing. ``intent`` is deliberately never used: it is the eval-side description
- of the desired behavior (what the grader checks for), so exposing it to the agent is a
- reward-hacking hole.
-
- Raises ``ValueError`` when ``inputs["instruction"]`` is missing or empty; a task with no
- instruction cannot be evaluated, so the runner fails that task rather than running an agent on
- an empty prompt.
- """
- instruction = self.inputs.get("instruction")
- if instruction:
- return str(instruction)
- raise ValueError(f"task {self.id!r} has no instruction: set inputs['instruction']")
-
@field_serializer("metrics", when_used="json")
def _serialize_metrics(self, metrics: list[Metric]) -> list[dict[str, Any]]:
"""Serialize local metric instances as descriptors for run bundles."""
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py
index cc00788024..c78f50f7c9 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py
@@ -111,10 +111,9 @@ def __init__(self, command: tuple[str, ...]) -> None:
self.command = command
async def communicate(self, input: bytes) -> tuple[bytes, bytes]:
- # Default prompt is exactly the instruction from inputs — no runtime framing — and never
- # leaks the eval-side `intent`.
- assert input == b"Question?"
- assert b"Answer." not in input # intent stays eval-side
+ # Default prompt is task-agnostic: it states the task and invites workspace edits.
+ assert b"Task id: task/1" in input
+ assert b"Intent:" in input
final_output_path = Path(self.command[self.command.index("--output-last-message") + 1])
final_output_path.write_text("codex answer", encoding="utf-8")
return b'{"type":"event"}\n', b""
@@ -129,7 +128,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
work_root=tmp_path / "codex",
process_factory=fake_process_factory,
)
- task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Question?"})
+ task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"prompt": "Question?"})
trials = await runtime.run_tasks([task])
@@ -151,44 +150,6 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
assert (tmp_path / "codex" / "000000-task-1" / "stdout.jsonl").read_text(encoding="utf-8") == '{"type":"event"}\n'
-@pytest.mark.asyncio
-async def test_codex_task_json_omits_grader_only_fields(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
- # The docker variant mounts the evidence dir into the sandbox (danger-full-access), so the persisted
- # task.json must never carry grader-only fields — otherwise the agent could read `intent` (desired
- # behavior) or the held-out `reference` back out of /evidence and reward-hack. Enforced on the shared
- # base runtime so both the local and docker variants persist an agent-safe task.json.
- class FakeProcess:
- returncode = 0
-
- def __init__(self, command: tuple[str, ...]) -> None:
- self.command = command
-
- async def communicate(self, input: bytes) -> tuple[bytes, bytes]:
- final_output_path = Path(self.command[self.command.index("--output-last-message") + 1])
- final_output_path.write_text("ok", encoding="utf-8")
- return b"", b""
-
- async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
- return FakeProcess(command)
-
- monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}")
- runtime = codex_runtime.CodexCliAgentRuntime(work_root=tmp_path / "codex", process_factory=fake_process_factory)
- task = AgentEvalTask(
- id="task/1",
- intent="SECRET_GRADER_INTENT",
- inputs={"instruction": "do the thing"},
- reference={"expected": "HELD_OUT_GROUND_TRUTH"},
- )
-
- await runtime.run_tasks([task])
-
- task_json = (tmp_path / "codex" / "000000-task-1" / "task.json").read_text(encoding="utf-8")
- assert "SECRET_GRADER_INTENT" not in task_json # intent is eval-side desired-behavior metadata
- assert "HELD_OUT_GROUND_TRUTH" not in task_json # reference is grader-only ground truth
- assert '"intent"' not in task_json and '"reference"' not in task_json # dropped entirely, not just empty
- assert "do the thing" in task_json # agent-safe fields (id, inputs) are still persisted
-
-
@pytest.mark.asyncio
async def test_codex_docker_cli_agent_runtime_runs_codex_in_container_and_writes_evidence(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
@@ -204,7 +165,7 @@ def __init__(self, command: tuple[str, ...]) -> None:
self.command = command
async def communicate(self, input: bytes) -> tuple[bytes, bytes]:
- assert input == b"Question?" # prompt is the instruction verbatim
+ assert b"Task id: task/1" in input
evidence_mount = self.command[self.command.index(f"{auth_path.resolve()}:/root/.codex/auth.json:ro") + 4]
evidence_dir = Path(evidence_mount.split(":/evidence", maxsplit=1)[0])
(evidence_dir / "final_output.txt").write_text("docker codex answer", encoding="utf-8")
@@ -221,7 +182,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
auth_path=auth_path,
process_factory=fake_process_factory,
)
- task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Question?"})
+ task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"prompt": "Question?"})
trials = await runtime.run_tasks([task])
@@ -277,7 +238,7 @@ async def fake_wait_for(awaitable: Any, timeout: float) -> Any:
work_root=tmp_path / "codex",
process_factory=fake_process_factory,
)
- task = AgentEvalTask(id="task-timeout", intent="Answer.", inputs={"instruction": "Q?"})
+ task = AgentEvalTask(id="task-timeout", intent="Answer.", inputs={"prompt": "Q?"})
trials = await runtime.run_tasks([task])
@@ -305,7 +266,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
work_root=tmp_path / "codex",
process_factory=fake_process_factory,
)
- task = AgentEvalTask(id="task-2", intent="Answer.", inputs={"instruction": "Q?"})
+ task = AgentEvalTask(id="task-2", intent="Answer.", inputs={"prompt": "Q?"})
trials = await runtime.run_tasks([task])
@@ -327,9 +288,10 @@ def __init__(self, command: tuple[str, ...]) -> None:
self.command = command
async def communicate(self, input: bytes) -> tuple[bytes, bytes]:
- # Seed files are staged into the workspace before the agent runs.
+ # Seed files are staged before the agent runs and listed (by name) in the prompt.
workspace_dir = Path(self.command[self.command.index("--cd") + 1])
assert (workspace_dir / "buggy.py").read_text(encoding="utf-8") == "def add(a, b)\n return a + b\n"
+ assert b"buggy.py" in input
final_output_path = Path(self.command[self.command.index("--output-last-message") + 1])
final_output_path.write_text("fixed it", encoding="utf-8")
return b"", b""
@@ -345,7 +307,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
task = AgentEvalTask(
id="fix-bug",
intent="Fix the syntax error.",
- inputs={"instruction": "fix the bug", "files": {"buggy.py": "def add(a, b)\n return a + b\n"}},
+ inputs={"files": {"buggy.py": "def add(a, b)\n return a + b\n"}},
)
trials = await runtime.run_tasks([task])
@@ -399,9 +361,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess:
work_root=tmp_path / "codex",
process_factory=fake_process_factory,
)
- task = AgentEvalTask(
- id="probe", intent="probe", inputs={"instruction": "run", "files": {"p.txt": {"kind": "thread_probe"}}}
- )
+ task = AgentEvalTask(id="probe", intent="probe", inputs={"files": {"p.txt": {"kind": "thread_probe"}}})
trials = await runtime.run_tasks([task])
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py
index ad62495708..54d41c3431 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py
@@ -165,10 +165,13 @@ def _fake_sdk() -> SandboxSDK:
def _task(
*,
task_id: str = "task-1",
- instruction: str | None = "Instruction text.",
+ prompt: str | None = "Prompt text.",
+ instruction: str | None = None,
workspace_dir: Path | None = None,
) -> AgentEvalTask:
inputs: dict[str, Any] = {}
+ if prompt is not None:
+ inputs["prompt"] = prompt
if instruction is not None:
inputs["instruction"] = instruction
if workspace_dir is not None:
@@ -200,23 +203,22 @@ def fake_import(
docker_sandbox._load_agents_sdk()
-def test_manifest_uses_instruction_and_never_leaks_intent() -> None:
- # The instruction is surfaced verbatim; `task.intent` is eval-side metadata and must never leak to
- # the agent (reward-hacking hole), so it is never a fallback.
+@pytest.mark.parametrize(
+ ("task", "expected_prompt"),
+ [
+ (_task(prompt="Prompt text.", instruction="Instruction text."), "Prompt text."),
+ (_task(prompt=None, instruction="Instruction text."), "Instruction text."),
+ (_task(prompt=None, instruction=None), "Intent text."),
+ ],
+)
+def test_manifest_uses_prompt_instruction_intent_fallback(
+ task: AgentEvalTask,
+ expected_prompt: str,
+) -> None:
runtime = DockerSandboxAgentRuntime()
- manifest = runtime._build_manifest(_task(instruction="Instruction text."), _fake_sdk())
-
- content = manifest.entries["instruction.md"].content.decode("utf-8")
- assert content == "Instruction text."
- assert "Intent text." not in content
-
+ manifest = runtime._build_manifest(task, _fake_sdk())
-def test_manifest_raises_when_task_has_no_instruction() -> None:
- # A task with no instruction cannot be evaluated; building its manifest raises rather than
- # producing an empty prompt (and `task.intent` must never leak as a fallback).
- runtime = DockerSandboxAgentRuntime()
- with pytest.raises(ValueError, match="no instruction"):
- runtime._build_manifest(_task(instruction=None), _fake_sdk())
+ assert manifest.entries["instruction.md"].content.decode("utf-8") == expected_prompt
def test_manifest_maps_workspace_dir_to_local_dir(tmp_path: Path) -> None:
@@ -236,7 +238,7 @@ def test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields() -> None
task = AgentEvalTask(
id="task-1",
intent="Intent text.",
- inputs={"instruction": "Instruction text."},
+ inputs={"prompt": "Prompt text."},
reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"},
)
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py
index 64af6626a1..39870b6020 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py
@@ -14,7 +14,6 @@
from __future__ import annotations
-import copy
import importlib.util
import json
import os
@@ -69,45 +68,11 @@ def _task() -> AgentEvalTask:
# --- hermetic: fake nemo_fabric so CI exercises the runner+evaluator+metric+evidence chain ----------
-class _FakeEnvironment:
- def __init__(self, *, provider: str = "local", workspace: str | None = None, artifacts: str | None = None) -> None:
- self.provider = provider
- self.workspace = workspace
- self.artifacts = artifacts
-
-
-class _FakeRuntimeCfg:
- def __init__(self, artifacts: str | None = None) -> None:
- self.artifacts = artifacts
-
-
class _FakeConfig:
- """Stand-in for nemo_fabric.FabricConfig supporting the config-first helpers the runtime uses."""
-
- def __init__(self) -> None:
- self.environment: _FakeEnvironment | None = None
- self.runtime = _FakeRuntimeCfg()
- self.models: dict[str, Any] = {}
- self.relay: dict[str, Any] | None = None
-
@classmethod
def from_mapping(cls, mapping: dict[str, Any]) -> _FakeConfig:
return cls()
- def model_copy(self, *, deep: bool = False) -> _FakeConfig:
- clone = _FakeConfig()
- clone.environment = copy.deepcopy(self.environment)
- clone.runtime = _FakeRuntimeCfg(self.runtime.artifacts)
- clone.models = copy.deepcopy(self.models)
- clone.relay = copy.deepcopy(self.relay)
- return clone
-
- def enable_relay(
- self, *, project: str | None = None, output_dir: str | None = None, config: Any = None
- ) -> _FakeConfig:
- self.relay = {"project": project, "output_dir": output_dir, "config": config}
- return self
-
class _FakeProfile:
def __init__(self, **kwargs: Any) -> None:
@@ -159,20 +124,19 @@ async def test_fabric_runner_eval_exposes_trajectory_to_metric(tmp_path: Path, m
(tmp_path / "out.txt").write_text("DONE\n", encoding="utf-8")
class _FakeClient:
- # Fabric is a plain reusable facade (not an async context manager).
+ async def __aenter__(self) -> _FakeClient:
+ return self
+
+ async def __aexit__(self, *exc: Any) -> bool:
+ return False
+
async def run(self, agent: Any, **kwargs: Any) -> _FakeResult:
return _FakeResult(artifacts)
- class _FakeRunRequest:
- def __init__(self, **kwargs: Any) -> None:
- self.__dict__.update(kwargs)
-
module = types.ModuleType("nemo_fabric")
- module.Fabric = _FakeClient # type: ignore[attr-defined]
+ module.FabricClient = _FakeClient # type: ignore[attr-defined]
module.FabricConfig = _FakeConfig # type: ignore[attr-defined]
module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined]
- module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined]
- module.RunRequest = _FakeRunRequest # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "nemo_fabric", module)
# Trajectory capture builds the profile from nemo_relay's typed config objects (lazy import); stub
diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py
index 5805cc9425..cc5d97b759 100644
--- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py
+++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py
@@ -5,11 +5,9 @@
from __future__ import annotations
-import copy
import json
import sys
import types
-from collections.abc import Iterator, Mapping
from pathlib import Path
from typing import Any
@@ -19,48 +17,14 @@
from nemo_evaluator_sdk.values.evidence import EVIDENCE_FORMAT_ATIF, EVIDENCE_TRACE
-class _FakeEnvironment:
- """Stand-in for nemo_fabric.EnvironmentConfig (the runtime sets workspace/provider/artifacts)."""
-
- def __init__(self, *, provider: str = "local", workspace: str | None = None, artifacts: str | None = None) -> None:
- self.provider = provider
- self.workspace = workspace
- self.artifacts = artifacts
-
-
-class _FakeRuntimeCfg:
- def __init__(self, artifacts: str | None = None) -> None:
- self.artifacts = artifacts
-
-
class _FakeConfig:
- """Stand-in for nemo_fabric.FabricConfig with the config-first helpers the runtime composes onto."""
-
def __init__(self, mapping: dict[str, Any]) -> None:
self.mapping = mapping
- self.environment: _FakeEnvironment | None = None
- self.runtime = _FakeRuntimeCfg()
- self.models: dict[str, Any] = dict(mapping.get("models", {}))
- self.relay: dict[str, Any] | None = None # records enable_relay(...)
@classmethod
def from_mapping(cls, mapping: dict[str, Any]) -> _FakeConfig:
return cls(mapping)
- def model_copy(self, *, deep: bool = False) -> _FakeConfig:
- clone = _FakeConfig(self.mapping)
- clone.environment = copy.deepcopy(self.environment)
- clone.runtime = _FakeRuntimeCfg(self.runtime.artifacts)
- clone.models = copy.deepcopy(self.models)
- clone.relay = copy.deepcopy(self.relay)
- return clone
-
- def enable_relay(
- self, *, project: str | None = None, output_dir: str | None = None, config: Any = None
- ) -> _FakeConfig:
- self.relay = {"project": project, "output_dir": output_dir, "config": config}
- return self
-
class _FakeProfile:
def __init__(self, *, name: str | None = None, models: Any = None, mapping: Any = None) -> None:
@@ -73,14 +37,6 @@ def from_mapping(cls, mapping: dict[str, Any]) -> _FakeProfile:
return cls(name=mapping.get("name"), mapping=mapping)
-class _FakeRunRequest:
- """Stand-in for nemo_fabric.RunRequest (Fabric.run folds input + request id into it)."""
-
- def __init__(self, *, input: Any = None, request_id: str | None = None) -> None:
- self.input = input
- self.request_id = request_id
-
-
class _FakeRelayConfig:
"""Stand-in for nemo_relay.observability's typed config objects (AtifConfig/AtofConfig/...)."""
@@ -163,20 +119,23 @@ def _install_fake_fabric(monkeypatch: pytest.MonkeyPatch, handler: Any) -> type:
"""Inject a fake ``nemo_fabric`` module (the runtime imports it lazily); return the client class."""
class _FakeClient:
- # Fabric is a plain reusable facade (not an async context manager).
recorded: list[dict[str, Any]] = []
+ async def __aenter__(self) -> _FakeClient:
+ return self
+
+ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
+ return False
+
async def run(self, agent: Any, **kwargs: Any) -> Any:
_FakeClient.recorded.append({"agent": agent, **kwargs})
return handler(agent, kwargs)
_FakeClient.recorded = []
module = types.ModuleType("nemo_fabric")
- module.Fabric = _FakeClient # type: ignore[attr-defined]
+ module.FabricClient = _FakeClient # type: ignore[attr-defined]
module.FabricConfig = _FakeConfig # type: ignore[attr-defined]
module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined]
- module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined]
- module.RunRequest = _FakeRunRequest # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, "nemo_fabric", module)
# The runtime builds the trajectory profile from nemo_relay's typed config objects (lazy import);
@@ -193,7 +152,7 @@ async def run(self, agent: Any, **kwargs: Any) -> Any:
return _FakeClient
-_TASK = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Ping?"})
+_TASK = AgentEvalTask(id="task/1", intent="Answer.", inputs={"prompt": "Ping?"})
_CONFIG = {"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}}
@@ -231,44 +190,19 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
assert trial.evidence.descriptors["stdout"].ref == str(tmp_path / "stdout.txt")
result_file = tmp_path / "fabric" / "000000-task-1" / "fabric_result.json"
assert json.loads(result_file.read_text(encoding="utf-8"))["status"] == "succeeded"
- # Config-first: the model is set on the config's default model and relay (ATIF trajectory) is
- # enabled on the config, rather than layered as profile overlays.
- composed = client_cls.recorded[0]["agent"]
- assert composed.models["default"] == {"provider": "openai", "model": "openai/gpt-5.4"}
- assert composed.relay is not None # capture_trajectory defaults on -> enable_relay(...) called
- assert client_cls.recorded[0]["request"].request_id == "task/1"
+ # Model applied as a profile overlay (Harbor pattern) + the Relay ATIF trajectory overlay.
+ profiles = client_cls.recorded[0]["profiles"]
+ names = [p.name for p in profiles]
+ assert "eval_model" in names
+ assert "eval_trajectory" in names # capture_trajectory defaults on
+ model_profile = next(p for p in profiles if p.name == "eval_model")
+ assert model_profile.models == {"default": {"provider": "openai", "model": "openai/gpt-5.4"}}
+ assert client_cls.recorded[0]["request_id"] == "task/1"
# Telemetry reference is preserved end-to-end (uri + trace_id), not just provider/kind.
assert trial.evidence.metadata["telemetry"][0]["uri"] == "file:///relay"
assert trial.evidence.metadata["telemetry"][0]["trace_id"] == "tid-1"
-@pytest.mark.asyncio
-async def test_fabric_runtime_prompt_excludes_intent_and_frames_inputs(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- # The prompt is exactly the task instruction — no runtime framing. `task.intent` is eval-side
- # "desired behavior" metadata and must never reach the agent (reward-hacking hole); other inputs
- # keys are not templated into the prompt either.
- task = AgentEvalTask(
- id="task/2",
- intent="SECRET_GRADER_INTENT",
- inputs={"instruction": "Ping?", "files": {"data.csv": "s3://seed"}, "hint": "be terse"},
- )
-
- def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
- return _FakeResult(status="succeeded", output="ok")
-
- client_cls = _install_fake_fabric(monkeypatch, handler)
- runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric")
-
- await runtime.run_tasks([task])
-
- prompt = client_cls.recorded[0]["request"].input
- assert prompt == "Ping?" # the instruction verbatim, nothing else
- assert "SECRET_GRADER_INTENT" not in prompt # intent stays eval-side
- assert "be terse" not in prompt # non-instruction inputs are not templated in
-
-
@pytest.mark.asyncio
async def test_fabric_runtime_maps_atif_artifact_to_trace_evidence(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
@@ -292,60 +226,10 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
assert trace.ref == str(tmp_path / "trajectory.atif.json")
-def _workspace_from_config(config: Any) -> Path:
- """Pull the staged workspace path out of the composed per-task config."""
- return Path(config.environment.workspace)
-
-
-def _resolve_like_fabric(config: Any, profiles: list[Any], section: str, key: str) -> Any:
- """Mirror Fabric's resolver: start from the config, then apply each profile as a winning overlay in
- order (last wins). Used to assert what value actually reaches the harness for a config/profile key.
- """
- if section == "environment":
- value = getattr(config.environment, key, None) if config.environment is not None else None
- elif section == "models":
- value = config.models.get(key)
- else: # pragma: no cover - only the two sections above are exercised
- raise ValueError(section)
- for profile in profiles:
- overlay = getattr(profile, "mapping", None)
- if isinstance(overlay, Mapping) and isinstance(overlay.get(section), Mapping):
- if overlay[section].get(key) is not None:
- value = overlay[section][key]
- return value
-
-
-@pytest.mark.asyncio
-async def test_caller_profiles_cannot_override_evaluator_owned_settings(
- tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-) -> None:
- # Fabric applies caller-supplied profiles over the config (last-wins), so the evaluator's per-task
- # workspace (isolation + `workspace` evidence integrity) and model-under-eval must remain the final,
- # authoritative layer. A caller profile that sets these must NOT win.
- caller_profile = {
- "name": "caller",
- "environment": {"workspace": "/caller/hijacked-workspace"},
- "models": {"default": {"provider": "openai", "model": "caller/rogue-model"}},
- }
-
- def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
- return _FakeResult(status="succeeded", output="ok")
-
- client_cls = _install_fake_fabric(monkeypatch, handler)
- runtime = fabric_runtime.FabricAgentRuntime(
- config=_CONFIG, model="openai/gpt-5.4", work_root=tmp_path / "fabric", profiles=[caller_profile]
- )
-
- await runtime.run_tasks([_TASK])
-
- config = client_cls.recorded[0]["agent"]
- profiles = client_cls.recorded[0]["profiles"]
- eval_workspace = config.environment.workspace # the per-task dir the evaluator composed
- eval_model = config.models["default"]
-
- # After Fabric applies the caller profile, the evaluator's workspace + model must still win.
- assert _resolve_like_fabric(config, profiles, "environment", "workspace") == eval_workspace
- assert _resolve_like_fabric(config, profiles, "models", "default") == eval_model
+def _workspace_from_profiles(profiles: list[Any]) -> Path:
+ """Pull the staged workspace path out of the ``eval_workspace`` profile overlay."""
+ profile = next(p for p in profiles if p.name == fabric_runtime._WORKSPACE_PROFILE_NAME)
+ return Path(profile.mapping["environment"]["workspace"])
@pytest.mark.asyncio
@@ -360,7 +244,7 @@ async def test_fabric_runtime_seeds_workspace_and_exposes_workspace_evidence(
def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
# The harness runs in the staged workspace; simulate an edit it leaves behind.
- workspace = _workspace_from_config(agent)
+ workspace = _workspace_from_profiles(kwargs["profiles"])
(workspace / "result.txt").write_text("done", encoding="utf-8")
return _FakeResult(status="succeeded", output={"response": "ok"})
@@ -371,10 +255,10 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
trial = trials[0]
assert trial.status == "completed"
- # The composed config carries environment.workspace (the harness's cwd) with provider=local.
- composed = client_cls.recorded[0]["agent"]
- assert composed.environment.provider == "local"
- workspace = _workspace_from_config(composed)
+ # A dedicated workspace profile overlay carries environment.workspace (the harness's cwd).
+ profiles = client_cls.recorded[0]["profiles"]
+ assert fabric_runtime._WORKSPACE_PROFILE_NAME in [p.name for p in profiles]
+ workspace = _workspace_from_profiles(profiles)
# The seed file is staged and the agent's edit is present in the same dir.
assert (workspace / "calc.py").read_text(encoding="utf-8") == "value = 1\n"
assert (workspace / "result.txt").read_text(encoding="utf-8") == "done"
@@ -383,8 +267,9 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
workspace_evidence = trial.evidence.descriptors["workspace"]
assert workspace_evidence.kind == "filesystem"
assert workspace_evidence.ref == str(workspace)
- # Seed-file contents are not inlined into the prompt (they are already on disk in the workspace).
- harness_input = client_cls.recorded[0]["request"].input
+ # Seed-file contents are listed by name in the input, not dumped inline (they are already on disk).
+ harness_input = client_cls.recorded[0]["input"]
+ assert "calc.py" in harness_input
assert "value = 1" not in harness_input
@@ -402,7 +287,9 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
trials = await runtime.run_tasks([_TASK]) # _TASK has no 'files' input
- workspace = _workspace_from_config(client_cls.recorded[0]["agent"])
+ profiles = client_cls.recorded[0]["profiles"]
+ assert fabric_runtime._WORKSPACE_PROFILE_NAME in [p.name for p in profiles]
+ workspace = _workspace_from_profiles(profiles)
assert workspace.is_dir()
assert trials[0].evidence is not None
assert trials[0].evidence.descriptors["workspace"].ref == str(workspace)
@@ -430,7 +317,7 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
@pytest.mark.asyncio
-async def test_fabric_runtime_capture_trajectory_false_skips_relay(
+async def test_fabric_runtime_capture_trajectory_false_skips_relay_overlay(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
@@ -441,7 +328,8 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
await runtime.run_tasks([_TASK])
- assert client_cls.recorded[0]["agent"].relay is None
+ names = [p.name for p in client_cls.recorded[0]["profiles"]]
+ assert "eval_trajectory" not in names
@pytest.mark.asyncio
@@ -552,34 +440,3 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
assert result.outputs[0].name == "agent_phase_success"
assert result.outputs[0].value is True
-
-
-@pytest.mark.asyncio
-async def test_fabric_runtime_normalizes_runoutput_response(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
- # Newer Fabric wraps RunResult.output in a RunOutput Mapping (not a plain JSON value); the runtime
- # must copy it into a plain dict so it round-trips through the trial's JsonValue response field.
- class _FakeRunOutput(Mapping):
- def __init__(self, data: dict[str, Any]) -> None:
- self._data = dict(data)
-
- def __getitem__(self, key: str) -> Any:
- return self._data[key]
-
- def __iter__(self) -> Iterator[str]:
- return iter(self._data)
-
- def __len__(self) -> int:
- return len(self._data)
-
- def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult:
- return _FakeResult(status="succeeded", output=_FakeRunOutput({"response": "PONG", "returncode": 0}))
-
- _install_fake_fabric(monkeypatch, handler)
- runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric")
-
- trial = (await runtime.run_tasks([_TASK]))[0]
-
- assert trial.status == "completed"
- assert trial.output is not None
- assert trial.output.output_text == "PONG" # extracted from the normalized mapping
- assert trial.output.response == {"response": "PONG", "returncode": 0} # plain dict, not RunOutput
diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml
index d7bb978d9f..1147a71a50 100644
--- a/packages/nemo_platform/pyproject.toml
+++ b/packages/nemo_platform/pyproject.toml
@@ -351,7 +351,7 @@ nemo-safe-synthesizer-plugin = [
"gunicorn>=23.0.0",
"httpx>=0.27.2",
"nemo-platform-plugin",
- "nemo-safe-synthesizer==0.1.7",
+ "nemo-safe-synthesizer==0.1.2",
"pydantic[email]>=2.9.2",
"pydantic-settings>=2.2.1",
"python-multipart~=0.0.9",
diff --git a/plugins/nemo-safe-synthesizer/constraints.txt b/plugins/nemo-safe-synthesizer/constraints.txt
index ae8172735b..6f487f70d6 100644
--- a/plugins/nemo-safe-synthesizer/constraints.txt
+++ b/plugins/nemo-safe-synthesizer/constraints.txt
@@ -1,17 +1,17 @@
-# Security floor constraints -- generated from [tool.uv] constraint-dependencies in pyproject.toml
-# Vendored from https://github.com/NVIDIA-NeMo/Safe-Synthesizer/blob/v0.1.7/constraints.txt
-# on 2026-07-10. Local additions keep pyarrow/cryptography on CVE-fixed
-# versions and the AWS SDK packages aligned with NeMo Platform's
-# aiobotocore-compatible range.
+# Vendored from https://github.com/NVIDIA-NeMo/Safe-Synthesizer/blob/v0.1.2/constraints.txt
+# on 2026-06-30. Local additions keep the AWS SDK packages aligned with
+# NeMo Platform's aiobotocore-compatible range.
#
# Pass to pip/uv with: pip install -c constraints.txt
# uv pip install -c constraints.txt
+
+# Security floor constraints -- generated from [tool.uv] constraint-dependencies in pyproject.toml
GitPython>=3.1.50
-PyJWT>=2.13.0
+PyJWT>=2.12.0
Pygments>=2.20.0
-aiohttp>=3.14.0
+aiohttp>=3.13.4
cbor2>=5.9.0
-cryptography>=48.0.1,<49
+cryptography>=46.0.7
grpcio>=1.80.0
idna>=3.15
jsonpath-ng>=1.8.0
@@ -21,14 +21,11 @@ mistune>=3.2.1
nbconvert>=7.17.1
pandas<3
pillow>=12.2.0
-pyarrow>=23.0.1,<24
pymdown-extensions>=10.21.3
python-dotenv>=1.2.2
python-multipart>=0.0.27
requests>=2.33.0
-starlette>=1.0.1
tornado>=6.5.5
-urllib3>=2.7.0
# Runtime compatibility constraints.
boto3>=1.40.46,<1.40.62
diff --git a/plugins/nemo-safe-synthesizer/openapi/openapi.yaml b/plugins/nemo-safe-synthesizer/openapi/openapi.yaml
index a2624974c2..0cd08d3d7a 100644
--- a/plugins/nemo-safe-synthesizer/openapi/openapi.yaml
+++ b/plugins/nemo-safe-synthesizer/openapi/openapi.yaml
@@ -821,11 +821,45 @@ components:
description: The fraction of invalid records that will stop generation after
the ``patience`` limit is reached. Must be in [0, 1].
default: 0.8
- structured_generation:
- allOf:
- - $ref: '#/components/schemas/StructuredGenerationParameters'
- description: Structured generation parameters controlling schema-constrained
- output.
+ use_structured_generation:
+ type: boolean
+ title: use_structured_generation
+ description: Whether to use structured generation for better format control.
+ default: false
+ structured_generation_backend:
+ type: string
+ enum:
+ - auto
+ - xgrammar
+ - guidance
+ - outlines
+ - lm-format-enforcer
+ title: structured_generation_backend
+ description: 'The backend used by vLLM when ``use_structured_generation``
+ is ``True``. Supported backends: ''outlines'', ''guidance'', ''xgrammar'',
+ ''lm-format-enforcer''. ''auto'' will allow vLLM to choose the backend.'
+ default: auto
+ structured_generation_schema_method:
+ type: string
+ enum:
+ - auto
+ - regex
+ - json_schema
+ - structural_tag
+ title: structured_generation_schema_method
+ description: The method used to generate the schema from your dataset and
+ pass it to the generation backend. 'auto' picks 'structural_tag' on xgrammar-capable
+ backends and 'regex' otherwise. 'regex' uses a custom regex construction
+ method that tends to be more comprehensive than 'json_schema' at the cost
+ of speed. 'structural_tag' uses XGrammar Structural Tag to compose schema-constrained
+ JSONL output.
+ default: auto
+ structured_generation_use_single_sequence:
+ type: boolean
+ title: structured_generation_use_single_sequence
+ description: Whether to use a regex that matches exactly one sequence or
+ record if ``max_sequences_per_example`` is 1.
+ default: false
enforce_timeseries_fidelity:
type: boolean
title: enforce_timeseries_fidelity
@@ -1788,57 +1822,6 @@ components:
type: array
title: StringFilter
type: object
- StructuredGenerationParameters:
- properties:
- enabled:
- type: boolean
- title: enabled
- description: Whether to use structured generation for better format control.
- default: false
- backend:
- type: string
- enum:
- - auto
- - xgrammar
- - guidance
- - outlines
- - lm-format-enforcer
- title: backend
- description: 'The backend used by vLLM when structured generation is enabled.
- Supported backends: ''outlines'', ''guidance'', ''xgrammar'', ''lm-format-enforcer''.
- ''auto'' will allow vLLM to choose the backend.'
- default: auto
- schema_method:
- type: string
- enum:
- - auto
- - regex
- - json_schema
- - structural_tag
- title: schema_method
- description: The method used to generate the schema from your dataset and
- pass it to the generation backend. 'auto' picks 'structural_tag' on xgrammar-capable
- backends and 'regex' otherwise. 'regex' uses a custom regex construction
- method that tends to be more comprehensive than 'json_schema' at the cost
- of speed. 'structural_tag' uses XGrammar Structural Tag to compose schema-constrained
- JSONL output.
- default: auto
- use_single_sequence:
- type: boolean
- title: use_single_sequence
- description: Whether to use a regex that matches exactly one sequence or
- record if ``max_sequences_per_example`` is 1.
- default: false
- type: object
- title: StructuredGenerationParameters
- description: 'Configuration for vLLM structured generation.
-
-
- These parameters control whether generation is constrained to schema-shaped
-
- output, which backend enforces the constraint, and how the constraint schema
-
- is built.'
TimeSeriesParameters:
properties:
is_timeseries:
@@ -1856,8 +1839,8 @@ components:
type: string
timestamp_interval_seconds:
title: Timestamp Interval Seconds
- description: Positive whole-number interval in seconds between timestamps.
- If not provided, the timestamp column will be used to infer the interval.
+ description: Interval in seconds between timestamps. If not provided, the
+ timestamp column will be used to infer the interval.
type: integer
timestamp_format:
title: Timestamp Format
diff --git a/plugins/nemo-safe-synthesizer/pyproject.toml b/plugins/nemo-safe-synthesizer/pyproject.toml
index 10a3806c8d..2d0e3eba57 100644
--- a/plugins/nemo-safe-synthesizer/pyproject.toml
+++ b/plugins/nemo-safe-synthesizer/pyproject.toml
@@ -12,7 +12,7 @@ dependencies = [
"httpx>=0.27.2",
"nemo-platform",
"nemo-platform-plugin",
- "nemo-safe-synthesizer==0.1.7",
+ "nemo-safe-synthesizer==0.1.2",
"pydantic[email]>=2.9.2",
"pydantic-settings>=2.2.1",
"python-multipart~=0.0.9",
diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py
index 3bfba8263c..1c69204ea4 100644
--- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py
+++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py
@@ -31,7 +31,7 @@ class SafeSynthesizerConfig(NemoConfig):
),
)
runtime_venv: str = ".nemo/safe-synthesizer-runtime"
- runtime_package: str = "nemo-safe-synthesizer[engine,cu129]==0.1.7"
+ runtime_package: str = "nemo-safe-synthesizer[engine,cu129]==0.1.2"
runtime_python_version: str = "3.11"
runtime_python: str | None = None
default_job_resource_memory_request: str = "16G"
diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py
index 82b5a893f3..5f1183ec81 100644
--- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py
+++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py
@@ -5,6 +5,7 @@
from __future__ import annotations
+import platform
import shutil
import subprocess
from pathlib import Path
@@ -22,7 +23,21 @@
RUNTIME_CONSTRAINTS_FILE = Path("plugins/nemo-safe-synthesizer/constraints.txt")
FLASHINFER_CU129_INDEX_URL = "https://flashinfer.ai/whl/cu129"
PYTORCH_CU129_INDEX_URL = "https://download.pytorch.org/whl/cu129"
-VLLM_CU129_INDEX_URL = "https://wheels.vllm.ai/ee0da84ab9e04ac7610e28580af62c365e898389/cu129"
+VLLM_CU129_VERSION = "0.20.0"
+
+
+def vllm_cu129_wheel() -> str:
+ """Return the direct vLLM CUDA 12.9 wheel URL for the current host."""
+ machine = platform.machine().lower()
+ if machine in {"amd64", "x86_64"}:
+ arch = "x86_64"
+ elif machine in {"aarch64", "arm64"}:
+ arch = "aarch64"
+ else:
+ raise RuntimeError(f"Unsupported architecture for vLLM CUDA 12.9 wheel: {platform.machine()}")
+
+ wheel = f"vllm-{VLLM_CU129_VERSION}%2Bcu129-cp38-abi3-manylinux_2_31_{arch}.whl"
+ return f"vllm @ https://github.com/vllm-project/vllm/releases/download/v{VLLM_CU129_VERSION}/{wheel}"
def runtime_package_index_options(runtime_package: str) -> list[str]:
@@ -34,8 +49,6 @@ def runtime_package_index_options(runtime_package: str) -> list[str]:
FLASHINFER_CU129_INDEX_URL,
"--extra-index-url",
PYTORCH_CU129_INDEX_URL,
- "--extra-index-url",
- VLLM_CU129_INDEX_URL,
]
@@ -43,9 +56,7 @@ def runtime_package_extra_requirements(runtime_package: str) -> list[str]:
"""Return direct requirements needed by the selected runtime package."""
if "cu129" not in runtime_package:
return []
- # Safe Synthesizer 0.1.7 declares its cu129 vLLM dependency directly; the
- # runtime only needs to add the vLLM wheel index above.
- return []
+ return [vllm_cu129_wheel()]
def repo_root() -> Path:
diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py b/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py
index 95f6cb68ef..6c40851e56 100644
--- a/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py
+++ b/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py
@@ -51,10 +51,8 @@ def test_cuda_runtime_package_adds_cu129_sources():
runtime.FLASHINFER_CU129_INDEX_URL,
"--extra-index-url",
runtime.PYTORCH_CU129_INDEX_URL,
- "--extra-index-url",
- runtime.VLLM_CU129_INDEX_URL,
]
- assert runtime.runtime_package_extra_requirements(runtime_package) == []
+ assert runtime.runtime_package_extra_requirements(runtime_package) == [runtime.vllm_cu129_wheel()]
def test_non_cu129_runtime_package_does_not_add_cu129_sources():
@@ -92,7 +90,7 @@ def fake_run(command, **kwargs):
assert "--extra-index-url" in calls[2][0]
assert runtime.FLASHINFER_CU129_INDEX_URL in calls[2][0]
assert runtime.PYTORCH_CU129_INDEX_URL in calls[2][0]
- assert runtime.VLLM_CU129_INDEX_URL in calls[2][0]
+ assert runtime.vllm_cu129_wheel() in calls[2][0]
assert str(tmp_path / "plugins/nemo-safe-synthesizer") not in calls[2][0]
assert "nemo-safe-synthesizer[engine,cu129]" in calls[2][0]
assert calls[3][0] == [
diff --git a/pytest.ini b/pytest.ini
index bf7f15fbf4..cc60fa0df8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -66,7 +66,6 @@ markers =
e2e_config(*layers, harness=...): Ordered list of repo-root-relative config paths and/or inline dict overlays; harness config stays separate from platform config
subprocess_only: Test only works in subprocess mode (not on Kubernetes); skipped when NMP_BASE_URL is set
container_only: Test requires a container backend (Docker or Kubernetes); skipped unless NMP_BASE_URL is set
- requires_gpu: Container e2e requiring GPU job scheduling; skipped unless --feature gpu is passed
regression: Regression tests - test individual functional microservices for baseline functionality
infrastructure: Infrastructure tests - ensure services are compatible with customer infrastructure
canary: Canary tests - test deployed integration environments like top of tree
diff --git a/release/assets.yaml b/release/assets.yaml
new file mode 100644
index 0000000000..ec1f8adbc0
--- /dev/null
+++ b/release/assets.yaml
@@ -0,0 +1,15 @@
+sdk:
+ - id: nemo-platform
+ - id: nemo-platform-plugin
+
+# Container images eligible for release publishing. This list is the single
+# source of truth: the release-consumer repository reads it at the release ref
+# to decide which images to stage. Each id must match an image name pushed to
+# the dev registry tagged by this repository's commit SHA, and should have a
+# catalog metadata entry on the consumer side.
+container:
+ - id: nmp-automodel-tasks
+ - id: nmp-automodel-training
+ - id: nmp-unsloth-training
+ - id: auditor-tasks
+ - id: safe-synthesizer-tasks
diff --git a/script/dev-install-fabric.sh b/script/dev-install-fabric.sh
index 6c966fc324..cb20509872 100755
--- a/script/dev-install-fabric.sh
+++ b/script/dev-install-fabric.sh
@@ -53,7 +53,7 @@ if [ ! -d "$FABRIC_REPO" ]; then
fi
echo "Building + installing nemo-fabric[codex,relay] from $FABRIC_REPO into $VENV_PY ..."
uv pip install --python "$VENV_PY" "${FABRIC_REPO}[codex,relay]"
-"$VENV_PY" -c "import nemo_fabric; from nemo_fabric import Fabric, RunResult; print('nemo_fabric OK:', nemo_fabric.__file__)"
+"$VENV_PY" -c "import nemo_fabric; from nemo_fabric import FabricClient, RunResult; print('nemo_fabric OK:', nemo_fabric.__file__)"
# 2. nemo-relay gateway binary (codex -> OTLP -> gateway -> trajectory-*.atif.json). Required for
# trajectory capture; the pip `nemo-relay` package does NOT ship this executable.
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py
index c86b06fdb9..adda911e12 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py
@@ -628,12 +628,10 @@ def _default_prompt_template(target: Model | Agent) -> dict[str, Any]:
def _task_row(task: AgentEvalTask) -> dict[str, Any]:
- # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}`
- # template): a dataset `prompt` column or the agent `instruction`.
return {
**task.inputs,
"task_id": task.id,
- "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"),
+ "prompt": task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent,
}
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py
index 9a904d8952..1443f7146a 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py
@@ -68,7 +68,7 @@ def __init__(
self._work_root = Path(work_root).expanduser() if work_root is not None else None
self._codex_bin = codex_bin
self._timeout_s = timeout_s
- self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt
+ self._prompt_builder = prompt_builder or default_codex_prompt
self._process_factory = process_factory or asyncio.create_subprocess_exec
self._runtime_name = runtime_name
@@ -95,17 +95,15 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC
evidence_dir.mkdir(parents=True, exist_ok=True)
workspace_dir.mkdir(parents=True, exist_ok=True)
+ prompt = self._prompt_builder(task)
prompt_path = evidence_dir / "prompt.txt"
task_path = evidence_dir / "task.json"
stdout_path = evidence_dir / "stdout.jsonl"
stderr_path = evidence_dir / "stderr.txt"
final_output_path = evidence_dir / "final_output.txt"
- # Persist the task for debugging, but never the grader-only fields: the docker variant mounts
- # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired
- # behavior) or `reference` (held-out ground truth) here would let the agent read them back out
- # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes.
- task_path.write_text(task.model_dump_json(indent=2, exclude={"intent", "reference"}), encoding="utf-8")
+ prompt_path.write_text(prompt, encoding="utf-8")
+ task_path.write_text(task.model_dump_json(indent=2), encoding="utf-8")
command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path)
process: Any | None = None
@@ -115,10 +113,6 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC
# synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this
# runs on the event loop shared by every concurrent task, so a blocking seed would stall them all.
seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY))
- # Build the prompt after seeding and inside the guarded block: an instruction-less task
- # raises here, failing just this task instead of aborting the run (and seeding wins if both).
- prompt = self._prompt_builder(task)
- prompt_path.write_text(prompt, encoding="utf-8")
process = await self._process_factory(
*command,
stdin=subprocess.PIPE,
@@ -392,6 +386,30 @@ def print_codex_agent_models(*, codex_bin: str = "codex") -> None:
print(slug)
+def default_codex_prompt(task: AgentEvalTask) -> str:
+ """Frame a task for Codex as an agent that works in its current directory.
+
+ Task-agnostic: it states the intent and inputs and invites the agent to read/create/edit files,
+ rather than constraining the answer to a single text reply. Seed files (``inputs[SEED_FILES_INPUT_KEY]``)
+ are listed by name instead of dumped inline — the agent finds them already in its workspace. Pass a
+ custom :data:`CodexPromptBuilder` to the runtime to override this framing for a specific benchmark.
+ """
+ body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY}
+ lines = [f"Task id: {task.id}", f"Intent: {task.intent}"]
+ if body_inputs:
+ lines += ["", "Inputs:", json.dumps(body_inputs, indent=2, default=str)]
+ seeded = task.inputs.get(SEED_FILES_INPUT_KEY)
+ if isinstance(seeded, Mapping) and seeded:
+ lines += ["", "These files are already in your working directory:"]
+ lines += [f" - {path}" for path in seeded]
+ lines += [
+ "",
+ "Complete the task by working in your current directory. You may read, create, and edit files "
+ "as needed. When you are done, briefly summarize what you changed.",
+ ]
+ return "\n".join(lines) + "\n"
+
+
def _failed_codex_trial(
task: AgentEvalTask,
evidence_dir: Path,
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py
index 2fcdff1982..be39b8e1f8 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py
@@ -136,15 +136,13 @@ async def _run_task(
) -> AgentEvalTrial:
evidence_dir = self._evidence_dir(index, task, config)
evidence_dir.mkdir(parents=True, exist_ok=True)
+ prompt = _task_prompt(task)
+ manifest = self._build_manifest(task, sdk)
+ agent = self._build_agent(manifest, sdk)
client = self._build_client(sdk)
sandbox = None
try:
- # Build the prompt inside the guarded block: an instruction-less task raises here and fails
- # just this task rather than aborting the whole run.
- prompt = task.agent_prompt()
- manifest = self._build_manifest(task, sdk)
- agent = self._build_agent(manifest, sdk)
sandbox = await client.create(
manifest=manifest,
options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE),
@@ -165,7 +163,7 @@ def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any:
# workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose
# grader-only fields (e.g. ``reference`` held-out ground truth) to the agent.
entries: dict[str, Any] = {
- "instruction.md": sdk.File(content=task.agent_prompt().encode("utf-8")),
+ "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")),
"output": sdk.Dir(),
}
workspace_dir = task.inputs.get("workspace_dir")
@@ -297,6 +295,10 @@ def _validated_workspace_dir(workspace_dir: Any) -> Path:
return resolved
+def _task_prompt(task: AgentEvalTask) -> str:
+ return str(task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent)
+
+
async def _maybe_await(value: Awaitable[Any] | Any) -> Any:
if inspect.isawaitable(value):
return await value
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py
index 7e7f02468c..c0cd728cfd 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py
@@ -7,11 +7,7 @@
NeMo Fabric Python SDK and adapts each normalized Fabric ``RunResult`` into an
:class:`AgentEvalTrial`. The harness is chosen by the supplied Fabric config's
``harness.adapter_id`` (never inferred from a model); an optional ``model`` slug
-is applied as the config's default model, mirroring Fabric's own Harbor integration.
-
-Per-task settings (workspace, model, trajectory capture) are composed directly onto
-a copy of the supplied config via the SDK's config helpers (``model_copy`` +
-``enable_relay`` + ``environment``), rather than layered as profile overlays.
+is applied as a final profile overlay, mirroring Fabric's own Harbor integration.
Every task runs in its own fresh workspace: the runtime seeds it from
``inputs['files']`` (a no-op when there are none), runs the harness in it (via
@@ -42,7 +38,6 @@
CandidateEvidence,
EvidenceDescriptor,
)
-from pydantic import JsonValue
if TYPE_CHECKING:
# Annotations use nemo_fabric's real types (single source of truth). nemo_fabric is an optional
@@ -50,10 +45,9 @@
# loaded lazily at runtime (see ``run_tasks``). Drop the ty:ignore once nemo-fabric is a
# resolvable dependency and the checker can see it.
from nemo_fabric import ( # ty: ignore[unresolved-import]
- Fabric,
+ FabricClient,
FabricConfig,
FabricProfileConfig,
- RunOutput,
RunResult,
)
@@ -72,18 +66,18 @@
# Per-task workspace: where seed files are staged and where the harness reads/writes. We
# create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence.
_WORKSPACE_SUBDIR = "workspace"
+_WORKSPACE_PROFILE_NAME = "eval_workspace"
# Evidence key + descriptor kind for the staged workspace, consumed by the
# workspace-reading metrics.
_WORKSPACE_EVIDENCE_KEY = "workspace"
_WORKSPACE_EVIDENCE_KIND = "filesystem"
-# File-exporter output names we choose for the Relay ATIF/ATOF trajectory (Relay accepts these as inputs).
+# Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as inputs).
+_TRAJECTORY_PROFILE_NAME = "eval_trajectory"
_ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json"
_ATOF_FILENAME = "events.atof.jsonl"
-# Names for the trailing overlays that re-assert the evaluator-owned per-task settings (see
-# ``_eval_lock_profiles``): Fabric applies caller profiles over the config, so these must trail them.
-_WORKSPACE_PROFILE_NAME = "eval_workspace"
-_MODEL_PROFILE_NAME = "eval_model"
-_ARTIFACTS_PROFILE_NAME = "eval_artifacts"
+# Fabric telemetry-profile selectors (file exporter, no OTLP endpoint).
+_TELEMETRY_PROVIDER = "relay"
+_TELEMETRY_MODE = "sdk"
# ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence.
_ATIF_ARTIFACT_KIND = "atif"
@@ -127,77 +121,69 @@ async def run_tasks(
try:
# nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported
# lazily here rather than at module load.
- from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import]
+ from nemo_fabric import FabricClient, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import]
except ImportError as exc:
raise RuntimeError(_MISSING_FABRIC_MSG) from exc
resolved_config = config or AgentEvalRunConfig()
agent_config = FabricConfig.from_mapping(self._config)
- # Fail fast (once) if trajectory capture is requested but the nemo-relay gateway isn't
- # importable, rather than failing every task the same way inside the per-task guard.
- if self._capture_trajectory:
- try:
- import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import]
- except ImportError as exc:
- raise RuntimeError(_MISSING_RELAY_MSG) from exc
- # Caller-supplied profile overlays pass through as-is; this runtime's per-task workspace, model,
- # and trajectory settings are composed directly onto a copy of the config (config-first), not
- # layered as profiles.
- base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles]
+ base_profiles = self._build_profiles(FabricProfileConfig)
semaphore = asyncio.Semaphore(resolved_config.parallelism)
- # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle
- # context manager — so it is created once and reused across tasks with no cleanup.
- client = Fabric()
+ async with FabricClient() as client:
- async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial:
- async with semaphore:
- return await self._run_task(client, agent_config, base_profiles, index, task, resolved_config)
+ async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial:
+ async with semaphore:
+ return await self._run_task(
+ client, agent_config, base_profiles, FabricProfileConfig, index, task, resolved_config
+ )
- return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks)))
+ return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks)))
async def _run_task(
self,
- client: Fabric,
+ client: FabricClient,
agent_config: FabricConfig,
base_profiles: list[FabricProfileConfig],
+ profile_cls: type[FabricProfileConfig],
index: int,
task: AgentEvalTask,
config: AgentEvalRunConfig,
) -> AgentEvalTrial:
- # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules
- # lookup, not a re-load, so the type is used where it's constructed instead of threaded down.
- from nemo_fabric import FabricProfileConfig, RunRequest # ty: ignore[unresolved-import]
-
evidence_dir = self._evidence_dir(index, task, config)
evidence_dir.mkdir(parents=True, exist_ok=True)
+ profiles = list(base_profiles)
+ if self._capture_trajectory:
+ # Enable Relay's ATIF file exporter, writing the trajectory under this task's durable
+ # evidence dir; Fabric promotes the resulting file into RunResult.artifacts. Both the
+ # Fabric artifact root and the relay output dir must exist and be durable.
+ relay_dir = evidence_dir / _RELAY_SUBDIR
+ artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR
+ relay_dir.mkdir(parents=True, exist_ok=True)
+ artifacts_dir.mkdir(parents=True, exist_ok=True)
+ profiles.append(self._trajectory_profile(profile_cls, relay_dir=relay_dir, artifacts_dir=artifacts_dir))
+
# Every task runs in its own fresh workspace: seed any ``inputs['files']`` into it (a no-op when
- # there are none), point the harness at it, and expose it as ``workspace`` filesystem evidence —
- # a uniform per-task dir that maps cleanly onto a per-task container volume later. Seeding runs
- # inside the guarded block so a bad seed (a path escaping the workspace, an unresolvable fileset)
- # fails just this task, not the whole run; it is synchronous and may block (a fileset handler
- # downloads), so it is offloaded off the shared event loop.
+ # there are none), point the harness at it via ``environment.workspace``, and expose it as
+ # ``workspace`` filesystem evidence — a uniform per-task dir that maps cleanly onto a per-task
+ # container volume later. Seeding runs inside the guarded block so a bad seed (a path escaping
+ # the workspace, an unresolvable fileset) fails just this task, not the whole run; it is
+ # synchronous and may block (a fileset handler downloads), so it is offloaded off the shared
+ # event loop.
workspace_dir = evidence_dir / _WORKSPACE_SUBDIR
workspace_dir.mkdir(parents=True, exist_ok=True)
try:
- # Stage seed files into the workspace for their on-disk side effect; the prompt is the task
- # instruction only, so the returned paths are unused.
- await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY))
- task_config = self._compose_config(agent_config, evidence_dir, workspace_dir)
- # Caller ``base_profiles`` are applied by Fabric over the config; the evaluator-owned
- # settings are re-asserted as trailing overlays so they win over any caller profile.
- lock_profiles = self._eval_lock_profiles(
- FabricProfileConfig, workspace_dir=workspace_dir, evidence_dir=evidence_dir
- )
+ seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY))
+ profiles.append(self._workspace_profile(profile_cls, workspace_dir=workspace_dir))
result = await asyncio.wait_for(
- # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``.
client.run(
- task_config,
- profiles=[*base_profiles, *lock_profiles],
+ agent_config,
+ profiles=profiles,
+ input=_fabric_input(task, seeded_files),
+ request_id=task.id,
base_dir=self._base_dir,
- request=RunRequest(input=task.agent_prompt(), request_id=task.id),
),
timeout=self._timeout_s,
)
@@ -228,17 +214,13 @@ def _to_trial(
if result.status != "succeeded":
return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata)
- # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52),
- # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the
- # trial's ``JsonValue``-typed response.
- output = _normalize_output(result.output)
return AgentEvalTrial(
id=f"{task.id}:fabric",
task_id=task.id,
status=AgentEvalTrialStatus.COMPLETED,
output=AgentOutput(
- output_text=_extract_output_text(output),
- response=output,
+ output_text=_extract_output_text(result.output),
+ response=result.output,
metadata={**base_metadata, "evidence_dir": str(evidence_dir)},
),
evidence=self._evidence(result, result_path, workspace_dir),
@@ -315,90 +297,31 @@ def _failed_trial(
},
)
- def _compose_config(
- self,
- agent_config: FabricConfig,
- evidence_dir: Path,
- workspace_dir: Path,
- ) -> FabricConfig:
- # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules
- # lookup, not a re-load, so the type is used where it's constructed instead of threaded down.
- from nemo_fabric import EnvironmentConfig # ty: ignore[unresolved-import]
-
- # Config-first composition (the SDK's recommended in-memory pattern): copy the base config and
- # apply this task's workspace, model, and trajectory settings directly onto it, rather than
- # layering FabricProfileConfig overlays.
- cfg = agent_config.model_copy(deep=True)
-
- # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from
- # it). ``provider="local"`` is required by the native planner. Any config-supplied
- # environment.workspace is overridden per task.
- environment = cfg.environment or EnvironmentConfig(provider="local")
- environment.provider = environment.provider or "local"
- environment.workspace = str(workspace_dir)
- cfg.environment = environment
-
- # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor).
+ def _build_profiles(self, profile_cls: type[FabricProfileConfig]) -> list[FabricProfileConfig]:
+ profiles = [profile_cls.from_mapping(profile) for profile in self._profiles]
if self._model:
+ # Apply the model as a final profile overlay (mirrors nemo_fabric.integrations.harbor).
provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai"
- cfg.models["default"] = {"provider": provider, "model": self._model}
-
- if self._capture_trajectory:
- # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the
- # Fabric artifact root so the promoted ``trajectory-*.atif.json`` persists. Requires the
- # ``nemo-relay`` gateway on PATH in the runtime.
- relay_dir = evidence_dir / _RELAY_SUBDIR
- artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR
- relay_dir.mkdir(parents=True, exist_ok=True)
- artifacts_dir.mkdir(parents=True, exist_ok=True)
- cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir))
- cfg.runtime.artifacts = str(artifacts_dir)
- cfg.environment.artifacts = str(artifacts_dir)
-
- return cfg
-
- def _eval_lock_profiles(
- self,
- profile_cls: type[FabricProfileConfig],
- *,
- workspace_dir: Path,
- evidence_dir: Path,
- ) -> list[FabricProfileConfig]:
- # ``_compose_config`` composes the evaluator's per-task settings onto the config, but Fabric
- # applies caller-supplied profiles OVER the config (last-wins), so a caller profile could
- # otherwise override them. Re-assert the evaluator-owned settings here as trailing overlays —
- # applied after the caller profiles — so the per-task workspace (isolation + ``workspace``
- # evidence integrity), the model under evaluation, and the trajectory artifact location stay
- # authoritative and non-overridable.
- overlays = [
- profile_cls.from_mapping(
- {"name": _WORKSPACE_PROFILE_NAME, "environment": {"workspace": str(workspace_dir)}}
- )
- ]
- if self._model:
- provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai"
- overlays.append(
- profile_cls.from_mapping(
- {"name": _MODEL_PROFILE_NAME, "models": {"default": {"provider": provider, "model": self._model}}}
+ profiles.append(
+ profile_cls(
+ name="eval_model",
+ models={"default": {"provider": provider, "model": self._model}},
)
)
- if self._capture_trajectory:
- artifacts_dir = str(evidence_dir / _ARTIFACTS_SUBDIR)
- overlays.append(
- profile_cls.from_mapping(
- {
- "name": _ARTIFACTS_PROFILE_NAME,
- "runtime": {"artifacts": artifacts_dir},
- "environment": {"artifacts": artifacts_dir},
- }
- )
- )
- return overlays
-
- def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
+ return profiles
+
+ def _trajectory_profile(
+ self, profile_cls: type[FabricProfileConfig], *, relay_dir: Path, artifacts_dir: Path
+ ) -> FabricProfileConfig:
+ # Relay ATIF/ATOF file exporter (mode=sdk): the harness emits its trajectory to a local
+ # nemo-relay gateway, which writes ``trajectory-*.atif.json`` under ``relay_dir``. No OTLP
+ # collector endpoint is involved. Requires the ``nemo-relay`` gateway on PATH in the runtime.
+ # The Fabric artifact root is pinned to a durable dir so the promoted trajectory persists.
+ #
# The observability component is built from nemo_relay's own typed config objects so Relay owns
# its schema (no hand-maintained dict that silently drifts when Relay changes it); imported
- # lazily since nemo-relay, like nemo-fabric, is an optional native dependency.
+ # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. ``schema_version``
+ # is omitted — ``FabricProfileConfig`` defaults it.
try:
from nemo_relay.observability import ( # ty: ignore[unresolved-import]
AtifConfig,
@@ -410,6 +333,7 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
raise RuntimeError(_MISSING_RELAY_MSG) from exc
relay_dir_str = str(relay_dir)
+ artifacts_dir_str = str(artifacts_dir)
observability = ComponentSpec(
config=ObservabilityConfig(
atif=AtifConfig(
@@ -427,7 +351,33 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]:
),
)
)
- return {"version": 1, "components": [observability.to_dict()]}
+ return profile_cls.from_mapping(
+ {
+ "name": _TRAJECTORY_PROFILE_NAME,
+ "description": "Capture the agent trajectory as ATIF via the NeMo Relay file exporter.",
+ "runtime": {"artifacts": artifacts_dir_str},
+ "environment": {"artifacts": artifacts_dir_str},
+ "telemetry": {
+ "enabled": True,
+ "provider": _TELEMETRY_PROVIDER,
+ "mode": _TELEMETRY_MODE,
+ "output_dir": relay_dir_str,
+ "config": {"version": 1, "components": [observability.to_dict()]},
+ },
+ }
+ )
+
+ def _workspace_profile(self, profile_cls: type[FabricProfileConfig], *, workspace_dir: Path) -> FabricProfileConfig:
+ # Point the harness at this task's staged workspace via ``environment.workspace`` (the codex-cli
+ # adapter resolves its cwd from it). Set as a final profile overlay, the same mechanism the
+ # trajectory profile uses for ``environment.artifacts``.
+ return profile_cls.from_mapping(
+ {
+ "name": _WORKSPACE_PROFILE_NAME,
+ "description": "Run the harness in the per-task evaluation workspace seeded with the task inputs.",
+ "environment": {"workspace": str(workspace_dir)},
+ }
+ )
def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path:
root = self._work_root
@@ -438,16 +388,22 @@ def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunCon
return Path(root) / task_dir
-def _normalize_output(output: RunOutput | JsonValue) -> JsonValue:
- """Unwrap a Fabric ``RunResult.output`` into the plain JSON value the trial response stores.
+def _fabric_input(task: AgentEvalTask, seeded_files: Sequence[str] = ()) -> str:
+ """Frame the task as the harness's input text.
- Newer Fabric wraps output in a ``RunOutput`` (the RunOutput response contract), which is a
- ``Mapping``; copy it into a plain dict (equivalent to its ``to_mapping()``). Raw/older JSON outputs
- are already JSON values and pass through unchanged.
+ When files were staged into the workspace, list them by name and invite the agent to work in its
+ current directory rather than dumping their contents inline; the seed-files key is dropped from
+ the echoed inputs since those files are already on disk.
"""
- if isinstance(output, Mapping):
- return dict(output)
- return output
+ body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY}
+ lines = [f"Task id: {task.id}", f"Intent: {task.intent}"]
+ if body_inputs:
+ lines += ["", f"Inputs: {body_inputs}"]
+ if seeded_files:
+ lines += ["", "These files are already in your working directory:"]
+ lines += [f" - {path}" for path in seeded_files]
+ lines += ["", "Complete the task by reading, creating, and editing files in your current directory."]
+ return "\n".join(lines) + "\n"
def _extract_output_text(output: object) -> str | None:
diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py
index 6c0da4ac21..445af23e24 100644
--- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py
+++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py
@@ -97,23 +97,6 @@ def _id_must_not_be_empty(cls, value: str) -> str:
raise ValueError("task id must not be empty")
return value
- def agent_prompt(self) -> str:
- """The intent-free prompt handed to the agent under evaluation.
-
- Exactly the task's natural-language instruction (``inputs["instruction"]``), with no
- runtime-added framing. ``intent`` is deliberately never used: it is the eval-side description
- of the desired behavior (what the grader checks for), so exposing it to the agent is a
- reward-hacking hole.
-
- Raises ``ValueError`` when ``inputs["instruction"]`` is missing or empty; a task with no
- instruction cannot be evaluated, so the runner fails that task rather than running an agent on
- an empty prompt.
- """
- instruction = self.inputs.get("instruction")
- if instruction:
- return str(instruction)
- raise ValueError(f"task {self.id!r} has no instruction: set inputs['instruction']")
-
@field_serializer("metrics", when_used="json")
def _serialize_metrics(self, metrics: list[Metric]) -> list[dict[str, Any]]:
"""Serialize local metric instances as descriptors for run bundles."""
diff --git a/services/studio/src/nmp/studio/env_mappings.py b/services/studio/src/nmp/studio/env_mappings.py
index c46a6f68f1..5ff3fe3f17 100644
--- a/services/studio/src/nmp/studio/env_mappings.py
+++ b/services/studio/src/nmp/studio/env_mappings.py
@@ -82,7 +82,7 @@ class EnvMapping:
EnvMapping(
marker="STUDIO_UI_VITE_FF_DATA_DESIGNER_ENABLED",
config_path="studio.feature_flags.data_designer_enabled",
- default="true",
+ default="false",
),
EnvMapping(
marker="STUDIO_UI_VITE_FF_DATASETS_ENABLED",
diff --git a/tests/unit/release/test_write_release_bundle_metadata.py b/tests/unit/release/test_write_release_bundle_metadata.py
new file mode 100644
index 0000000000..1096d67f00
--- /dev/null
+++ b/tests/unit/release/test_write_release_bundle_metadata.py
@@ -0,0 +1,441 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import hashlib
+import importlib.util
+import json
+import zipfile
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+
+
+def load_bundle_metadata_module() -> ModuleType:
+ script_path = Path(__file__).parents[3] / ".github/scripts/write_release_bundle_metadata.py"
+ spec = importlib.util.spec_from_file_location("write_release_bundle_metadata", script_path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+bundle_metadata = load_bundle_metadata_module()
+BundleMetadataError = bundle_metadata.BundleMetadataError
+
+
+def selected_artifacts(*artifacts: dict[str, str]) -> str:
+ return json.dumps(list(artifacts), separators=(",", ":"))
+
+
+def write_wheel(
+ sdk_artifacts_dir: Path,
+ sdk_id: str,
+ *,
+ filename: str = "nemo_platform-1.0.0-py3-none-any.whl",
+ version: str = "1.0.0",
+ metadata_files: list[str] | None = None,
+) -> Path:
+ artifact_dir = sdk_artifacts_dir / f"release-sdk-{sdk_id}"
+ artifact_dir.mkdir(parents=True, exist_ok=True)
+ wheel_path = artifact_dir / filename
+ metadata_files = metadata_files if metadata_files is not None else ["nemo_platform-1.0.0.dist-info/METADATA"]
+
+ with zipfile.ZipFile(wheel_path, "w") as wheel:
+ for metadata_file in metadata_files:
+ wheel.writestr(
+ metadata_file,
+ f"""Metadata-Version: 2.1
+Name: nemo-platform
+Version: {version}
+""",
+ )
+ wheel.writestr("nemo_platform-1.0.0.dist-info/WHEEL", "Wheel-Version: 1.0\n")
+
+ return wheel_path
+
+
+def write_metadata(
+ tmp_path: Path,
+ *,
+ selected: str | None = None,
+ cadence: str = "release",
+ release_label: str = "1.0.0",
+ release_date_json: str = '"2026-06-30"',
+ source_sha: str = "a" * 40,
+) -> tuple[Path, Path]:
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ bundle_dir = tmp_path / "release-bundle"
+ write_wheel(sdk_artifacts_dir, "nemo-platform")
+
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=bundle_dir,
+ selected_artifacts_json=selected or selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence=cadence,
+ release_label=release_label,
+ release_date_json=release_date_json,
+ source_sha=source_sha,
+ )
+ return sdk_artifacts_dir, bundle_dir
+
+
+def read_manifest(bundle_dir: Path) -> dict[str, object]:
+ return json.loads((bundle_dir / "release-manifest.json").read_text(encoding="utf-8"))
+
+
+def parse_checksums(bundle_dir: Path) -> dict[str, str]:
+ entries = {}
+ for line in (bundle_dir / "checksums.txt").read_text(encoding="utf-8").splitlines():
+ digest, path = line.split(" ", 1)
+ entries[path] = digest
+ return entries
+
+
+def sha256(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def test_one_sdk_wheel_writes_manifest_and_checksums(tmp_path: Path):
+ sdk_artifacts_dir, bundle_dir = write_metadata(tmp_path)
+ source_wheel = sdk_artifacts_dir / "release-sdk-nemo-platform/nemo_platform-1.0.0-py3-none-any.whl"
+ bundled_wheel = bundle_dir / "wheels/nemo_platform-1.0.0-py3-none-any.whl"
+
+ assert bundled_wheel.read_bytes() == source_wheel.read_bytes()
+ assert read_manifest(bundle_dir) == {
+ "cadence": "release",
+ "release_label": "1.0.0",
+ "release_date": "2026-06-30",
+ "source_sha": "a" * 40,
+ "artifacts": [
+ {
+ "type": "sdk",
+ "id": "nemo-platform",
+ "version": "1.0.0",
+ "path": "wheels/nemo_platform-1.0.0-py3-none-any.whl",
+ }
+ ],
+ }
+
+ assert parse_checksums(bundle_dir) == {
+ "release-manifest.json": sha256(bundle_dir / "release-manifest.json"),
+ "wheels/nemo_platform-1.0.0-py3-none-any.whl": sha256(bundled_wheel),
+ }
+
+
+def test_one_sdk_wheel_can_be_downloaded_directly_to_artifacts_dir(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ bundle_dir = tmp_path / "release-bundle"
+ nested_wheel = write_wheel(sdk_artifacts_dir, "nemo-platform")
+ direct_wheel = sdk_artifacts_dir / nested_wheel.name
+ nested_wheel.rename(direct_wheel)
+ nested_wheel.parent.rmdir()
+
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=bundle_dir,
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+ bundled_wheel = bundle_dir / "wheels/nemo_platform-1.0.0-py3-none-any.whl"
+ assert bundled_wheel.read_bytes() == direct_wheel.read_bytes()
+ assert read_manifest(bundle_dir)["artifacts"][0]["path"] == ( # type: ignore[index]
+ "wheels/nemo_platform-1.0.0-py3-none-any.whl"
+ )
+
+
+def test_release_date_json_null_becomes_manifest_null(tmp_path: Path):
+ _, bundle_dir = write_metadata(tmp_path, release_date_json="null")
+
+ assert read_manifest(bundle_dir)["release_date"] is None
+
+
+def test_checksums_only_include_manifest_and_wheels(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ bundle_dir = tmp_path / "release-bundle"
+ write_wheel(sdk_artifacts_dir, "nemo-platform")
+ bundle_dir.mkdir()
+ (bundle_dir / "stale.txt").write_text("old file\n", encoding="utf-8")
+
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=bundle_dir,
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+ assert set(parse_checksums(bundle_dir)) == {
+ "release-manifest.json",
+ "wheels/nemo_platform-1.0.0-py3-none-any.whl",
+ }
+
+
+def test_rc_label_stays_release_label_and_wheel_version_comes_from_metadata(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ bundle_dir = tmp_path / "release-bundle"
+ write_wheel(
+ sdk_artifacts_dir,
+ "nemo-platform",
+ filename="nemo_platform-1.0.0rc0-py3-none-any.whl",
+ version="1.0.0rc0",
+ )
+
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=bundle_dir,
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="rc",
+ release_label="1.0.0-rc0",
+ release_date_json="null",
+ source_sha="b" * 40,
+ )
+
+ manifest = read_manifest(bundle_dir)
+ assert manifest["release_label"] == "1.0.0-rc0"
+ assert manifest["artifacts"][0]["version"] == "1.0.0rc0" # type: ignore[index]
+
+
+def test_missing_sdk_artifact_directory_fails_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="missing downloaded SDK artifact directory"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_zero_wheels_fails_clearly(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ (sdk_artifacts_dir / "release-sdk-nemo-platform").mkdir(parents=True)
+
+ with pytest.raises(BundleMetadataError, match="expected exactly one wheel"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_multiple_wheels_for_one_sdk_fails_clearly(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ write_wheel(sdk_artifacts_dir, "nemo-platform")
+ write_wheel(
+ sdk_artifacts_dir,
+ "nemo-platform",
+ filename="nemo_platform-1.0.1-py3-none-any.whl",
+ version="1.0.1",
+ )
+
+ with pytest.raises(BundleMetadataError, match="expected exactly one wheel"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_multiple_directly_downloaded_wheels_fail_clearly(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ nested_wheel = write_wheel(sdk_artifacts_dir, "nemo-platform")
+ direct_wheel = sdk_artifacts_dir / nested_wheel.name
+ nested_wheel.rename(direct_wheel)
+ nested_wheel.parent.rmdir()
+ write_wheel(
+ sdk_artifacts_dir,
+ "extra",
+ filename="nemo_platform-1.0.1-py3-none-any.whl",
+ version="1.0.1",
+ ).rename(
+ sdk_artifacts_dir / "nemo_platform-1.0.1-py3-none-any.whl",
+ )
+
+ with pytest.raises(BundleMetadataError, match="expected exactly one wheel"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_unsupported_artifact_type_fails_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="unsupported artifact type"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts({"type": "helm", "id": "platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_container_artifacts_become_metadata_only_entries(tmp_path: Path):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ bundle_dir = tmp_path / "release-bundle"
+ write_wheel(sdk_artifacts_dir, "nemo-platform")
+
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=bundle_dir,
+ selected_artifacts_json=selected_artifacts(
+ {"type": "sdk", "id": "nemo-platform"},
+ {"type": "container", "id": "nmp-automodel-tasks"},
+ {"type": "container", "id": "nmp-unsloth-training"},
+ ),
+ cadence="rc",
+ release_label="1.0.0-rc1",
+ release_date_json="null",
+ source_sha="c" * 40,
+ )
+
+ artifacts = read_manifest(bundle_dir)["artifacts"]
+ assert artifacts[1:] == [ # type: ignore[index]
+ {"type": "container", "id": "nmp-automodel-tasks", "version": "1.0.0-rc1"},
+ {"type": "container", "id": "nmp-unsloth-training", "version": "1.0.0-rc1"},
+ ]
+ # Container entries are metadata-only: no path, and nothing extra in checksums.
+ assert set(parse_checksums(bundle_dir)) == {
+ "release-manifest.json",
+ "wheels/nemo_platform-1.0.0-py3-none-any.whl",
+ }
+
+
+def test_container_only_selection_is_valid(tmp_path: Path):
+ bundle_dir = tmp_path / "release-bundle"
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=bundle_dir,
+ selected_artifacts_json=selected_artifacts(
+ {"type": "container", "id": "nmp-automodel-tasks"},
+ {"type": "container", "id": "nmp-unsloth-training"},
+ ),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+ # Container-only bundle: only container entries, no wheels, checksums = manifest only.
+ assert read_manifest(bundle_dir)["artifacts"] == [
+ {"type": "container", "id": "nmp-automodel-tasks", "version": "1.0.0"},
+ {"type": "container", "id": "nmp-unsloth-training", "version": "1.0.0"},
+ ]
+ assert set(parse_checksums(bundle_dir)) == {"release-manifest.json"}
+
+
+def test_empty_selection_fails_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="non-empty list"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json="[]",
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_duplicate_container_ids_fail_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="duplicate container id: nmp-automodel-tasks"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts(
+ {"type": "sdk", "id": "nemo-platform"},
+ {"type": "container", "id": "nmp-automodel-tasks"},
+ {"type": "container", "id": "nmp-automodel-tasks"},
+ ),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_unsafe_container_id_fails_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="container id must be a safe single path segment"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts(
+ {"type": "sdk", "id": "nemo-platform"},
+ {"type": "container", "id": "../evil"},
+ ),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_duplicate_selected_sdk_ids_fail_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="duplicate sdk id: nemo-platform"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=tmp_path / "downloaded-artifacts",
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts(
+ {"type": "sdk", "id": "nemo-platform"},
+ {"type": "sdk", "id": "nemo-platform"},
+ ),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
+
+
+def test_malformed_release_date_json_fails_clearly(tmp_path: Path):
+ with pytest.raises(BundleMetadataError, match="release_date_json must be valid JSON"):
+ write_metadata(tmp_path, release_date_json="2026-06-30")
+
+
+@pytest.mark.parametrize(
+ "metadata_files",
+ [
+ [],
+ [
+ "nemo_platform-1.0.0.dist-info/METADATA",
+ "other-1.0.0.dist-info/METADATA",
+ ],
+ ],
+)
+def test_missing_or_duplicate_wheel_metadata_fails_clearly(tmp_path: Path, metadata_files: list[str]):
+ sdk_artifacts_dir = tmp_path / "downloaded-artifacts"
+ write_wheel(sdk_artifacts_dir, "nemo-platform", metadata_files=metadata_files)
+
+ with pytest.raises(BundleMetadataError, match="expected exactly one METADATA file"):
+ bundle_metadata.write_release_bundle_metadata(
+ sdk_artifacts_dir=sdk_artifacts_dir,
+ bundle_dir=tmp_path / "release-bundle",
+ selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}),
+ cadence="release",
+ release_label="1.0.0",
+ release_date_json="null",
+ source_sha="a" * 40,
+ )
diff --git a/uv.lock b/uv.lock
index 0bbb03ade0..a708953e70 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5412,10 +5412,10 @@ requires-dist = [
{ name = "nemo-platform-sdk", marker = "extra == 'nmp-common'", editable = "sdk/python/nemo-platform" },
{ name = "nemo-platform-sdk", marker = "extra == 'plugins'", editable = "sdk/python/nemo-platform" },
{ name = "nemo-platform-sdk", marker = "extra == 'services'", editable = "sdk/python/nemo-platform" },
- { name = "nemo-safe-synthesizer", marker = "extra == 'all'", specifier = "==0.1.7" },
- { name = "nemo-safe-synthesizer", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = "==0.1.7" },
- { name = "nemo-safe-synthesizer", marker = "extra == 'plugins'", specifier = "==0.1.7" },
- { name = "nemo-safe-synthesizer", marker = "extra == 'services'", specifier = "==0.1.7" },
+ { name = "nemo-safe-synthesizer", marker = "extra == 'all'", specifier = "==0.1.2" },
+ { name = "nemo-safe-synthesizer", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = "==0.1.2" },
+ { name = "nemo-safe-synthesizer", marker = "extra == 'plugins'", specifier = "==0.1.2" },
+ { name = "nemo-safe-synthesizer", marker = "extra == 'services'", specifier = "==0.1.2" },
{ name = "nemoguardrails", extras = ["tracing"], marker = "extra == 'all'", specifier = "==0.23.0" },
{ name = "nemoguardrails", extras = ["tracing"], marker = "extra == 'guardrails-service'", specifier = "==0.23.0" },
{ name = "nemoguardrails", extras = ["tracing"], marker = "extra == 'nemo-guardrails-plugin'", specifier = "==0.23.0" },
@@ -6050,7 +6050,7 @@ dev = [
[[package]]
name = "nemo-safe-synthesizer"
-version = "0.1.7"
+version = "0.1.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "colorama", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
@@ -6066,10 +6066,9 @@ dependencies = [
{ name = "setuptools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
{ name = "structlog", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
{ name = "tqdm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
- { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
]
wheels = [
- { url = "https://files.pythonhosted.org/packages/5c/31/573f66c9b87e663d23f061fb1f65ef5b8dc72a0597bff7d4b044e88c2a23/nemo_safe_synthesizer-0.1.7-py3-none-any.whl", hash = "sha256:07ad037e6ded8f7020039fa88efdab4aaaf805ea52fb8238380086e0811eaaf0", size = 586219, upload-time = "2026-07-10T18:34:02.825Z" },
+ { url = "https://files.pythonhosted.org/packages/e7/f8/2ec77a08595640b331c420c6978cb626be6f28c770481c0e5a3e900a27ee/nemo_safe_synthesizer-0.1.2-py3-none-any.whl", hash = "sha256:fbf6f9179052d0ac27ad4493238009a5e1c7ae7775354f062d5e90725890b0ce", size = 561741, upload-time = "2026-06-04T20:03:40.12Z" },
]
[[package]]
@@ -6127,7 +6126,7 @@ requires-dist = [
{ name = "lark", marker = "extra == 'nemo-platform-plugin'", specifier = ">=1.1.0" },
{ name = "nemo-platform", editable = "packages/nemo_platform" },
{ name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" },
- { name = "nemo-safe-synthesizer", specifier = "==0.1.7" },
+ { name = "nemo-safe-synthesizer", specifier = "==0.1.2" },
{ name = "openai", marker = "extra == 'nemo-platform-plugin'", specifier = ">=1.109.1" },
{ name = "pydantic", marker = "extra == 'nemo-platform-plugin'", specifier = ">=2.10.3" },
{ name = "pydantic", extras = ["email"], specifier = ">=2.9.2" },
diff --git a/web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts b/web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts
new file mode 100644
index 0000000000..a13495c487
--- /dev/null
+++ b/web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts
@@ -0,0 +1,16 @@
+/**
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Generated by Orval 🍺
+ * Do not edit manually.
+ * agents (plugin)
+ */
+
+export type AgentsGetDeploymentLogsParams = {
+ /**
+ * @minimum 0
+ * @maximum 10000
+ */
+ tail?: number;
+};
diff --git a/web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts b/web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts
new file mode 100644
index 0000000000..b2dd176038
--- /dev/null
+++ b/web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts
@@ -0,0 +1,20 @@
+/**
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Generated by Orval 🍺
+ * Do not edit manually.
+ * agents (plugin)
+ */
+import type { LogLine } from './LogLine.ts';
+
+/**
+ * Response body for ``GET /deployments/{name}/logs``.
+ */
+export interface DeploymentLogsResponse {
+ data: LogLine[];
+ /** Number of lines actually returned. */
+ total_lines: number;
+ /** Byte offset just past the returned tail; pass as Last-Event-ID to resume the stream without gaps. */
+ next_offset: number;
+}
diff --git a/web/packages/sdk/generated/agents/schema/LogLine.ts b/web/packages/sdk/generated/agents/schema/LogLine.ts
new file mode 100644
index 0000000000..1a16b61a43
--- /dev/null
+++ b/web/packages/sdk/generated/agents/schema/LogLine.ts
@@ -0,0 +1,24 @@
+/**
+ * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Generated by Orval 🍺
+ * Do not edit manually.
+ * agents (plugin)
+ */
+
+/**
+ * One line shaped to match ``PlatformJobLog`` so Studio's LogViewer renders it as-is.
+ */
+export interface LogLine {
+ /** ISO-8601 timestamp parsed from the line; empty when absent. */
+ timestamp: string;
+ /** Empty — kept for shape compatibility with jobs logs. */
+ job?: string;
+ /** Empty — kept for shape compatibility. */
+ job_step?: string;
+ /** Empty — kept for shape compatibility. */
+ job_task?: string;
+ /** The raw log line minus any parsed timestamp prefix. */
+ message: string;
+}
diff --git a/web/packages/studio/env/.env.dev.local.sample b/web/packages/studio/env/.env.dev.local.sample
index 10dce482bf..10d1819ee3 100644
--- a/web/packages/studio/env/.env.dev.local.sample
+++ b/web/packages/studio/env/.env.dev.local.sample
@@ -18,7 +18,7 @@ VITE_FF_BASE_MODELS_ENABLED='true'
VITE_FF_CODING_AGENT_STUDIO_ENABLED='false'
VITE_FF_CUSTOMIZER_ENABLED='false'
VITE_FF_DASHBOARD_ENABLED='false'
-VITE_FF_DATA_DESIGNER_ENABLED='true'
+VITE_FF_DATA_DESIGNER_ENABLED='false'
VITE_FF_DATASETS_ENABLED='true'
VITE_FF_DEPLOYMENTS_ENABLED='false'
VITE_FF_EVALUATOR_BENCHMARKS_ENABLED='false'
diff --git a/web/packages/studio/src/constants/featureFlags/featureFlags.ts b/web/packages/studio/src/constants/featureFlags/featureFlags.ts
index d60dc66a46..b02d4c8b1c 100644
--- a/web/packages/studio/src/constants/featureFlags/featureFlags.ts
+++ b/web/packages/studio/src/constants/featureFlags/featureFlags.ts
@@ -59,7 +59,7 @@ export const flagDefinitions = {
codingAgentStudioEnabled: previewFlag('VITE_FF_CODING_AGENT_STUDIO_ENABLED', false),
customizerEnabled: previewFlag('VITE_FF_CUSTOMIZER_ENABLED', false),
dashboardEnabled: previewFlag('VITE_FF_DASHBOARD_ENABLED', false),
- dataDesignerEnabled: previewFlag('VITE_FF_DATA_DESIGNER_ENABLED', true),
+ dataDesignerEnabled: previewFlag('VITE_FF_DATA_DESIGNER_ENABLED'),
datasetsEnabled: previewFlag('VITE_FF_DATASETS_ENABLED', true),
deploymentsEnabled: previewFlag('VITE_FF_DEPLOYMENTS_ENABLED'),
evaluatorBenchmarksEnabled: previewFlag('VITE_FF_EVALUATOR_BENCHMARKS_ENABLED', false),
diff --git a/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx b/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx
index ce4b2c26bd..2bda8c418f 100644
--- a/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx
+++ b/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx
@@ -65,7 +65,6 @@ const LandingComposer = ({
data-testid="dashboard-landing-composer"
onSubmit={handleSubmit}
className="w-full rounded-lg border border-base bg-surface-base p-2 shadow-xl"
- data-tour="dashboard-get-started"
>