diff --git a/.github/actions/build-policy-wasm/action.yaml b/.github/actions/build-policy-wasm/action.yaml index 863406d262..98d3219447 100644 --- a/.github/actions/build-policy-wasm/action.yaml +++ b/.github/actions/build-policy-wasm/action.yaml @@ -10,6 +10,4 @@ runs: steps: - name: Build policy WASM shell: bash - env: - REPO_ROOT: ${{ github.action_path }}/../../.. - run: "${REPO_ROOT}/script/build_policy_wasm.sh" + run: ./script/build_policy_wasm.sh diff --git a/.github/actions/setup-kind-cluster/action.yaml b/.github/actions/setup-kind-cluster/action.yaml index 57d591c75f..961d6a8b09 100644 --- a/.github/actions/setup-kind-cluster/action.yaml +++ b/.github/actions/setup-kind-cluster/action.yaml @@ -141,6 +141,7 @@ runs: run: | e2e/k8s/scripts/prepull_kind_images.sh \ "${NMP_E2E_REGISTRY}/nmp-api:${NMP_E2E_TAG}" \ + "${NMP_E2E_REGISTRY}/nmp-core:${NMP_E2E_TAG}" \ "${NMP_E2E_REGISTRY}/nmp-cpu-tasks:${NMP_E2E_TAG}" - name: Install NeMo Platform diff --git a/.github/assets/ngc/containers/nmp-core.md b/.github/assets/ngc/containers/nmp-core.md new file mode 100644 index 0000000000..04b7a72176 --- /dev/null +++ b/.github/assets/ngc/containers/nmp-core.md @@ -0,0 +1,11 @@ +## NeMo Platform Core Container + +This container image provides required services for a deployed version of the NeMo Platform. + +### Resources + +[Documentation](https://docs.nvidia.com/nemo-platform) + +### License + +This container is licensed under the [Apache License 2.0](https://github.com/NVIDIA-NeMo/nemo-platform/blob/main/LICENSE). diff --git a/.github/scripts/write_release_bundle_metadata.py b/.github/scripts/write_release_bundle_metadata.py new file mode 100644 index 0000000000..4eff45ad1d --- /dev/null +++ b/.github/scripts/write_release_bundle_metadata.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Write release bundle metadata for downloaded SDK wheel artifacts. + +Container artifacts are metadata-only manifest entries: the image bits are +built and staged by the release consumer from its dev registry at the bundle's +source SHA, so container entries carry no path or checksum. +""" + +import argparse +import hashlib +import json +import re +import shutil +import sys +import zipfile +from email.parser import BytesParser +from email.policy import default +from pathlib import Path +from typing import Literal + +Cadence = Literal["nightly", "rc", "release"] + + +class BundleMetadataError(Exception): + """Raised when the release bundle metadata cannot be written safely.""" + + +def safe_artifact_id(artifact_type: str, artifact_id: str) -> str: + if not re.fullmatch(r"[A-Za-z0-9._-]+", artifact_id) or artifact_id in {".", ".."}: + raise BundleMetadataError(f"selected {artifact_type} id must be a safe single path segment: {artifact_id}") + return artifact_id + + +def parse_release_date_json(value: str) -> str | None: + try: + parsed = json.loads(value) + except json.JSONDecodeError as error: + raise BundleMetadataError(f"release_date_json must be valid JSON: {error.msg}") from error + + if parsed is not None and not isinstance(parsed, str): + raise BundleMetadataError("release_date_json must be a JSON string or null") + return parsed + + +def artifact_ref(artifact_type: object, artifact_id: object) -> str: + return f"{artifact_type}:{artifact_id}" + + +def parse_selected_artifact_ids(value: str) -> dict[str, list[str]]: + """Parse selected_artifacts_json into ids grouped by artifact type.""" + try: + parsed = json.loads(value) + except json.JSONDecodeError as error: + raise BundleMetadataError(f"selected_artifacts_json must be valid JSON: {error.msg}") from error + + if not isinstance(parsed, list) or not parsed: + raise BundleMetadataError("selected_artifacts_json must be a non-empty list") + + ids_by_type: dict[str, list[str]] = {"sdk": [], "container": []} + seen: dict[str, set[str]] = {artifact_type: set() for artifact_type in ids_by_type} + for artifact in parsed: + if not isinstance(artifact, dict): + raise BundleMetadataError("selected_artifacts_json entries must be objects") + + artifact_type = artifact.get("type") + artifact_id = artifact.get("id") + if artifact_type not in ids_by_type: + raise BundleMetadataError( + f"unsupported artifact type in bundle selection: {artifact_ref(artifact_type, artifact_id)}" + ) + if not isinstance(artifact_id, str) or not artifact_id: + raise BundleMetadataError(f"selected {artifact_type} artifact id must be a non-empty string") + + checked_id = safe_artifact_id(artifact_type, artifact_id) + if checked_id in seen[artifact_type]: + raise BundleMetadataError(f"selected_artifacts_json contains duplicate {artifact_type} id: {checked_id}") + + seen[artifact_type].add(checked_id) + ids_by_type[artifact_type].append(checked_id) + + # A bundle may be SDK-only, container-only, or mixed; the non-empty-list + # check above already guarantees at least one artifact of some type. + return ids_by_type + + +def find_sdk_wheel(sdk_artifacts_dir: Path, sdk_id: str, *, single_sdk_artifact: bool) -> Path: + artifact_dir = sdk_artifacts_dir / f"release-sdk-{sdk_id}" + if not artifact_dir.is_dir(): + if single_sdk_artifact: + # download-artifact extracts one pattern match directly into the target path. + wheels = sorted(sdk_artifacts_dir.glob("*.whl")) + if wheels: + if len(wheels) != 1: + raise BundleMetadataError(f"expected exactly one wheel in {sdk_artifacts_dir}, found {len(wheels)}") + return wheels[0] + raise BundleMetadataError(f"missing downloaded SDK artifact directory: {artifact_dir}") + + wheels = sorted(artifact_dir.glob("*.whl")) + if len(wheels) != 1: + raise BundleMetadataError(f"expected exactly one wheel in {artifact_dir}, found {len(wheels)}") + return wheels[0] + + +def read_wheel_version(path: Path) -> str: + with zipfile.ZipFile(path) as wheel: + metadata_files = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata_files) != 1: + raise BundleMetadataError( + f"expected exactly one METADATA file in wheel {path.name}, found {len(metadata_files)}" + ) + + metadata = BytesParser(policy=default).parsebytes(wheel.read(metadata_files[0])) + + version = metadata["Version"] + if not version: + raise BundleMetadataError(f"wheel metadata must include Version: {path.name}") + return str(version) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def bundle_relative_path(bundle_dir: Path, path: Path) -> str: + return path.relative_to(bundle_dir).as_posix() + + +def write_checksums(bundle_dir: Path) -> Path: + checksums_path = bundle_dir / "checksums.txt" + files = [ + bundle_dir / "release-manifest.json", + *sorted(path for path in (bundle_dir / "wheels").rglob("*") if path.is_file()), + ] + + with checksums_path.open("w", encoding="utf-8") as checksums: + for path in files: + checksums.write(f"{file_sha256(path)} {bundle_relative_path(bundle_dir, path)}\n") + + return checksums_path + + +def prepare_bundle_dir(bundle_dir: Path) -> Path: + wheels_dir = bundle_dir / "wheels" + if wheels_dir.exists(): + shutil.rmtree(wheels_dir) + + bundle_dir.mkdir(parents=True, exist_ok=True) + for filename in ("release-manifest.json", "checksums.txt"): + path = bundle_dir / filename + if path.exists(): + path.unlink() + + wheels_dir.mkdir() + return wheels_dir + + +def write_release_bundle_metadata( + *, + sdk_artifacts_dir: Path, + bundle_dir: Path, + selected_artifacts_json: str, + cadence: Cadence, + release_label: str, + release_date_json: str, + source_sha: str, +) -> dict[str, object]: + if not release_label: + raise BundleMetadataError("release_label is required") + if not source_sha: + raise BundleMetadataError("source_sha is required") + + ids_by_type = parse_selected_artifact_ids(selected_artifacts_json) + sdk_ids = ids_by_type["sdk"] + release_date = parse_release_date_json(release_date_json) + wheels_dir = prepare_bundle_dir(bundle_dir) + + artifacts: list[dict[str, str]] = [] + for sdk_id in sdk_ids: + source_wheel = find_sdk_wheel(sdk_artifacts_dir, sdk_id, single_sdk_artifact=len(sdk_ids) == 1) + wheel_version = read_wheel_version(source_wheel) + wheel_path = wheels_dir / source_wheel.name + if wheel_path.exists(): + raise BundleMetadataError(f"duplicate wheel filename in bundle: {source_wheel.name}") + + shutil.copy2(source_wheel, wheel_path) + artifacts.append( + { + "type": "sdk", + "id": sdk_id, + "version": wheel_version, + "path": bundle_relative_path(bundle_dir, wheel_path), + } + ) + + # Container artifacts are metadata-only: the consumer stages the images + # from its dev registry by source_sha and tags them with release_label. + for container_id in ids_by_type["container"]: + artifacts.append( + { + "type": "container", + "id": container_id, + "version": release_label, + } + ) + + manifest: dict[str, object] = { + "cadence": cadence, + "release_label": release_label, + "release_date": release_date, + "source_sha": source_sha, + "artifacts": artifacts, + } + + manifest_path = bundle_dir / "release-manifest.json" + manifest_path.write_text(f"{json.dumps(manifest, indent=2)}\n", encoding="utf-8") + write_checksums(bundle_dir) + return manifest + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sdk-artifacts-dir", required=True, type=Path) + parser.add_argument("--bundle-dir", required=True, type=Path) + parser.add_argument("--selected-artifacts-json", required=True) + parser.add_argument("--cadence", required=True, choices=["nightly", "rc", "release"]) + parser.add_argument("--release-label", required=True) + parser.add_argument("--release-date-json", required=True) + parser.add_argument("--source-sha", required=True) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + try: + manifest = write_release_bundle_metadata( + sdk_artifacts_dir=args.sdk_artifacts_dir, + bundle_dir=args.bundle_dir, + selected_artifacts_json=args.selected_artifacts_json, + cadence=args.cadence, + release_label=args.release_label, + release_date_json=args.release_date_json, + source_sha=args.source_sha, + ) + except (BundleMetadataError, OSError, zipfile.BadZipFile) as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + + print(f"Wrote release bundle metadata for {len(manifest['artifacts'])} artifact(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/.github/wheel-constraints/nemo-platform-services.txt b/.github/wheel-constraints/nemo-platform-services.txt index a779d6db9d..888cbc4b88 100644 --- a/.github/wheel-constraints/nemo-platform-services.txt +++ b/.github/wheel-constraints/nemo-platform-services.txt @@ -43,7 +43,7 @@ langchain==1.3.13 lark==1.3.1 litellm<1.92 # 1.92.0 native build has no py3.14 wheel nemo-anonymizer==0.2.1 -nemo-safe-synthesizer==0.1.7 +nemo-safe-synthesizer==0.1.2 nemoguardrails==0.23.0 ngcsdk==4.20.1 nvidia-ml-py==13.610.43 diff --git a/.github/workflows/ngc-metadata.yaml b/.github/workflows/ngc-metadata.yaml index 5da4fc41fe..c25fbfb970 100644 --- a/.github/workflows/ngc-metadata.yaml +++ b/.github/workflows/ngc-metadata.yaml @@ -2,9 +2,6 @@ name: NGC Metadata Sync on: workflow_call: - secrets: - AIRE_NGC_GITHUB_PLATFORM_RW: - required: true workflow_dispatch: permissions: diff --git a/.github/workflows/release-bundle.yaml b/.github/workflows/release-bundle.yaml new file mode 100644 index 0000000000..a91f2e4a77 --- /dev/null +++ b/.github/workflows/release-bundle.yaml @@ -0,0 +1,674 @@ +name: Release Bundle + +on: + workflow_call: + inputs: + cadence: + description: "Release cadence: nightly, rc, or release." + required: true + type: string + source_sha: + description: "Expected source commit SHA for RC/stable releases." + required: false + type: string + default: "" + version: + description: "SemVer core version for RC or stable releases, for example 1.0.0." + required: false + type: string + default: "" + release_date: + description: "Optional. Provide only for first-time registration of this release version. Leave blank for reruns or when this release version is already registered. Format: YYYY-MM-DD." + required: false + type: string + default: "" + release_scope: + description: "Release artifact scope: all, sdks, containers, or custom." + required: false + type: string + default: "all" + sdk_ids: + description: "Comma-separated SDK IDs for custom releases." + required: false + type: string + default: "" + container_ids: + description: "Comma-separated container IDs for custom releases." + required: false + type: string + default: "" + send_notifications: + description: "Whether downstream release jobs should send notifications." + required: false + type: boolean + default: true + secrets: + CI_DISPATCH_TOKEN: + description: "Token with access to send repository dispatch events to the dispatch repo." + required: true + CI_DISPATCH_REPO: + description: "Repository that receives release bundle handoff dispatch events." + required: true +permissions: + contents: read + +jobs: + plan-release: + name: Plan release + runs-on: ubuntu-latest + # Normalize nightly and RC/stable planner results into one contract for downstream jobs. + # Later jobs should read needs.plan-release.outputs.* without caring which cadence produced them. + outputs: + source_repo: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_repo || github.repository }} + source_ref: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_ref || steps.resolve-release-metadata.outputs.source_ref }} + release_label: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.release_label || steps.resolve-release-metadata.outputs.release_label }} + nightly_timestamp: ${{ steps.resolve-nightly-source.outputs.nightly_timestamp }} + selected_artifacts_json: ${{ steps.plan-assets.outputs.selected_artifacts_json }} + sdk_count: ${{ steps.plan-assets.outputs.sdk_count }} + container_count: ${{ steps.plan-assets.outputs.container_count }} + sdk_matrix: ${{ steps.plan-assets.outputs.sdk_matrix }} + release_tag: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.release_tag || steps.resolve-release-metadata.outputs.release_tag }} + release_date_json: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.release_date_json || steps.resolve-release-metadata.outputs.release_date_json }} + source_sha: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_sha || inputs.source_sha }} + # Previous stable tag that bounds the generated release notes range (RC/stable only; empty for nightly). + notes_start_tag: ${{ steps.resolve-release-metadata.outputs.notes_start_tag }} + steps: + # Fetch tags so RC auto-increment can use plain local Git. + - name: Checkout workflow code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 + fetch-tags: true + + # Fail fast on cadence-specific inputs before source lookup or asset planning: + # nightly may pin an exact source SHA, while RC/stable require source SHA and version. + - name: Validate cadence contract + shell: bash + env: + CADENCE: ${{ inputs.cadence }} + SOURCE_SHA: ${{ inputs.source_sha }} + VERSION: ${{ inputs.version }} + RELEASE_DATE: ${{ inputs.release_date }} + run: | + set -euo pipefail + + case "${CADENCE}" in + nightly) + if [[ -n "${VERSION}" || -n "${RELEASE_DATE}" ]]; then + echo "::error::nightly releases must not provide version or release_date inputs" + exit 1 + fi + ;; + rc | release) + if [[ -z "${SOURCE_SHA}" || -z "${VERSION}" ]]; then + echo "::error::rc and stable releases require source_sha and version inputs" + exit 1 + fi + ;; + *) + echo "::error::Unknown release cadence: ${CADENCE}" + exit 1 + ;; + esac + + if [[ -n "${SOURCE_SHA}" && ! "${SOURCE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::error::source_sha must be a full 40-character commit SHA" + exit 1 + fi + + # Nightly defaults to the repository default branch, but may pin an exact source SHA. + - name: Resolve nightly source + id: resolve-nightly-source + if: inputs.cadence == 'nightly' + shell: bash + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + + timestamp="$(date -u +%Y%m%d%H%M%S)" + + if [[ -n "${SOURCE_SHA}" ]]; then + source_ref="${SOURCE_SHA}" + source_sha="${SOURCE_SHA}" + else + [[ -n "${DEFAULT_BRANCH}" ]] || { echo "::error::repository default branch is empty"; exit 1; } + source_ref="refs/heads/${DEFAULT_BRANCH}" + git fetch --no-tags --depth=1 origin "${source_ref}" + source_sha="$(git rev-parse --verify "FETCH_HEAD^{commit}")" + fi + + { + printf 'source_repo=%s\n' "${GITHUB_REPOSITORY}" + printf 'source_ref=%s\n' "${source_ref}" + printf 'source_sha=%s\n' "${source_sha}" + printf 'nightly_timestamp=%s\n' "${timestamp}" + printf 'release_label=nightly-%s\n' "${timestamp}" + printf 'release_tag=\n' + printf 'release_date_json=null\n' + } >>"${GITHUB_OUTPUT}" + + # RC/stable use the selected workflow branch plus manual version inputs: + # compute the release label/tag, auto-increment RC tags, and format release_date. + - name: Resolve RC/stable metadata + id: resolve-release-metadata + if: inputs.cadence != 'nightly' + shell: bash + env: + CADENCE: ${{ inputs.cadence }} + SOURCE_SHA: ${{ inputs.source_sha }} + VERSION: ${{ inputs.version }} + RELEASE_DATE: ${{ inputs.release_date }} + run: | + set -euo pipefail + + if [[ "${GITHUB_REF}" != refs/heads/* ]]; then + echo "::error::rc and stable releases must run from a branch ref" + exit 1 + fi + + if ! git cat-file -e "${SOURCE_SHA}^{commit}" 2>/dev/null; then + echo "::error::source_sha is not a commit in this repository: ${SOURCE_SHA}" + exit 1 + fi + + if ! git merge-base --is-ancestor "${SOURCE_SHA}" "${GITHUB_SHA}"; then + echo "::error::source_sha must be reachable from the selected workflow branch" + exit 1 + fi + + if [[ ! "${VERSION}" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "::error::version must be SemVer core MAJOR.MINOR.PATCH, for example 1.0.0" + exit 1 + fi + + if [[ "${CADENCE}" == "rc" ]]; then + rc_prefix="${VERSION}-rc" + existing_rcs="$(git tag -l "${rc_prefix}*" | grep -E "^${rc_prefix}[0-9]+$" | sort -V || true)" + + if [[ -z "${existing_rcs}" ]]; then + next_rc=0 + else + last_rc="$(echo "${existing_rcs}" | tail -n 1)" + last_rc_number="${last_rc#"${rc_prefix}"}" + next_rc=$((10#${last_rc_number} + 1)) + fi + + release_label="${rc_prefix}${next_rc}" + else + release_label="${VERSION}" + fi + + release_date_json="null" + if [[ -n "${RELEASE_DATE}" ]]; then + if [[ ! "${RELEASE_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then + echo "::error::release_date must use YYYY-MM-DD format" + exit 1 + fi + if [[ "$(date -u -d "${RELEASE_DATE}" +%Y-%m-%d 2>/dev/null)" != "${RELEASE_DATE}" ]]; then + echo "::error::release_date must be a valid calendar date" + exit 1 + fi + release_date_json="\"${RELEASE_DATE}\"" + fi + + # Generated release notes span from the previous stable release so RC and stable both + # show the full delta since the last stable. Empty when no prior stable tag exists. + # Hardened for `set -euo pipefail`: `|| true` covers the no-stable-tags case, and awk + # reads to EOF (no early `exit`/`head`) so `sort` is never killed by SIGPIPE. + notes_start_tag="" + prev_stable="$( + { git tag -l | grep -E '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' || true; printf '%s\n' "${VERSION}"; } \ + | sort -V -u \ + | awk -v v="${VERSION}" '$0==v{print prev} {prev=$0}' + )" + if [[ -n "${prev_stable}" && "${prev_stable}" != "${VERSION}" ]]; then + notes_start_tag="${prev_stable}" + fi + + { + printf 'source_ref=%s\n' "${GITHUB_REF}" + printf 'release_label=%s\n' "${release_label}" + printf 'release_tag=%s\n' "${release_label}" + printf 'release_date_json=%s\n' "${release_date_json}" + printf 'notes_start_tag=%s\n' "${notes_start_tag}" + } >>"${GITHUB_OUTPUT}" + + - name: Checkout selected source + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.cadence == 'nightly' && steps.resolve-nightly-source.outputs.source_sha || inputs.source_sha }} + path: source + + # Plan release artifacts before any release side effects. release_scope: + # all -> every catalog SDK + every catalog container (default) + # sdks -> every catalog SDK, no containers + # containers -> every catalog container, no SDKs + # custom -> exactly sdk_ids + container_ids (either may be empty) + # Container image bits are not built here; container entries are recorded + # in the manifest as typed artifacts and the release consumer stages them + # from the dev registry by this release's source SHA. + - name: Plan release assets + id: plan-assets + shell: bash + env: + RELEASE_SCOPE: ${{ inputs.release_scope }} + SDK_IDS: ${{ inputs.sdk_ids }} + CONTAINER_IDS: ${{ inputs.container_ids }} + run: | + set -euo pipefail + + catalog="source/release/assets.yaml" + [[ -f "${catalog}" ]] || { echo "::error::release asset catalog is missing: ${catalog}"; exit 1; } + + # Read and validate a catalog section ('.sdk' / '.container') into a + # JSON id array: non-empty string ids, no duplicates. + read_catalog() { + local section="$1" kind="$2" ids + ids="$(yq -o=json "${section} // []" "${catalog}" | jq -c 'map(.id)')" + if ! jq -e 'type == "array" and all(.[]; type == "string" and length > 0)' <<<"${ids}" >/dev/null; then + echo "::error::release/assets.yaml ${kind} must contain non-empty string id values" + exit 1 + fi + if ! jq -e 'length == (unique | length)' <<<"${ids}" >/dev/null; then + echo "::error::release/assets.yaml ${kind} contains duplicate ids" + exit 1 + fi + printf '%s' "${ids}" + } + + # Parse a comma-separated custom id list and validate it against its + # catalog: no empty entries, no duplicates, no unknown ids. + parse_custom_ids() { + local raw="$1" kind="$2" catalog_json="$3" parsed unknown + parsed="$(jq -Rnc --arg value "${raw}" '$value | split(",") | map(gsub("^\\s+|\\s+$"; ""))')" + if jq -e 'any(.[]; length == 0)' <<<"${parsed}" >/dev/null; then + echo "::error::${kind}_ids contains an empty entry" + exit 1 + fi + if ! jq -e 'length == (unique | length)' <<<"${parsed}" >/dev/null; then + echo "::error::${kind}_ids contains duplicate entries" + exit 1 + fi + unknown="$(jq -nc --argjson selected "${parsed}" --argjson catalog "${catalog_json}" '$selected - $catalog')" + if [[ "${unknown}" != "[]" ]]; then + echo "::error::unknown ${kind} id(s): $(jq -r 'join(", ")' <<<"${unknown}")" + exit 1 + fi + printf '%s' "${parsed}" + } + + sdk_catalog="$(read_catalog '.sdk' sdk)" + container_catalog="$(read_catalog '.container' container)" + + case "${RELEASE_SCOPE}" in + all | sdks | containers | custom) + ;; + *) + echo "::error::Unknown release_scope: ${RELEASE_SCOPE} (expected all, sdks, containers, or custom)" + exit 1 + ;; + esac + + if [[ "${RELEASE_SCOPE}" != "custom" && ( -n "${SDK_IDS}" || -n "${CONTAINER_IDS}" ) ]]; then + echo "::error::sdk_ids/container_ids can only be used when release_scope is custom" + exit 1 + fi + + case "${RELEASE_SCOPE}" in + all) + selected_sdk_ids="${sdk_catalog}" + selected_container_ids="${container_catalog}" + ;; + sdks) + selected_sdk_ids="${sdk_catalog}" + selected_container_ids='[]' + ;; + containers) + selected_sdk_ids='[]' + selected_container_ids="${container_catalog}" + ;; + custom) + selected_sdk_ids='[]' + selected_container_ids='[]' + [[ -n "${SDK_IDS}" ]] && selected_sdk_ids="$(parse_custom_ids "${SDK_IDS}" sdk "${sdk_catalog}")" + [[ -n "${CONTAINER_IDS}" ]] && selected_container_ids="$(parse_custom_ids "${CONTAINER_IDS}" container "${container_catalog}")" + ;; + esac + + total="$(jq -n --argjson s "${selected_sdk_ids}" --argjson c "${selected_container_ids}" '($s | length) + ($c | length)')" + if [[ "${total}" == "0" ]]; then + echo "::error::release selection is empty; choose a non-empty release_scope or provide sdk_ids/container_ids" + exit 1 + fi + + sdk_artifacts_json="$(jq -nc --argjson ids "${selected_sdk_ids}" '$ids | map({type: "sdk", id: .})')" + container_artifacts_json="$(jq -nc --argjson ids "${selected_container_ids}" '$ids | map({type: "container", id: .})')" + selected_artifacts_json="$(jq -nc --argjson sdks "${sdk_artifacts_json}" --argjson containers "${container_artifacts_json}" '$sdks + $containers')" + sdk_matrix="$(jq -nc --argjson artifacts "${sdk_artifacts_json}" '{include: $artifacts}')" + sdk_count="$(jq -r 'length' <<<"${selected_sdk_ids}")" + container_count="$(jq -r 'length' <<<"${selected_container_ids}")" + + { + printf 'selected_artifacts_json=%s\n' "${selected_artifacts_json}" + printf 'sdk_count=%s\n' "${sdk_count}" + printf 'container_count=%s\n' "${container_count}" + printf 'sdk_matrix=%s\n' "${sdk_matrix}" + } >>"${GITHUB_OUTPUT}" + + echo "Planned SDK release artifacts: $(jq -r 'if length == 0 then "(none)" else join(", ") end' <<<"${selected_sdk_ids}")" + echo "Planned container release artifacts: $(jq -r 'if length == 0 then "(none)" else join(", ") end' <<<"${selected_container_ids}")" + + reserve-release-tag: + name: Reserve release tag + needs: plan-release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout source at release SHA + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ needs.plan-release.outputs.source_sha }} + fetch-tags: true + + # Reserve the tag before build jobs so a tagged RC attempt consumes its RC number. + - name: Create and push tag + shell: bash + env: + CADENCE: ${{ inputs.cadence }} + RELEASE_TAG: ${{ needs.plan-release.outputs.release_tag }} + SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} + run: | + set -euo pipefail + + if [[ "${CADENCE}" == "nightly" ]]; then + echo "Nightly cadence: no release tag to reserve." + exit 0 + fi + + [[ -n "${RELEASE_TAG}" ]] || { echo "::error::release_tag is required"; exit 1; } + [[ -n "${SOURCE_SHA}" ]] || { echo "::error::source_sha is required"; exit 1; } + + if git show-ref --tags --verify --quiet "refs/tags/${RELEASE_TAG}"; then + existing_sha="$(git rev-list -n 1 "refs/tags/${RELEASE_TAG}")" + if [[ "${existing_sha}" == "${SOURCE_SHA}" ]]; then + echo "Release tag '${RELEASE_TAG}' already points at '${SOURCE_SHA}'." + exit 0 + fi + + echo "::error::release tag '${RELEASE_TAG}' already points at '${existing_sha}', not '${SOURCE_SHA}'" + exit 1 + fi + + git tag "${RELEASE_TAG}" + git push origin "refs/tags/${RELEASE_TAG}" + echo "Created release tag '${RELEASE_TAG}' at '${SOURCE_SHA}'." + + build-sdks: + name: Build SDK (${{ matrix.id }}) + needs: [plan-release, reserve-release-tag] + # Skip cleanly for container-only releases (sdk_matrix would be empty). + if: ${{ needs.plan-release.outputs.sdk_count != '0' }} + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.plan-release.outputs.sdk_matrix) }} + steps: + - name: Checkout workflow code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + path: workflow + + - name: Checkout source + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: ${{ needs.plan-release.outputs.source_repo }} + ref: ${{ needs.plan-release.outputs.source_sha }} + path: source + fetch-depth: 0 + fetch-tags: true + + # Toolchain setup + version stamp + `uv build` are factored into the + # composite action, which ci.yaml's wheel-test job also calls so the + # test wheel and the published wheel come out of one code path. The + # action runs out of the workflow checkout above, not the + # release-target source checkout. + - name: Build SDK wheel + id: build-sdk-wheel + uses: ./workflow/.github/actions/build-nemo-platform-wheel + with: + package: ${{ matrix.id }} + source-root: source + out-dir: ${{ github.workspace }}/release-sdk-wheel + cadence: ${{ inputs.cadence }} + release-label: ${{ needs.plan-release.outputs.release_label }} + nightly-timestamp: ${{ needs.plan-release.outputs.nightly_timestamp }} + + - name: Upload SDK wheel + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-sdk-${{ matrix.id }} + path: ${{ steps.build-sdk-wheel.outputs.wheel-path }} + if-no-files-found: error + overwrite: true + retention-days: 1 + + assemble-release-bundle: + name: Assemble release bundle + needs: [plan-release, reserve-release-tag, build-sdks] + # build-sdks is skipped for container-only releases; still assemble as long as + # planning and tag reservation succeeded and the SDK builds did not fail. The + # reserve-release-tag gate is required because build-sdks is also skipped when + # tag creation fails (its dependency failed), so its result alone would let a + # failed tag reservation still build and dispatch the bundle. + if: ${{ !cancelled() && needs.plan-release.result == 'success' && needs.reserve-release-tag.result == 'success' && (needs.build-sdks.result == 'success' || needs.build-sdks.result == 'skipped') }} + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + # Selects the consumer's fetch path: nightly/RC = Actions artifact, stable = GitHub Release asset. + source: ${{ inputs.cadence != 'release' && 'artifact' || 'release' }} + bundle_artifact_id: ${{ steps.upload-release-bundle.outputs.artifact-id }} + bundle_artifact_name: release-bundle-${{ needs.plan-release.outputs.release_label }} + bundle_artifact_digest: ${{ steps.upload-release-bundle.outputs.artifact-digest }} + release_tag: ${{ steps.upload-release.outputs.release_tag }} + release_checksums_digest: ${{ steps.upload-release.outputs.checksums_digest }} + steps: + - name: Checkout workflow code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + # No SDK wheels exist for container-only releases; skip the download + # (the metadata writer handles an empty/absent artifacts dir). + - name: Download SDK wheels + if: ${{ needs.plan-release.outputs.sdk_count != '0' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: release-sdk-* + path: downloaded-sdk-artifacts + merge-multiple: false + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + python-version: "3.11" + + - name: Write release bundle metadata + shell: bash + env: + CADENCE: ${{ inputs.cadence }} + RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} + RELEASE_DATE_JSON: ${{ needs.plan-release.outputs.release_date_json }} + SELECTED_ARTIFACTS_JSON: ${{ needs.plan-release.outputs.selected_artifacts_json }} + SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} + run: | + set -euo pipefail + + uv run --no-project python .github/scripts/write_release_bundle_metadata.py \ + --sdk-artifacts-dir downloaded-sdk-artifacts \ + --bundle-dir release-bundle \ + --selected-artifacts-json "${SELECTED_ARTIFACTS_JSON}" \ + --cadence "${CADENCE}" \ + --release-label "${RELEASE_LABEL}" \ + --release-date-json "${RELEASE_DATE_JSON}" \ + --source-sha "${SOURCE_SHA}" + + - name: Verify release bundle checksums + shell: bash + run: | + set -euo pipefail + + cd release-bundle + sha256sum -c checksums.txt + + # Nightly/RC: hand off via an Actions artifact. RCs still reserve a git tag, but do + # not create a GitHub Release page until the RC release-page flow is settled. + - name: Upload release bundle artifact + id: upload-release-bundle + if: inputs.cadence != 'release' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-bundle-${{ needs.plan-release.outputs.release_label }} + path: release-bundle/ + if-no-files-found: error + overwrite: true + + # Stable: attach each artifact as its own asset on the tag reserve-release-tag pushed, so + # users can grab a single file instead of one opaque blob. + - name: Upload release bundle to release + id: upload-release + if: inputs.cadence == 'release' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.plan-release.outputs.release_tag }} + CADENCE: ${{ inputs.cadence }} + NOTES_START_TAG: ${{ needs.plan-release.outputs.notes_start_tag }} + run: | + set -euo pipefail + + [[ -n "${RELEASE_TAG}" ]] || { echo "::error::release_tag is required for ${CADENCE} releases"; exit 1; } + + # checksums.txt covers release-manifest.json + every wheel, so its digest anchors the + # whole release. The dispatch carries it; the consumer re-verifies with sha256sum -c. + checksums_digest="$(sha256sum release-bundle/checksums.txt | cut -d' ' -f1)" + + # Categorized notes are generated by GitHub from .github/release.yml (merged PRs grouped + # by their conventional-commit label). NOTES_START_TAG pins the range to the previous + # stable tag; when empty (no prior stable) GitHub falls back to its default previous-tag + # detection. + notes_range=() + if [[ -n "${NOTES_START_TAG:-}" ]]; then + notes_range=(--notes-start-tag "${NOTES_START_TAG}") + fi + + # Wheels are absent for container-only releases; nullglob keeps the + # unmatched glob from being passed to gh literally. + shopt -s nullglob + wheels=(release-bundle/wheels/*.whl) + shopt -u nullglob + + # Upload each artifact as its own asset. GitHub flattens asset names to basenames; the + # consumer restores wheels/ from the manifest paths. Re-run model: if a release already + # exists for this tag the command fails ("a release with the same tag name already + # exists"); delete it (gh release delete "${RELEASE_TAG}") to redo. + # Body is GitHub's generated, categorized notes (.github/release.yml); --title keeps the + # tag as the release name so no auto-title is generated. + gh release create "${RELEASE_TAG}" \ + release-bundle/release-manifest.json \ + release-bundle/checksums.txt \ + "${wheels[@]}" \ + --title "${RELEASE_TAG}" \ + --generate-notes \ + "${notes_range[@]}" + + { + printf 'release_tag=%s\n' "${RELEASE_TAG}" + printf 'checksums_digest=%s\n' "${checksums_digest}" + } >>"${GITHUB_OUTPUT}" + + dispatch-release-bundle: + name: Dispatch release bundle + needs: assemble-release-bundle + runs-on: ubuntu-latest + steps: + - name: Send release bundle pointer + shell: bash + env: + GH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }} + DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} + BUNDLE_REPO: ${{ github.repository }} + SOURCE: ${{ needs.assemble-release-bundle.outputs.source }} + BUNDLE_WORKFLOW_RUN_ID: ${{ github.run_id }} + BUNDLE_WORKFLOW_RUN_ATTEMPT: ${{ github.run_attempt }} + BUNDLE_ARTIFACT_ID: ${{ needs.assemble-release-bundle.outputs.bundle_artifact_id }} + BUNDLE_ARTIFACT_NAME: ${{ needs.assemble-release-bundle.outputs.bundle_artifact_name }} + BUNDLE_ARTIFACT_DIGEST: ${{ needs.assemble-release-bundle.outputs.bundle_artifact_digest }} + RELEASE_TAG: ${{ needs.assemble-release-bundle.outputs.release_tag }} + RELEASE_CHECKSUMS_DIGEST: ${{ needs.assemble-release-bundle.outputs.release_checksums_digest }} + SEND_NOTIFICATIONS: ${{ inputs.send_notifications }} + run: | + set -euo pipefail + + [[ -n "${GH_TOKEN:-}" ]] || { echo "::error::CI_DISPATCH_TOKEN must be set"; exit 1; } + [[ -n "${DISPATCH_REPO:-}" ]] || { echo "::error::CI_DISPATCH_REPO must be set"; exit 1; } + [[ -n "${BUNDLE_REPO:-}" ]] || { echo "::error::bundle repo is required"; exit 1; } + + # assemble-release-bundle picks the storage backend by cadence (nightly/RC -> Actions + # artifact, stable -> GitHub Release asset) and reports it as `source`. The consumer uses + # this same field to fail-closed select its fetch path. + case "${SOURCE}" in + artifact) + [[ -n "${BUNDLE_ARTIFACT_ID:-}" ]] || { echo "::error::bundle artifact id is required"; exit 1; } + [[ -n "${BUNDLE_ARTIFACT_NAME:-}" ]] || { echo "::error::bundle artifact name is required"; exit 1; } + [[ -n "${BUNDLE_ARTIFACT_DIGEST:-}" ]] || { echo "::error::bundle artifact digest is required"; exit 1; } + bundle="$(jq -n \ + --arg repo "${BUNDLE_REPO}" \ + --arg workflow_run_id "${BUNDLE_WORKFLOW_RUN_ID}" \ + --arg workflow_run_attempt "${BUNDLE_WORKFLOW_RUN_ATTEMPT}" \ + --arg artifact_id "${BUNDLE_ARTIFACT_ID}" \ + --arg artifact_name "${BUNDLE_ARTIFACT_NAME}" \ + --arg artifact_digest "${BUNDLE_ARTIFACT_DIGEST}" \ + --argjson send_notifications "${SEND_NOTIFICATIONS}" \ + '{ + source: "artifact", + repo: $repo, + workflow_run_id: $workflow_run_id, + workflow_run_attempt: $workflow_run_attempt, + artifact_id: $artifact_id, + artifact_name: $artifact_name, + artifact_digest: $artifact_digest, + send_notifications: $send_notifications + }')" + ;; + release) + [[ -n "${RELEASE_TAG:-}" ]] || { echo "::error::release tag is required"; exit 1; } + [[ -n "${RELEASE_CHECKSUMS_DIGEST:-}" ]] || { echo "::error::release checksums digest is required"; exit 1; } + bundle="$(jq -n \ + --arg repo "${BUNDLE_REPO}" \ + --arg workflow_run_id "${BUNDLE_WORKFLOW_RUN_ID}" \ + --arg tag "${RELEASE_TAG}" \ + --arg checksums_digest "${RELEASE_CHECKSUMS_DIGEST}" \ + --argjson send_notifications "${SEND_NOTIFICATIONS}" \ + '{ + source: "release", + repo: $repo, + workflow_run_id: $workflow_run_id, + tag: $tag, + checksums_digest: $checksums_digest, + send_notifications: $send_notifications + }')" + ;; + *) + echo "::error::unknown bundle source: '${SOURCE}'" + exit 1 + ;; + esac + + jq -n --argjson bundle "${bundle}" \ + '{event_type: "release-bundle-produced", client_payload: {bundle: $bundle}}' \ + | gh api -X POST "/repos/${DISPATCH_REPO}/dispatches" --input - diff --git a/.github/workflows/release-nightly.yaml b/.github/workflows/release-nightly.yaml new file mode 100644 index 0000000000..c47c35a77c --- /dev/null +++ b/.github/workflows/release-nightly.yaml @@ -0,0 +1,59 @@ +name: "Release: Nightly" + +on: + schedule: + # Runs Monday through Friday at 8:00 PM America/Los_Angeles. + - cron: "0 20 * * 1-5" + timezone: "America/Los_Angeles" + workflow_dispatch: + inputs: + source_sha: + description: "Optional exact source commit SHA. Defaults to the repository default branch." + required: false + type: string + default: "" + release_scope: + description: "Release artifact scope." + required: false + type: choice + default: all + options: + - all + - sdks + - containers + - custom + sdk_ids: + description: "Comma-separated SDK IDs for custom releases." + required: false + type: string + default: "" + container_ids: + description: "Comma-separated container IDs for custom releases." + required: false + type: string + default: "" + send_notifications: + description: "Send downstream release notifications." + required: false + type: boolean + default: true + +permissions: + contents: read + +jobs: + release-bundle: + name: Produce release bundle + permissions: + contents: write + uses: ./.github/workflows/release-bundle.yaml + with: + cadence: nightly + source_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.source_sha || '' }} + release_scope: ${{ github.event_name == 'workflow_dispatch' && inputs.release_scope || 'all' }} + sdk_ids: ${{ github.event_name == 'workflow_dispatch' && inputs.sdk_ids || '' }} + container_ids: ${{ github.event_name == 'workflow_dispatch' && inputs.container_ids || '' }} + send_notifications: ${{ github.event_name != 'workflow_dispatch' || inputs.send_notifications }} + secrets: + CI_DISPATCH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }} + CI_DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} diff --git a/.github/workflows/release-rc.yaml b/.github/workflows/release-rc.yaml new file mode 100644 index 0000000000..759d1749d2 --- /dev/null +++ b/.github/workflows/release-rc.yaml @@ -0,0 +1,59 @@ +name: "Release: RC" + +on: + workflow_dispatch: + inputs: + source_sha: + description: "Exact source commit SHA to release from the selected workflow branch." + required: true + type: string + base_version: + description: "Base SemVer core release version, for example 1.0.0." + required: true + type: string + release_date: + description: "Optional. Provide only for first-time registration of this release version. Leave blank for reruns or when this release version is already registered. Format: YYYY-MM-DD." + required: false + type: string + default: "" + release_scope: + description: "Release artifact scope." + required: false + type: choice + default: all + options: + - all + - sdks + - containers + - custom + sdk_ids: + description: "Comma-separated SDK IDs for custom releases." + required: false + type: string + default: "" + container_ids: + description: "Comma-separated container IDs for custom releases." + required: false + type: string + default: "" + +permissions: + contents: read + +jobs: + release-bundle: + name: Produce release bundle + uses: ./.github/workflows/release-bundle.yaml + permissions: + contents: write + with: + cadence: rc + source_sha: ${{ inputs.source_sha }} + version: ${{ inputs.base_version }} + release_date: ${{ inputs.release_date }} + release_scope: ${{ inputs.release_scope }} + sdk_ids: ${{ inputs.sdk_ids }} + container_ids: ${{ inputs.container_ids }} + secrets: + CI_DISPATCH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }} + CI_DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} diff --git a/.github/workflows/release-stable.yaml b/.github/workflows/release-stable.yaml new file mode 100644 index 0000000000..93b2c27e2c --- /dev/null +++ b/.github/workflows/release-stable.yaml @@ -0,0 +1,116 @@ +name: "Release: Stable" +run-name: Stable release ${{ inputs.version }} from ${{ inputs.source_sha }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + source_sha: + description: "Exact source commit SHA to release from the selected workflow branch." + required: true + type: string + version: + description: "Stable SemVer core release version, for example 1.0.0." + required: true + type: string + release_date: + description: "Optional. Provide only for first-time registration of this release version. Leave blank for reruns or when this release version is already registered. Format: YYYY-MM-DD." + required: false + type: string + default: "" + release_scope: + description: "Release artifact scope." + required: false + type: choice + default: all + options: + - all + - sdks + - containers + - custom + sdk_ids: + description: "Comma-separated SDK IDs for custom releases." + required: false + type: string + default: "" + container_ids: + description: "Comma-separated container IDs for custom releases." + required: false + type: string + default: "" + +concurrency: + group: release-stable + cancel-in-progress: false + +permissions: + contents: read + +jobs: + preview-stable-release: + name: Preview stable release ${{ inputs.version }} from ${{ inputs.source_sha }} + runs-on: ubuntu-latest + steps: + - name: Summarize stable release request + shell: bash + env: + VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ inputs.source_sha }} + RELEASE_DATE: ${{ inputs.release_date }} + RELEASE_SCOPE: ${{ inputs.release_scope }} + SDK_IDS: ${{ inputs.sdk_ids }} + CONTAINER_IDS: ${{ inputs.container_ids }} + run: | + set -euo pipefail + + source_link="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${SOURCE_SHA}" + + { + echo "## Stable release approval preview" + echo + echo "| Field | Value |" + echo "| --- | --- |" + echo "| Version | ${VERSION} |" + echo "| Stable tag | ${VERSION} |" + echo "| Source SHA | [${SOURCE_SHA}](${source_link}) |" + echo "| Workflow branch | ${GITHUB_REF_NAME} |" + echo "| Release date | ${RELEASE_DATE:-not provided} |" + echo "| Release scope | ${RELEASE_SCOPE} |" + if [[ "${RELEASE_SCOPE}" == "custom" ]]; then + echo "| SDK IDs | ${SDK_IDS:-not provided} |" + echo "| Container IDs | ${CONTAINER_IDS:-not provided} |" + fi + echo "| Destination | Stable git tag, release bundle artifact, downstream \`release-bundle-produced\` dispatch |" + echo + echo "This preview is informational only. \`release-bundle.yaml\` still performs the authoritative validation before creating tags, building bundles, or dispatching downstream." + } >>"${GITHUB_STEP_SUMMARY}" + + approve-stable-release: + name: Approve public stable ${{ inputs.version }} from ${{ inputs.source_sha }} + needs: preview-stable-release + runs-on: ubuntu-latest + environment: release-stable + steps: + - name: Record stable release approval + run: | + echo "release-stable approval passed for ${VERSION} from ${SOURCE_SHA}." + env: + VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ inputs.source_sha }} + + release-bundle: + name: Produce release bundle + needs: approve-stable-release + uses: ./.github/workflows/release-bundle.yaml + permissions: + contents: write + with: + cadence: release + source_sha: ${{ inputs.source_sha }} + version: ${{ inputs.version }} + release_date: ${{ inputs.release_date }} + release_scope: ${{ inputs.release_scope }} + sdk_ids: ${{ inputs.sdk_ids }} + container_ids: ${{ inputs.container_ids }} + secrets: + CI_DISPATCH_TOKEN: ${{ secrets.CI_DISPATCH_TOKEN }} + CI_DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 350a37e5f5..0000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,1166 +0,0 @@ -name: Release -run-name: >- - Release ${{ github.event_name == 'schedule' && 'nightly' || inputs['release-type'] }} - by @${{ github.actor }} - -on: - schedule: - # Runs Monday through Friday at 8:00 PM America/Los_Angeles. - - cron: "0 20 * * 1-5" - timezone: "America/Los_Angeles" - workflow_dispatch: - inputs: - release-type: - description: "Release type." - required: false - type: choice - default: nightly - options: - - nightly - - stable - source-sha: - description: "Exact source commit SHA. Required for stable releases; optional for nightlies." - required: false - type: string - default: "" - version: - description: "Stable SemVer core release version, for example 1.0.0." - required: false - type: string - default: "" - release-scope: - description: "Release artifact scope." - required: false - type: choice - default: all - options: - - all - - wheels - - containers - - helm - - custom - wheel-ids: - description: >- - Custom only. Comma-separated wheel IDs. - Allowed: nemo-platform, nemo-platform-plugin. - required: false - type: string - default: "" - container-ids: - description: >- - Custom only. Comma-separated container IDs. - Allowed: nmp-api, nmp-cpu-tasks, nmp-automodel-tasks, - nmp-automodel-training, nmp-unsloth-training, auditor-tasks, - safe-synthesizer-tasks. - required: false - type: string - default: "" - include-helm: - description: "Include the Helm chart if defining a strict list of release artifacts instead of the default (release everything)." - required: false - type: boolean - default: false - update-ngc-metadata: - description: "Synchronize NGC metadata from the this branch." - required: false - type: boolean - default: false - send-notifications: - description: "Send release start and end notifications." - required: false - type: boolean - default: true - dry-run: - description: >- - Validate and package selected artifacts without publishing, dispatching, - polling, deploying, or notifying. - required: false - type: boolean - default: false - -concurrency: - group: release-${{ inputs['dry-run'] && github.run_id || 'live' }} - cancel-in-progress: false - -permissions: - contents: read - -env: - # Releasable artifact catalog. - # - # Keep this in the workflow so contributors can see release intent without - # chasing a second config file. When this changes, update the matching - # workflow_dispatch input descriptions above; GitHub does not support dynamic - # descriptions in the Run workflow form. - # - # Wheel fields: - # id - value users type in wheel-ids and the release artifact id - # package - package passed to uv build --package - # path - package directory checked before any later release work starts - RELEASE_WHEELS_JSON: >- - [ - {"id":"nemo-platform","package":"nemo-platform","path":"packages/nemo_platform"}, - {"id":"nemo-platform-plugin","package":"nemo-platform-plugin","path":"packages/nemo_platform_plugin"} - ] - # Container fields: - # id - value users type in container-ids and the published image name - # target - docker buildx bake target that must exist in docker-bake.hcl - # - # Every container id must also have .github/assets/ngc/containers/.md so - # the published image has matching NGC catalog metadata. - RELEASE_CONTAINERS_JSON: >- - [ - {"id":"nmp-api","target":"nmp-api-docker"}, - {"id":"nmp-cpu-tasks","target":"nmp-cpu-tasks-docker"}, - {"id":"nmp-automodel-tasks","target":"nmp-automodel-tasks-docker"}, - {"id":"nmp-automodel-training","target":"nmp-automodel-training-docker"}, - {"id":"nmp-unsloth-training","target":"nmp-unsloth-training"}, - {"id":"auditor-tasks","target":"auditor-tasks-docker"}, - {"id":"safe-synthesizer-tasks","target":"safe-synthesizer-tasks-docker"} - ] - RELEASE_HELM_ID: nemo-platform - RELEASE_HELM_PATH: k8s/helm - RELEASE_HELM_REGISTRY: nvcr.io/0921617854601259/nemo-platform - RELEASE_NIGHTLY_WHEEL_INDEX: https://pypi.nvidia.com - RELEASE_STABLE_WHEEL_INDEX: https://pypi.org/simple - RELEASE_NIGHTLY_CONTAINER_REGISTRY: ghcr.io/nvidia-nemo/nemo-platform - RELEASE_STABLE_CONTAINER_REGISTRY: nvcr.io/nvidia/nemo-platform - # Make this package public in GitHub Packages after its first push. - RELEASE_NIGHTLY_HELM_OCI_REGISTRY: oci://ghcr.io/nvidia-nemo/nemo-platform - # Stable charts must be promoted from RELEASE_HELM_REGISTRY before polling. - RELEASE_STABLE_HELM_REPOSITORY: https://helm.ngc.nvidia.com/nvidia/nemo-platform - RELEASE_NGC_CATALOG_BASE: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-platform - -jobs: - plan-release: - name: Plan and validate release inputs - runs-on: ubuntu-latest - outputs: - release_type: ${{ steps.plan.outputs.release_type }} - release_scope: ${{ steps.plan.outputs.release_scope }} - source_sha: ${{ steps.plan.outputs.source_sha }} - version: ${{ steps.plan.outputs.version }} - release_label: ${{ steps.plan.outputs.release_label }} - nightly_timestamp: ${{ steps.plan.outputs.nightly_timestamp }} - wheel_version: ${{ steps.wheel-version.outputs.wheel_version }} - wheel_ids: ${{ steps.plan.outputs.wheel_ids }} - container_ids: ${{ steps.plan.outputs.container_ids }} - has_wheels: ${{ steps.plan.outputs.has_wheels }} - has_containers: ${{ steps.plan.outputs.has_containers }} - include_helm: ${{ steps.plan.outputs.include_helm }} - update_ngc_metadata: ${{ steps.plan.outputs.update_ngc_metadata }} - send_notifications: ${{ steps.plan.outputs.send_notifications }} - dry_run: ${{ steps.plan.outputs.dry_run }} - steps: - # This job is intentionally the first gate. It resolves the user's - # requested artifacts, checks custom ids against the inline catalog, then - # validates the selected source tree has the wheel package paths, bake - # targets, and NGC container overview files needed by later jobs. - - name: Resolve release plan - id: plan - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ github.token }} - script: | - const allWheels = JSON.parse(process.env.RELEASE_WHEELS_JSON); - const allContainers = JSON.parse(process.env.RELEASE_CONTAINERS_JSON); - const allWheelIds = allWheels.map((wheel) => wheel.id); - const allContainerIds = allContainers.map((container) => container.id); - const inputs = context.payload.inputs ?? {}; - const isManual = context.eventName === "workflow_dispatch"; - const releaseType = isManual ? inputs["release-type"] : "nightly"; - const releaseScope = isManual ? (inputs["release-scope"] || "all") : "all"; - const updateNgcMetadata = isManual && inputs["update-ngc-metadata"] === "true"; - const sendNotifications = inputs["send-notifications"] !== "false"; - const dryRun = isManual && inputs["dry-run"] === "true"; - - let sourceSha = isManual ? (inputs["source-sha"] ?? "").trim() : context.sha; - const version = releaseType === "stable" ? (inputs.version ?? "").trim() : ""; - - if (releaseType === "stable") { - if (!/^[0-9a-f]{40}$/i.test(sourceSha)) { - throw new Error("Stable releases require an exact 40-character source SHA."); - } - if (!/^\d+\.\d+\.\d+$/.test(version)) { - throw new Error("Stable releases require a MAJOR.MINOR.PATCH version."); - } - } else { - if (sourceSha && !/^[0-9a-f]{40}$/i.test(sourceSha)) { - throw new Error("A pinned nightly source must be an exact 40-character SHA."); - } - if (!sourceSha && dryRun) { - sourceSha = context.sha; - } else if (!sourceSha) { - const defaultBranch = context.payload.repository.default_branch; - const { data: commit } = await github.rest.repos.getCommit({ - ...context.repo, - ref: defaultBranch, - }); - sourceSha = commit.sha; - } - } - - sourceSha = sourceSha.toLowerCase(); - - const presets = { - all: { wheels: allWheels, containers: allContainers, includeHelm: true }, - wheels: { wheels: allWheels, containers: [], includeHelm: false }, - containers: { wheels: [], containers: allContainers, includeHelm: false }, - helm: { wheels: [], containers: [], includeHelm: true }, - }; - let selection = presets[releaseScope]; - - if (releaseScope === "custom") { - const selectArtifacts = (value, allowedArtifacts, allowedIds, label, inputName) => { - if (!value.trim()) { - return []; - } - const requestedList = value.split(",").map((id) => id.trim()); - const emptyEntry = requestedList.some((id) => id.length === 0); - if (emptyEntry) { - throw new Error(`${inputName} contains an empty entry.`); - } - const requestedIds = new Set(requestedList); - if (requestedIds.size !== requestedList.length) { - throw new Error(`${inputName} contains duplicate entries.`); - } - const unknownIds = [...requestedIds].filter( - (id) => !allowedIds.includes(id), - ); - if (unknownIds.length > 0) { - throw new Error( - `Unknown ${label} IDs: ${unknownIds.join(", ")}. ` - + `Allowed ${label} IDs: ${allowedIds.join(", ")}.`, - ); - } - return allowedArtifacts.filter((artifact) => requestedIds.has(artifact.id)); - }; - - selection = { - wheels: selectArtifacts( - inputs["wheel-ids"] || "", - allWheels, - allWheelIds, - "wheel", - "wheel-ids", - ), - containers: selectArtifacts( - inputs["container-ids"] || "", - allContainers, - allContainerIds, - "container", - "container-ids", - ), - includeHelm: inputs["include-helm"] === "true", - }; - - if ( - selection.wheels.length === 0 - && selection.containers.length === 0 - && !selection.includeHelm - ) { - throw new Error("A custom release must select at least one artifact."); - } - } - - if (!selection) { - throw new Error(`Unknown release scope: ${releaseScope}.`); - } - - const { wheels, containers, includeHelm } = selection; - const wheelIds = wheels.map((wheel) => wheel.id); - const containerIds = containers.map((container) => container.id); - - const nightlyTimestamp = releaseType === "nightly" - ? new Date().toISOString().replace(/\D/g, "").slice(0, 14) - : ""; - const releaseLabel = releaseType === "nightly" - ? `nightly-${nightlyTimestamp}` - : version; - - core.setOutput("release_type", releaseType); - core.setOutput("release_scope", releaseScope); - core.setOutput("source_sha", sourceSha); - core.setOutput("version", version); - core.setOutput("release_label", releaseLabel); - core.setOutput("nightly_timestamp", nightlyTimestamp); - core.setOutput("wheel_ids", JSON.stringify(wheelIds)); - core.setOutput("container_ids", JSON.stringify(containerIds)); - core.setOutput("wheel_artifacts", JSON.stringify(wheels)); - core.setOutput("container_artifacts", JSON.stringify(containers)); - core.setOutput("has_wheels", String(wheelIds.length > 0)); - core.setOutput("has_containers", String(containerIds.length > 0)); - core.setOutput("include_helm", String(includeHelm)); - core.setOutput("update_ngc_metadata", String(updateNgcMetadata)); - core.setOutput("send_notifications", String(sendNotifications)); - core.setOutput("dry_run", String(dryRun)); - - core.info( - `Planned ${releaseType} release ${releaseLabel} from ${sourceSha}`, - ); - - - name: Checkout selected source - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ steps.plan.outputs.source_sha }} - fetch-depth: 0 - fetch-tags: true - persist-credentials: false - - - name: Ensure selected source is an ancestor of this workflow revision - shell: bash - env: - SOURCE_SHA: ${{ steps.plan.outputs.source_sha }} - run: git merge-base --is-ancestor "${SOURCE_SHA}" "${GITHUB_SHA}" - - - name: Validate selected release artifacts - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - WHEEL_ARTIFACTS: ${{ steps.plan.outputs.wheel_artifacts }} - CONTAINER_ARTIFACTS: ${{ steps.plan.outputs.container_artifacts }} - with: - script: | - const fs = require("fs"); - const path = require("path"); - - const parseArtifacts = (envName) => JSON.parse(process.env[envName] || "[]"); - const readText = (filePath) => fs.readFileSync(filePath, "utf8"); - const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - const requireFile = (filePath, message) => { - if (!fs.existsSync(filePath)) { - throw new Error(`${message}: ${filePath}`); - } - }; - - const artifactList = (artifacts) => { - if (artifacts.length === 0) { - return "(none)"; - } - return artifacts.map((artifact) => artifact.id).join(", "); - }; - - const validateWheel = (wheel) => { - const pyprojectPath = path.join(wheel.path, "pyproject.toml"); - requireFile( - pyprojectPath, - `Wheel ${wheel.id} cannot be built because its package config is missing`, - ); - const pyproject = readText(pyprojectPath); - const namePattern = new RegExp( - `^\\s*name\\s*=\\s*["']${escapeRegExp(wheel.package)}["']\\s*$`, - "m", - ); - - if (!namePattern.test(pyproject)) { - throw new Error( - `Wheel ${wheel.id} expects package ${wheel.package}, ` - + `but ${pyprojectPath} does not declare that project name.`, - ); - } - }; - - const readBakeTargets = (bakePath) => { - requireFile(bakePath, "Container validation needs docker-bake.hcl"); - return new Set( - [...readText(bakePath).matchAll(/^target\s+"([^"]+)"/gm)] - .map((match) => match[1]), - ); - }; - - const ngcMetadataPath = (container) => path.join( - ".github", - "assets", - "ngc", - "containers", - `${container.id}.md`, - ); - - const validateContainer = (container, bakeTargets) => { - const bakePath = "docker-bake.hcl"; - if (!bakeTargets.has(container.target)) { - throw new Error( - `Container ${container.id} cannot be built because bake target ` - + `${container.target} is missing from ${bakePath}.`, - ); - } - requireFile( - ngcMetadataPath(container), - `Container ${container.id} is missing matching NGC metadata`, - ); - }; - - const writeSummary = async (wheels, containers) => { - const rows = [ - [ - {data: "Type", header: true}, - {data: "Selected", header: true}, - ], - ["Wheels", artifactList(wheels)], - ["Containers", artifactList(containers)], - ]; - - await core.summary - .addHeading("Release input validation") - .addTable(rows) - .write(); - }; - - const wheels = parseArtifacts("WHEEL_ARTIFACTS"); - const containers = parseArtifacts("CONTAINER_ARTIFACTS"); - const bakeTargets = readBakeTargets("docker-bake.hcl"); - - wheels.forEach(validateWheel); - containers.forEach((container) => validateContainer(container, bakeTargets)); - - await writeSummary(wheels, containers); - - core.info("Selected release artifacts are valid for the checked-out source."); - - - name: Resolve wheel version - id: wheel-version - if: steps.plan.outputs.has_wheels == 'true' - shell: bash - env: - WHEEL_IDS: ${{ steps.plan.outputs.wheel_ids }} - RELEASE_TYPE: ${{ steps.plan.outputs.release_type }} - RELEASE_VERSION: ${{ steps.plan.outputs.version }} - NIGHTLY_TIMESTAMP: ${{ steps.plan.outputs.nightly_timestamp }} - run: | - set -euo pipefail - - if [[ "${RELEASE_TYPE}" == "nightly" ]]; then - cadence="nightly" - else - cadence="release" - fi - - args=( - --source-root . - --sdk-id "$(jq -r '.[0]' <<< "${WHEEL_IDS}")" - --cadence "${cadence}" - --nightly-timestamp "${NIGHTLY_TIMESTAMP}" - --print-version - ) - if [[ "${RELEASE_TYPE}" == "stable" ]]; then - args+=(--release-label "${RELEASE_VERSION}") - fi - wheel_version="$(python3 .github/scripts/stamp_sdk_version.py "${args[@]}")" - echo "wheel_version=${wheel_version}" >> "${GITHUB_OUTPUT}" - - notify-start: - # Start alerts intentionally run for dry runs so the webhook can be tested. - name: Notify release start - needs: plan-release - if: needs.plan-release.outputs.send_notifications == 'true' - runs-on: ubuntu-latest - steps: - - name: Send Slack alert - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - SLACK_ALERTS_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }} - CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }} - INCLUDE_HELM: ${{ needs.plan-release.outputs.include_helm }} - DRY_RUN: ${{ needs.plan-release.outputs.dry_run }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.plan-release.outputs.source_sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUN_NUMBER: ${{ github.run_number }} - with: - script: | - const wheels = JSON.parse(process.env.WHEEL_IDS); - const containers = JSON.parse(process.env.CONTAINER_IDS); - const releaseType = process.env.RELEASE_TYPE; - const title = releaseType === "stable" - ? "*:ship: Release started*" - : "*:crescent_moon: Nightly release started*"; - const lines = [ - title, - `Release: ${process.env.RELEASE_LABEL}`, - `Commit: <${process.env.COMMIT_URL}|${process.env.SOURCE_SHA.slice(0, 7)}>`, - "", - "*Artifacts:*", - ]; - if (wheels.length > 0) { - lines.push("*:python: Wheels to publish:*"); - wheels.forEach((wheel) => lines.push(`- ${wheel}`)); - } - if (containers.length > 0) { - lines.push("*:docker_: Containers to publish:*"); - containers.forEach((container) => lines.push(`- ${container}`)); - } - if (process.env.INCLUDE_HELM === "true") { - lines.push("*:helm: Helm chart to publish:*", "- nemo-platform"); - } - lines.push( - process.env.DRY_RUN === "true" && "Mode: dry run (no publishing)", - `:link: <${process.env.RUN_URL}|Release run #${process.env.RUN_NUMBER}>`, - ); - const response = await fetch(process.env.SLACK_ALERTS_WEBHOOK, { - method: "POST", - headers: {"Content-Type": "application/json"}, - body: JSON.stringify({text: lines.filter((line) => line !== false).join("\n")}), - }); - if (!response.ok) { - core.setFailed(`Slack webhook returned ${response.status}.`); - } - - sync-ngc-metadata: - name: Synchronize NGC metadata - needs: plan-release - if: >- - needs.plan-release.outputs.update_ngc_metadata == 'true' && - needs.plan-release.outputs.dry_run != 'true' - uses: ./.github/workflows/ngc-metadata.yaml - secrets: - AIRE_NGC_GITHUB_PLATFORM_RW: ${{ secrets.AIRE_NGC_GITHUB_PLATFORM_RW }} - - dispatch-release-registration: - name: Dispatch release registration - needs: plan-release - if: >- - needs.plan-release.outputs.release_type == 'stable' && - needs.plan-release.outputs.dry_run != 'true' - runs-on: ubuntu-latest - steps: - - name: Dispatch registration workflow - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} - RELEASE_VERSION: ${{ needs.plan-release.outputs.version }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - SOURCE_RUN_URL: >- - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }} - CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }} - INCLUDE_HELM: ${{ needs.plan-release.outputs.include_helm }} - with: - github-token: ${{ secrets.CI_DISPATCH_TOKEN }} - script: | - const [owner, repo] = process.env.DISPATCH_REPO.split("/"); - - await github.rest.repos.createDispatchEvent({ - owner, - repo, - event_type: "register-release-artifacts", - client_payload: { - version: process.env.RELEASE_VERSION, - source_sha: process.env.SOURCE_SHA, - source_run_url: process.env.SOURCE_RUN_URL, - wheel_ids: JSON.parse(process.env.WHEEL_IDS), - container_ids: JSON.parse(process.env.CONTAINER_IDS), - helm_id: process.env.INCLUDE_HELM === "true" - ? process.env.RELEASE_HELM_ID - : null, - }, - }); - - dispatch-wheel-stage: - name: Dispatch wheel builds - needs: plan-release - if: >- - needs.plan-release.outputs.has_wheels == 'true' && - needs.plan-release.outputs.dry_run != 'true' - runs-on: ubuntu-latest - steps: - - name: Dispatch wheel builds - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - NIGHTLY_TIMESTAMP: ${{ needs.plan-release.outputs.nightly_timestamp }} - WHEEL_VERSION: ${{ needs.plan-release.outputs.wheel_version }} - WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }} - with: - github-token: ${{ secrets.CI_DISPATCH_TOKEN }} - script: | - const [owner, repo] = process.env.DISPATCH_REPO.split("/"); - - await github.rest.repos.createDispatchEvent({ - owner, - repo, - event_type: "stage-wheels", - client_payload: { - ref: process.env.SOURCE_SHA, - cadence: process.env.RELEASE_TYPE === "stable" ? "release" : "nightly", - release_label: process.env.RELEASE_LABEL, - nightly_timestamp: process.env.NIGHTLY_TIMESTAMP, - wheel_version: process.env.WHEEL_VERSION, - wheels: JSON.parse(process.env.WHEEL_IDS), - }, - }); - - dispatch-container-stage: - name: Dispatch container builds - needs: plan-release - if: >- - needs.plan-release.outputs.has_containers == 'true' && - needs.plan-release.outputs.dry_run != 'true' - runs-on: ubuntu-latest - steps: - - name: Dispatch container builds - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }} - with: - github-token: ${{ secrets.CI_DISPATCH_TOKEN }} - script: | - const [owner, repo] = process.env.DISPATCH_REPO.split("/"); - - await github.rest.repos.createDispatchEvent({ - owner, - repo, - event_type: "release", - client_payload: { - ref: process.env.SOURCE_SHA, - cadence: process.env.RELEASE_TYPE, - version: process.env.RELEASE_LABEL, - containers: JSON.parse(process.env.CONTAINER_IDS), - }, - }); - - stage-helm: - name: Stage Helm chart - needs: plan-release - if: needs.plan-release.outputs.include_helm == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - chart_version: ${{ steps.package.outputs.chart_version }} - permissions: - contents: read - packages: write - steps: - - name: Check out selected source - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ needs.plan-release.outputs.source_sha }} - persist-credentials: false - - - name: Set up Helm - uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 - with: - version: v4.2.1 - - - name: Install Helm push plugin - if: >- - needs.plan-release.outputs.release_type == 'stable' && - needs.plan-release.outputs.dry_run != 'true' - run: >- - helm plugin install https://github.com/chartmuseum/helm-push.git - --version v0.11.1 --verify=false - - - name: Package Helm chart - id: package - shell: bash - env: - HELM_CHART: ${{ env.RELEASE_HELM_PATH }} - NIGHTLY_IMAGE_REGISTRY: ${{ env.RELEASE_NIGHTLY_CONTAINER_REGISTRY }} - STABLE_IMAGE_REGISTRY: ${{ env.RELEASE_HELM_REGISTRY }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_VERSION: ${{ needs.plan-release.outputs.version }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - NIGHTLY_TIMESTAMP: ${{ needs.plan-release.outputs.nightly_timestamp }} - run: | - set -euo pipefail - - chart_dir="$(mktemp -d)" - package_dir="${RUNNER_TEMP}/helm-package" - trap 'rm -rf "${chart_dir}"' EXIT - cp -R "${HELM_CHART}/." "${chart_dir}/" - - helm repo add nvidia https://helm.ngc.nvidia.com/nvidia - helm dependency build "${chart_dir}" - - chart_version="$(yq -r '.version' "${chart_dir}/Chart.yaml")" - if [[ "${RELEASE_TYPE}" == "nightly" ]]; then - chart_version="${chart_version}-nightly-${NIGHTLY_TIMESTAMP}" - release_registry="${NIGHTLY_IMAGE_REGISTRY}" - else - # TODO: Decide whether stable chart versions should track the platform - # release version or remain independently managed in Chart.yaml. - chart_version="${RELEASE_VERSION}" - release_registry="${STABLE_IMAGE_REGISTRY}" - fi - - yq -i ".platformConfig.platform.image_registry = \"${release_registry}\"" "${chart_dir}/values.yaml" - yq -i ".api.image.repository = \"${release_registry}/nmp-api\"" "${chart_dir}/values.yaml" - yq -i ".core.image.repository = \"${release_registry}/nmp-api\"" "${chart_dir}/values.yaml" - - mkdir -p "${package_dir}" - helm package "${chart_dir}" \ - --version "${chart_version}" \ - --app-version "${RELEASE_LABEL}" \ - --destination "${package_dir}" - - echo "chart_version=${chart_version}" >> "${GITHUB_OUTPUT}" - echo "chart_package=${package_dir}/nemo-platform-${chart_version}.tgz" >> "${GITHUB_OUTPUT}" - - - name: Report packaged Helm chart - if: needs.plan-release.outputs.dry_run == 'true' - env: - CHART_PACKAGE: ${{ steps.package.outputs.chart_package }} - run: | - echo "::notice::Dry run: packaged ${CHART_PACKAGE}" - - - name: Push nightly Helm chart to GHCR - if: >- - needs.plan-release.outputs.release_type == 'nightly' && - needs.plan-release.outputs.dry_run != 'true' - shell: bash - env: - GHCR_TOKEN: ${{ github.token }} - HELM_OCI_REGISTRY: ${{ env.RELEASE_NIGHTLY_HELM_OCI_REGISTRY }} - CHART_PACKAGE: ${{ steps.package.outputs.chart_package }} - run: | - set -euo pipefail - - printf '%s' "${GHCR_TOKEN}" | helm registry login ghcr.io \ - --username "${GITHUB_ACTOR}" --password-stdin - helm push "${CHART_PACKAGE}" "${HELM_OCI_REGISTRY}" - - - name: Push stable Helm chart to NGC - if: >- - needs.plan-release.outputs.release_type == 'stable' && - needs.plan-release.outputs.dry_run != 'true' - shell: bash - env: - HELM_PASSWORD: ${{ secrets.AIRE_NVCR_GITHUB }} - RELEASE_REGISTRY: ${{ env.RELEASE_HELM_REGISTRY }} - CHART_VERSION: ${{ steps.package.outputs.chart_version }} - CHART_PACKAGE: ${{ steps.package.outputs.chart_package }} - run: | - set -euo pipefail - - helm_repository="https://helm.ngc.nvidia.com/${RELEASE_REGISTRY#nvcr.io/}" - printf '%s' "${HELM_PASSWORD}" | helm repo add nemo-platform "${helm_repository}" \ - --username "\$oauthtoken" --password-stdin - - if helm search repo nemo-platform/nemo-platform --devel --version "${CHART_VERSION}" \ - | grep -qv 'No results found'; then - echo "Helm chart version ${CHART_VERSION} already exists. Skipping." - else - helm cm-push "${CHART_PACKAGE}" nemo-platform - fi - - dispatch-release-deployment: - name: Dispatch release deployment - needs: [plan-release, stage-helm, poll-final-release] - if: >- - needs.plan-release.outputs.include_helm == 'true' && - needs.plan-release.outputs.dry_run != 'true' - runs-on: ubuntu-latest - steps: - - name: Dispatch deployment creation - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DISPATCH_REPO: ${{ secrets.CI_DISPATCH_REPO }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - HELM_VERSION: ${{ needs.stage-helm.outputs.chart_version }} - with: - github-token: ${{ secrets.CI_DISPATCH_TOKEN }} - script: | - const [owner, repo] = process.env.DISPATCH_REPO.split("/"); - - await github.rest.repos.createDispatchEvent({ - owner, - repo, - event_type: "create-release-deployment", - client_payload: { - ref: process.env.SOURCE_SHA, - cadence: process.env.RELEASE_TYPE === "stable" ? "release" : "nightly", - release_label: process.env.RELEASE_LABEL, - helm_version: process.env.HELM_VERSION, - }, - }); - - poll-final-release: - name: Wait for published release artifacts - needs: - - plan-release - - dispatch-release-registration - - dispatch-wheel-stage - - dispatch-container-stage - - stage-helm - if: >- - !cancelled() && - needs.plan-release.result == 'success' && - !contains(needs.*.result, 'failure') && - !contains(needs.*.result, 'cancelled') - runs-on: ubuntu-latest - timeout-minutes: 240 - permissions: - contents: read - env: - POLL_INTERVAL_SECONDS: "30" - steps: - - name: Skip final artifact polling - if: needs.plan-release.outputs.dry_run == 'true' - run: | - echo "::notice::Dry run: skipped final artifact polling" - - - name: Set up Helm for chart polling - if: >- - needs.plan-release.outputs.include_helm == 'true' && - needs.plan-release.outputs.dry_run != 'true' - uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 - with: - version: v4.2.1 - - - name: Wait for selected wheels in PyPI - if: >- - needs.plan-release.outputs.has_wheels == 'true' && - needs.plan-release.outputs.dry_run != 'true' - shell: bash - env: - WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }} - WHEEL_CATALOG: ${{ env.RELEASE_WHEELS_JSON }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - WHEEL_VERSION: ${{ needs.plan-release.outputs.wheel_version }} - run: | - set -euo pipefail - - if [[ "${RELEASE_TYPE}" == "nightly" ]]; then - wheel_index="${RELEASE_NIGHTLY_WHEEL_INDEX}" - else - wheel_index="${RELEASE_STABLE_WHEEL_INDEX}" - fi - - while IFS= read -r wheel_id; do - package="$(jq -r --arg id "${wheel_id}" '.[] | select(.id == $id) | .package' <<< "${WHEEL_CATALOG}")" - filename_prefix="${package//-/_}-${WHEEL_VERSION}" - wheel_url="${wheel_index}/${package}/" - - until curl --silent --location "${wheel_url}" | grep -Fq "${filename_prefix}"; do - echo "Waiting for ${package}==${WHEEL_VERSION} at ${wheel_url}" - sleep "${POLL_INTERVAL_SECONDS}" - done - echo "Found ${package}==${WHEEL_VERSION}" - done < <(jq -r '.[]' <<< "${WHEEL_IDS}") - - # Both final container registries are public, so this check deliberately - # uses docker manifest inspect without credentials. - - name: Wait for selected containers in the final registry - if: >- - needs.plan-release.outputs.has_containers == 'true' && - needs.plan-release.outputs.dry_run != 'true' - shell: bash - env: - CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }} - CONTAINER_TAG: ${{ needs.plan-release.outputs.release_label }} - CONTAINER_REGISTRY: >- - ${{ needs.plan-release.outputs.release_type == 'nightly' && - env.RELEASE_NIGHTLY_CONTAINER_REGISTRY || - env.RELEASE_STABLE_CONTAINER_REGISTRY }} - run: | - set -euo pipefail - - while IFS= read -r container_id; do - ref="${CONTAINER_REGISTRY}/${container_id}:${CONTAINER_TAG}" - until docker manifest inspect "${ref}" >/dev/null 2>&1; do - echo "Waiting for ${ref}" - sleep "${POLL_INTERVAL_SECONDS}" - done - echo "Found ${ref}" - done < <(jq -r '.[]' <<< "${CONTAINER_IDS}") - - # Both final Helm chart locations are public, so this check deliberately - # uses Helm without registry credentials. - - name: Wait for Helm chart in the final registry - if: >- - needs.plan-release.outputs.include_helm == 'true' && - needs.plan-release.outputs.dry_run != 'true' - shell: bash - env: - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - CHART_ID: ${{ env.RELEASE_HELM_ID }} - CHART_VERSION: ${{ needs.stage-helm.outputs.chart_version }} - NIGHTLY_HELM_OCI_REGISTRY: ${{ env.RELEASE_NIGHTLY_HELM_OCI_REGISTRY }} - STABLE_HELM_REPOSITORY: ${{ env.RELEASE_STABLE_HELM_REPOSITORY }} - run: | - set -euo pipefail - - if [[ "${RELEASE_TYPE}" == "nightly" ]]; then - chart_ref="${NIGHTLY_HELM_OCI_REGISTRY}/${CHART_ID}" - chart_is_available() { - helm show chart "${chart_ref}" --version "${CHART_VERSION}" >/dev/null 2>&1 - } - else - chart_is_available() { - helm repo add nemo-platform "${STABLE_HELM_REPOSITORY}" --force-update >/dev/null 2>&1 \ - && helm search repo nemo-platform/nemo-platform --devel --version "${CHART_VERSION}" \ - | grep -qv 'No results found' - } - fi - - until chart_is_available; do - echo "Waiting for ${CHART_ID}==${CHART_VERSION}" - sleep "${POLL_INTERVAL_SECONDS}" - done - echo "Found ${CHART_ID}==${CHART_VERSION}" - - alert-poll-delay: - name: Alert if release polling is delayed - needs: - - plan-release - - dispatch-release-registration - - dispatch-wheel-stage - - dispatch-container-stage - - stage-helm - if: >- - !cancelled() && - needs.plan-release.result == 'success' && - needs.plan-release.outputs.send_notifications == 'true' && - needs.plan-release.outputs.dry_run != 'true' && - !contains(needs.*.result, 'failure') && - !contains(needs.*.result, 'cancelled') - runs-on: ubuntu-latest - timeout-minutes: 125 - permissions: - actions: read - steps: - - name: Alert if final artifact polling exceeds two hours - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - SLACK_ALERTS_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.plan-release.outputs.source_sha }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUN_NUMBER: ${{ github.run_number }} - with: - script: | - // The poll job has a four-hour timeout; alert after two hours. - const pollJobName = "Wait for published release artifacts"; - const deadline = Date.now() + (2 * 60 * 60 * 1000); - const sleep = (milliseconds) => new Promise((resolve) => { - setTimeout(resolve, milliseconds); - }); - const findPollJob = async () => { - const {data} = await github.rest.actions.listJobsForWorkflowRun({ - ...context.repo, - run_id: context.runId, - per_page: 100, - }); - return data.jobs.find((job) => job.name === pollJobName); - }; - - while (Date.now() < deadline) { - const pollJob = await findPollJob(); - if (pollJob?.status === "completed") { - core.info("Final artifact polling completed before the alert threshold."); - return; - } - await sleep(Math.min(60_000, deadline - Date.now())); - } - - const pollJob = await findPollJob(); - if (pollJob?.status === "completed") { - core.info("Final artifact polling completed at the alert threshold."); - return; - } - - const title = process.env.RELEASE_TYPE === "stable" - ? "*:warning: Release artifact polling delayed*" - : "*:warning: Nightly artifact polling delayed*"; - const lines = [ - title, - `Release: ${process.env.RELEASE_LABEL}`, - `Commit: <${process.env.COMMIT_URL}|${process.env.SOURCE_SHA.slice(0, 7)}>`, - "", - "Final artifact polling has exceeded two hours.", - "", - `:link: <${process.env.RUN_URL}|Release run #${process.env.RUN_NUMBER}>`, - ]; - const response = await fetch(process.env.SLACK_ALERTS_WEBHOOK, { - method: "POST", - headers: {"Content-Type": "application/json"}, - body: JSON.stringify({text: lines.join("\n")}), - }); - if (!response.ok) { - core.setFailed(`Slack webhook returned ${response.status}.`); - } - - create-github-release: - name: Create GitHub Release - needs: [plan-release, poll-final-release] - if: >- - needs.plan-release.outputs.release_type == 'stable' && - needs.plan-release.outputs.release_scope == 'all' && - needs.plan-release.outputs.dry_run != 'true' && - needs.poll-final-release.result == 'success' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Check out release history - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ needs.plan-release.outputs.source_sha }} - fetch-depth: 0 - fetch-tags: true - persist-credentials: false - - - name: Create GitHub Release - shell: bash - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ needs.plan-release.outputs.version }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - run: | - set -euo pipefail - - notes_start_tag="$( - { - git tag -l | grep -E '^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' || true - printf '%s\n' "${RELEASE_TAG}" - } | sort -V -u | awk -v version="${RELEASE_TAG}" '$0 == version {print previous} {previous = $0}' - )" - notes_range=() - if [[ -n "${notes_start_tag}" && "${notes_start_tag}" != "${RELEASE_TAG}" ]]; then - notes_range=(--notes-start-tag "${notes_start_tag}") - fi - - gh release create "${RELEASE_TAG}" \ - --target "${SOURCE_SHA}" \ - --title "${RELEASE_TAG}" \ - --generate-notes \ - "${notes_range[@]}" - - signal-deployment: - name: Signal deployment - needs: [plan-release, poll-final-release] - if: >- - needs.poll-final-release.result == 'success' && - needs.plan-release.outputs.dry_run != 'true' - runs-on: ubuntu-latest - steps: - - run: echo "::notice::Placeholder - signal the completed release" - - notify-end: - name: Notify release result - needs: - - plan-release - - poll-final-release - - create-github-release - - signal-deployment - - stage-helm - if: >- - !cancelled() && - needs.plan-release.outputs.send_notifications == 'true' && - needs.plan-release.outputs.dry_run != 'true' - runs-on: ubuntu-latest - steps: - - name: Send Slack alert - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - SLACK_ALERTS_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }} - SLACK_RELEASE_WEBHOOK: ${{ secrets.SLACK_RELEASE_WEBHOOK }} - RELEASE_TYPE: ${{ needs.plan-release.outputs.release_type }} - RELEASE_LABEL: ${{ needs.plan-release.outputs.release_label }} - SOURCE_SHA: ${{ needs.plan-release.outputs.source_sha }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ needs.plan-release.outputs.source_sha }} - WHEEL_IDS: ${{ needs.plan-release.outputs.wheel_ids }} - WHEEL_CATALOG: ${{ env.RELEASE_WHEELS_JSON }} - WHEEL_VERSION: ${{ needs.plan-release.outputs.wheel_version }} - CONTAINER_IDS: ${{ needs.plan-release.outputs.container_ids }} - INCLUDE_HELM: ${{ needs.plan-release.outputs.include_helm }} - CHART_VERSION: ${{ needs.stage-helm.outputs.chart_version }} - NIGHTLY_WHEEL_INDEX: ${{ env.RELEASE_NIGHTLY_WHEEL_INDEX }} - STABLE_WHEEL_INDEX: ${{ env.RELEASE_STABLE_WHEEL_INDEX }} - NGC_CATALOG_BASE: ${{ env.RELEASE_NGC_CATALOG_BASE }} - POLL_RESULT: ${{ needs.poll-final-release.result }} - GITHUB_RELEASE_RESULT: ${{ needs.create-github-release.result }} - DEPLOYMENT_RESULT: ${{ needs.signal-deployment.result }} - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - RUN_NUMBER: ${{ github.run_number }} - with: - script: | - const releaseType = process.env.RELEASE_TYPE; - const results = [ - process.env.POLL_RESULT, - process.env.GITHUB_RELEASE_RESULT, - process.env.DEPLOYMENT_RESULT, - ]; - const failed = results.some((result) => ["failure", "cancelled"].includes(result)); - const published = process.env.POLL_RESULT === "success" && !failed; - const webhook = published - ? process.env.SLACK_RELEASE_WEBHOOK - : process.env.SLACK_ALERTS_WEBHOOK; - const title = published - ? (releaseType === "stable" - ? "*:ship: Release publish complete*" - : "*:crescent_moon: Nightly release publish complete*") - : (releaseType === "stable" - ? "*:alert: Release publish failed*" - : "*:alert: Nightly release publish failed*"); - const lines = [ - title, - `Release: ${process.env.RELEASE_LABEL}`, - `Commit: <${process.env.COMMIT_URL}|${process.env.SOURCE_SHA.slice(0, 7)}>`, - ]; - - if (published) { - const wheelIds = JSON.parse(process.env.WHEEL_IDS); - const wheelCatalog = JSON.parse(process.env.WHEEL_CATALOG); - const containerIds = JSON.parse(process.env.CONTAINER_IDS); - lines.push("", "*Artifacts published:*"); - if (wheelIds.length > 0) { - lines.push("*:python: Wheels published:*"); - for (const wheelId of wheelIds) { - const wheel = wheelCatalog.find((candidate) => candidate.id === wheelId); - const wheelIndex = releaseType === "nightly" - ? process.env.NIGHTLY_WHEEL_INDEX - : process.env.STABLE_WHEEL_INDEX; - const wheelUrl = releaseType === "nightly" - ? `${wheelIndex}/${wheel.package}/` - : `${wheelIndex.replace(/\/simple$/, "/project")}/${wheel.package}/${process.env.WHEEL_VERSION}/`; - lines.push(`- <${wheelUrl}|${wheel.package}: ${process.env.WHEEL_VERSION}>`); - } - } - if (containerIds.length > 0) { - lines.push("*:docker_: Containers published:*"); - for (const containerId of containerIds) { - const container = releaseType === "stable" - ? `<${process.env.NGC_CATALOG_BASE}/containers/${containerId}|${containerId}>` - : containerId; - lines.push(`- ${container}: ${process.env.RELEASE_LABEL}`); - } - } - if (process.env.INCLUDE_HELM === "true") { - const chart = releaseType === "stable" - ? `<${process.env.NGC_CATALOG_BASE}/helm-charts/nemo-platform|nemo-platform>` - : "nemo-platform"; - lines.push("*:helm: Helm chart published:*"); - lines.push(`- ${chart}: ${process.env.CHART_VERSION}`); - } - } else { - lines.push( - "", - "*Final release status:*", - `Final artifact poll: ${process.env.POLL_RESULT}`, - `GitHub release: ${process.env.GITHUB_RELEASE_RESULT}`, - `Deployment signal: ${process.env.DEPLOYMENT_RESULT}`, - ); - } - - lines.push("", `:link: <${process.env.RUN_URL}|Release run #${process.env.RUN_NUMBER}>`); - const response = await fetch(webhook, { - method: "POST", - headers: {"Content-Type": "application/json"}, - body: JSON.stringify({text: lines.join("\n")}), - }); - if (!response.ok) { - core.setFailed(`Slack webhook returned ${response.status}.`); - } diff --git a/.gitignore b/.gitignore index 7032d23e77..9e47336c0b 100644 --- a/.gitignore +++ b/.gitignore @@ -146,8 +146,3 @@ site/ # Generated Fern-only public API reference spec docs/fern/openapi/openapi.public.yaml - -# nektos/act files commonly used -.act-variables -.act-secrets - diff --git a/RELEASING.md b/RELEASING.md index 94974c82d9..74e44cd67f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,153 +1,138 @@ # Releasing NeMo Platform -[`release.yaml`](.github/workflows/release.yaml) is the single release workflow -for NeMo Platform. It handles scheduled nightlies and manually dispatched -nightly or stable releases. The release catalog is deliberately defined in that -workflow so contributors can see and validate every releasable artifact in one -place. +This document describes the end-to-end process for cutting and shipping a new version of NeMo Platform. -Anyone with permission to run repository workflows can start a release. A -stable release requires a specific source commit and version; a nightly can use -the default branch head. +> **Who can release?** Any team member can trigger the workflow. The `release-stable` GitHub Actions environment requires approval from a member of the `nmp_devops` team before the workflow proceeds. -## Before starting a stable release +--- -Choose the exact 40-character commit SHA and the `MAJOR.MINOR.PATCH` version -to release. The source must contain the desired generated SDKs. If the API -surface changed since the last SDK update, update the SDKs before releasing: +## Overview + +> The examples in this document use `0.1.2` as the version being released. + +``` +trigger release-stable.yaml with source_sha + version → nmp_devops approval → workflow tags source_sha → Platform-Deploy publishes to PyPI +``` + +Artifacts published on a stable release: +- `nemo-platform` on [pypi.org](https://pypi.org/project/nemo-platform/) +- `nemo-platform-plugin` on [pypi.org](https://pypi.org/project/nemo-platform-plugin/) + +Nightly builds go to `pypi.nvidia.com` (NVIDIA's internal/public PyPI mirror), **not** public PyPI. + +## Versioning model + +Release and nightly wheel versions are resolved at build time. The release workflow runs `.github/scripts/stamp_sdk_version.py`, then passes the resolved version to Hatch through `UV_DYNAMIC_VERSIONING_BYPASS`. + +Dynamic versioning is intentionally limited to packages that need release/nightly wheel metadata: +- `packages/nemo_platform` (`nemo-platform`) +- `packages/nemo_platform_plugin` (`nemo-platform-plugin`) +- `sdk/python/nemo-platform` (`nemo-platform-sdk`, consumed by the released wrappers and SDK tooling) + +All other first-party workspace packages use static stub versions, normally `0.0.0`, because they are implementation packages rather than independently released artifacts. Do not add `nmp-dynamic-versioning` to another package unless that package is added to the release catalog or otherwise needs published wheel metadata. + +`packages/nmp_build_tools` centralizes the Hatch version source and its defaults, but that package itself is also an internal stub-version package. The OpenAPI specs are schema inputs for SDK generation and intentionally keep a fixed `info.version: 0.0.0`; package release versions should not be copied into the specs. + +--- + +## Step 1 — Choose the source SHA and release version + +Pick the full 40-character commit SHA on `main` that should be released, plus the SemVer core version to publish, for example `0.1.2`. The stable workflow creates the release tag at `source_sha`, and the wheel build receives the package version from the workflow input. + +If the API surface changed since the last SDK update, regenerate the OpenAPI spec and SDKs before releasing: ```bash make update-sdk ``` -This regenerates the OpenAPI specifications and synchronizes the SDKs. The -specifications intentionally retain `info.version: 0.0.0`; do not copy the -release version into them. +This runs `make refresh-openapi` (regenerates `openapi/openapi.yaml` and plugin specs) and then syncs the Python and web SDKs via Stainless. Requires `STAINLESS_API_KEY` to be set — see `sdk/README.md` for setup instructions. The generated OpenAPI specs should keep `info.version: 0.0.0`. -## Release catalog +To find the right SHA: -The catalog in [`release.yaml`](.github/workflows/release.yaml) is the source -of truth for selection. Do not add a separate release manifest or configuration -file. When changing the catalog, also update the matching `workflow_dispatch` -input description in that workflow. +```bash +git log --oneline main | head -5 +# Pick the commit to release and copy its full 40-character SHA. +``` -| Type | IDs | -| --- | --- | -| Wheels | `nemo-platform`, `nemo-platform-plugin` | -| Containers | `nmp-api`, `nmp-cpu-tasks`, `nmp-automodel-tasks`, `nmp-automodel-training`, `nmp-unsloth-training`, `auditor-tasks`, `safe-synthesizer-tasks` | -| Helm chart | `nemo-platform` | +--- -For every selected wheel, the workflow checks that its package configuration -declares the expected project name. For every selected container, it checks the -Docker Bake target and the corresponding -`.github/assets/ngc/containers/.md` overview file. These checks happen -before any external release work is dispatched. +## Step 2 — Trigger the stable release workflow -## Starting a release +Navigate to the [`release-stable.yaml` workflow](https://github.com/NVIDIA-NeMo/nemo-platform/actions/workflows/release-stable.yaml) and click **Run workflow**. -Open the [Release workflow](https://github.com/NVIDIA-NeMo/nemo-platform/actions/workflows/release.yaml) -and select **Run workflow**. The form shows the allowed custom artifact IDs. +| Input | Required | Description | +|---|---|---| +| `source_sha` | Yes | The full 40-character commit SHA to release from (must be on `main`). | +| `version` | Yes | SemVer core version string to release, e.g. `0.1.2`. This becomes the stable git tag and wheel version. | +| `release_date` | No | `YYYY-MM-DD`. Provide only on the first run for a given version; leave blank on reruns. | +| `release_scope` | No | `all` (default) releases every catalog SDK and container. Use `sdks`, `containers`, or `custom` for narrower releases. | +| `sdk_ids` | No | Comma-separated SDK IDs for `release_scope: custom`; must exist in `release/assets.yaml`. | +| `container_ids` | No | Comma-separated container IDs for `release_scope: custom`; must exist in `release/assets.yaml`. | -| Input | Use | -| --- | --- | -| `release-type` | `nightly` by default. Select `stable` for a full release. | -| `source-sha` | Required for stable releases. Optional for nightlies; a normal nightly with no SHA uses the current default-branch head. A dry-run nightly with no SHA uses the workflow commit so a branch can be validated. | -| `version` | Required for stable releases. Enter the `MAJOR.MINOR.PATCH` release version. | -| `release-scope` | `all` by default. Select `wheels`, `containers`, `helm`, or `custom` for a subset. | -| `wheel-ids`, `container-ids` | Comma-separated IDs used only with `release-scope: custom`. Each ID must be in the catalog above; duplicates and empty entries fail validation. | -| `include-helm` | Includes the Helm chart in a custom release. | -| `update-ngc-metadata` | Also runs the reusable NGC metadata workflow for `nemo-platform` and `nemo-platform-dev`. It checks out the workflow ref, normally `main`. | -| `send-notifications` | Sends Slack start and final-status notifications. Defaults to `true`. | -| `dry-run` | Validates the selected source and packages the selected Helm chart, but does not publish, dispatch external work, poll, create a GitHub release, or signal deployment. The start notification intentionally still runs when notifications are enabled. | - -Examples: - -| Goal | Inputs | -| --- | --- | -| Scheduled-style nightly | Leave `release-type` as `nightly` and use the default `all` scope. | -| Stable full release | `release-type: stable`, `source-sha: <40-character SHA>`, `version: `, `release-scope: all`. | -| One container | `release-scope: custom`, `container-ids: nmp-automodel-tasks`. | -| Helm-only validation | `release-scope: helm`, `dry-run: true`. | - -Nightlies also run automatically Monday through Friday at 8:00 PM -America/Los_Angeles. - -## What the workflow does - -1. Resolves the source, release label, selected artifacts, and wheel version. - Stable versions use the supplied release version. Nightly labels use - `nightly-` and the wheel version is resolved by - `.github/scripts/stamp_sdk_version.py`. -2. Checks out the selected source and validates the selected wheel paths, - Docker Bake targets, and NGC overview files. -3. Optionally synchronizes NGC metadata, when requested on a non-dry-run. -4. Dispatches wheel, container, and stable-release registration work to the - configured internal release repository. The selected source SHA, release - type, version, and selected IDs are passed with the dispatch. -5. Packages the Helm chart. A nightly chart uses the `Chart.yaml` version with - `-nightly-` appended. A stable chart currently uses the - stable release version. Whether stable chart versions should instead remain - independently managed in `Chart.yaml` is an open policy decision. -6. Waits for every selected final artifact to become public before continuing. - The polling job times out after four hours and sends a Slack alert after two - hours if it is still waiting. -7. For a non-dry-run stable release with `release-scope: all`, creates the - GitHub release and tag at the selected SHA. GitHub generates the release - notes from the previous numeric SemVer tag. Subset releases do not create a - GitHub release or tag. - -The deployment-signalling job is currently a placeholder. A successful poll -means the selected artifacts are public; it does not yet mean a deployment was -created by this workflow. - -## Publication destinations - -| Artifact | Nightly | Stable | -| --- | --- | --- | -| Wheels | [`pypi.nvidia.com`](https://pypi.nvidia.com) | [PyPI](https://pypi.org) | -| Containers | `ghcr.io/nvidia-nemo/nemo-platform/:nightly-...` | `nvcr.io/nvidia/nemo-platform/:` and the public NGC catalog | -| Helm chart | OCI chart at `oci://ghcr.io/nvidia-nemo/nemo-platform` | Initially staged at `0921617854601259/nemo-platform`, then promoted to the public [NGC Helm repository](https://helm.ngc.nvidia.com/nvidia/nemo-platform) | - -The stable Helm promotion is external to this workflow. The workflow polls the -public NGC Helm repository, not the internal staging endpoint, before it marks -the release complete. - -## Notifications - -With `send-notifications: true`: - -- A start message is sent to `SLACK_ALERTS_WEBHOOK`, including the selected - artifacts and source commit. This is also sent for dry-runs so the webhook can - be tested. -- If final artifact polling exceeds two hours, a delay alert is sent to - `SLACK_ALERTS_WEBHOOK`. -- A successful non-dry-run release sends its completion message to - `SLACK_RELEASE_WEBHOOK`. A failed or cancelled release sends its final status - to `SLACK_ALERTS_WEBHOOK`. - -Dry-runs do not poll or send the delayed or final notification. - -## Required secrets - -| Secret | Used for | -| --- | --- | -| `CI_DISPATCH_REPO` | `owner/repo` of the internal release repository that receives release dispatches. | -| `CI_DISPATCH_TOKEN` | Authenticating those cross-repository dispatches. | -| `AIRE_NVCR_GITHUB` | Staging stable Helm charts in NGC. | -| `AIRE_NGC_GITHUB_PLATFORM_RW` | Optional NGC metadata synchronization. | -| `SLACK_ALERTS_WEBHOOK` | Release starts, delay alerts, and failed final statuses. | -| `SLACK_RELEASE_WEBHOOK` | Successful final release status. | +The workflow runs from the **`main` branch** by default. The `source_sha` must be reachable from that branch. + +**What the workflow does:** +1. Validates inputs and previews the release. +2. Pauses at the `approve-stable-release` gate — a member of the **`nmp_devops` team** must approve in the GitHub environment UI. +3. Creates and pushes a git tag (e.g. `0.1.2`) at `source_sha`. +4. Builds Python wheels for each SDK in `release/assets.yaml` using `.github/actions/build-nemo-platform-wheel`. +5. Assembles a release bundle with checksums and metadata. +6. Dispatches a `release-bundle-produced` event to the **Platform-Deploy** repository (`CI_DISPATCH_REPO` secret), which handles the actual PyPI publish. + +> If the PyPI publishing service is returning 5xx errors, the publish step in Platform-Deploy will fail. Wait for the service to recover and re-run the workflow with the same `source_sha` and `version` — the stable tag is already reserved so re-running is safe. -## Verifying a completed release +--- -The workflow summary records the selected wheels and containers. After a live -release completes, check the selected artifacts at their destination above. -For a full stable release, also verify that the GitHub tag and generated GitHub -release point to the requested source SHA. +## Step 3 — Verification -For a wheel release, a quick client check is: +Once the workflow completes, verify the release landed correctly: ```bash uv tool upgrade nemo-platform nemo --version +# Expected: nemo version ``` + +Also check: +- [pypi.org/project/nemo-platform](https://pypi.org/project/nemo-platform/) — version and description updated. +- [pypi.org/project/nemo-platform-plugin](https://pypi.org/project/nemo-platform-plugin/) — version updated. +- GitHub: a tag (e.g. `0.1.2`) exists on the release commit. + +--- + +## Container image eligibility + +The `container:` list in `release/assets.yaml` declares which container +images are eligible for release publishing. The bundle workflow records the +selected containers as `container`-typed entries in `release-manifest.json`, +and the release consumer stages those images after the SDK publish, reading +this list from this repository at the release ref. Eligibility is therefore +version-pinned: re-staging an old tag publishes the container set declared at +that commit. + +`release_scope` controls what a release includes (default `all`): + +| Scope | Includes | +| --- | --- | +| `all` | every catalog SDK + every catalog container (default) | +| `sdks` | every catalog SDK, no containers | +| `containers` | every catalog container, no SDKs | +| `custom` | exactly the comma-separated `sdk_ids` + `container_ids` (either may be empty) | + +`custom` enables single-artifact or arbitrary-subset releases (for example a +patch release of one container via `release_scope: custom`, +`container_ids: nmp-automodel-tasks`); `containers` releases the whole +container set with no SDK wheels. + +Adding an image here also requires a catalog metadata entry on the consumer +side. Images are built into the dev registry tagged with this repository's +commit SHA on every merge to main; release SHAs that predate that build +trigger need a manual image build first. + +--- + +## Nightly builds + +Nightly builds run automatically at 20:00 PT and publish to `pypi.nvidia.com`. They use the HEAD of `main` and version strings like `0.1.3.dev20260101120000`. No action required from the team. + +To trigger a nightly manually: [`release-nightly.yaml`](https://github.com/NVIDIA-NeMo/nemo-platform/actions/workflows/release-nightly.yaml) → **Run workflow** (no inputs required). Leave `send_notifications` enabled for real reruns; disable it only for quiet smoke/ad-hoc runs. diff --git a/conftest.py b/conftest.py index 64b1ec532f..5587105dce 100644 --- a/conftest.py +++ b/conftest.py @@ -284,20 +284,6 @@ def pytest_addoption(parser): default=True, help="Run integration tests (enabled by default)", ) - parser.addoption( - "--feature", - action="append", - default=[], - metavar="NAME", - help="Enable optional e2e feature sets (repeatable), e.g. --feature gpu", - ) - - -def _e2e_features_enabled(config: pytest.Config) -> set[str]: - features = {feature.lower() for feature in config.getoption("--feature") or []} - if os.environ.get("RUN_NSS_K8S_E2E") == "1": - features.add("gpu") - return features def pytest_runtest_setup(item): @@ -316,9 +302,6 @@ def pytest_runtest_setup(item): if "container_only" in [marker.name for marker in item.iter_markers()]: if not os.environ.get("NMP_BASE_URL"): skip_test("Skipping container-only test (requires NMP_BASE_URL)") - if "requires_gpu" in [marker.name for marker in item.iter_markers()]: - if "gpu" not in _e2e_features_enabled(item.config): - skip_test("Skipping GPU container e2e (pass --feature gpu)") from xdist.scheduler.loadgroup import LoadGroupScheduling # noqa: E402 diff --git a/docker-bake.hcl b/docker-bake.hcl index cafad318de..2c662977bd 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -291,6 +291,7 @@ group "docker" { group "docker-cpu" { targets = [ "nmp-api-docker", + "nmp-core-docker", "nmp-cpu-tasks-docker", ] } @@ -931,6 +932,9 @@ target "auditor-tasks-docker" { contexts = { root-lib-source-artifacts = "target:root-lib-source-artifacts" root-busybox = "target:root-busybox" + nmp-python-base = "target:nmp-python-base" + nmp-python-dev-base = "target:nmp-python-dev-base" + root-distroless-base-3-11 = "target:root-distroless-base-3-11" } dockerfile = "docker/Dockerfile.auditor-tasks" cache-to = maybe_registry_cache_to("auditor-tasks") diff --git a/docker/Dockerfile.auditor-tasks b/docker/Dockerfile.auditor-tasks index ab180bd9a4..f6d7846f5c 100644 --- a/docker/Dockerfile.auditor-tasks +++ b/docker/Dockerfile.auditor-tasks @@ -1,32 +1,15 @@ # hadolint global ignore=DL3059 -# The final image copies BusyBox shell utilities for parity with the task runtime. +# distroless images don't have a real shell - & and | are not enabled -ARG AUDITOR_PYTHON_IMAGE=python:3.13.14-slim-trixie +ARG NMP_PYTHON_BASE=nmp-python-base ARG CACHE_HOME=/tmp/.cache -FROM ${AUDITOR_PYTHON_IMAGE} AS py-builder +FROM nmp-python-dev-base AS py-builder -ARG CACHE_HOME -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - g++ \ - gcc \ - git \ - libffi-dev \ - libpq-dev \ - libssl-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv - -ENV UV_COMPILE_BYTECODE=1 \ - UV_LINK_MODE=copy +ARG TARGETARCH -WORKDIR /app +ARG CACHE_HOME RUN mkdir -p ${CACHE_HOME} && chmod 777 ${CACHE_HOME} # Create the directory structure to match the relative paths in pyproject.toml @@ -74,8 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "protobuf>=6.33.5,<7.0.0" \ "langchain-core>=1.2.22" \ "orjson>=3.11.6" \ - "cryptography>=48.0.1,<49" \ - "nltk>=3.10.0" + "nltk>=3.9.3" # CVE Remediation RUN --mount=type=cache,target=/root/.cache/uv \ @@ -86,17 +68,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "pyasn1>=0.6.3" \ "wheel>=0.46.2" \ "orjson>=3.11.6" \ - "cryptography>=48.0.1,<49" \ - "nltk>=3.10.0" + "nltk>=3.9.3" -FROM ${AUDITOR_PYTHON_IMAGE} AS base +FROM ${NMP_PYTHON_BASE} AS base ARG USERNAME=nvs ARG USER_UID=1000 ARG USER_GID=1000 WORKDIR /app -RUN apt-get update && apt-get upgrade -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* RUN groupadd --gid ${USER_GID} ${USERNAME} && \ useradd --uid ${USER_UID} --gid ${USER_GID} --create-home --shell /bin/bash ${USERNAME} diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training index 4abe4bc51d..cf3a384756 100644 --- a/docker/Dockerfile.nmp-unsloth-training +++ b/docker/Dockerfile.nmp-unsloth-training @@ -5,9 +5,8 @@ # # Install steps: # 1. `uv pip install unsloth --torch-backend=auto` plus explicit -# `transformers==5.3.0` and `huggingface-hub==1.3.0` pins. Unsloth's -# resolver allows transformers 5.3.0, and the explicit pins keep later -# platform glue installs from re-solving the HF stack. +# `transformers==4.57.6` and `huggingface-hub==0.36.2` pins (transformers +# 4.57.x requires hub <1.0; platform glue would otherwise pull hub 1.x). # Unsloth's resolver still pulls unsloth_zoo # and the rest of the HF stack (trl, peft, accelerate, datasets, # bitsandbytes, xformers, etc.). `--overrides preserve_base_torch.txt` @@ -71,8 +70,8 @@ ARG USERNAME=ubuntu ARG USER_UID=1000 ARG USER_GID=1000 ARG UNSLOTH_VERSION=2026.6.1 -ARG TRANSFORMERS_VERSION=5.3.0 -ARG HF_HUB_VERSION=1.3.0 +ARG TRANSFORMERS_VERSION=4.57.6 +ARG HF_HUB_VERSION=0.36.2 ARG BITSANDBYTES_VERSION=0.49.2 ARG BNB_MAX_JOBS=10 diff --git a/docker/Dockerfile.safe-synthesizer-tasks b/docker/Dockerfile.safe-synthesizer-tasks index 3330ae4088..15443c3464 100644 --- a/docker/Dockerfile.safe-synthesizer-tasks +++ b/docker/Dockerfile.safe-synthesizer-tasks @@ -11,10 +11,10 @@ ARG PYTHON_VERSION=3.13 ARG PYTHON_IMAGE=python:${PYTHON_VERSION}-slim-trixie ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.9.30 -ARG SAFE_SYNTHESIZER_RUNTIME_PACKAGE="nemo-safe-synthesizer[engine,cu129]==0.1.7" +ARG SAFE_SYNTHESIZER_RUNTIME_PACKAGE="nemo-safe-synthesizer[engine,cu129]==0.1.2" ARG FLASHINFER_CU129_INDEX_URL="https://flashinfer.ai/whl/cu129" ARG PYTORCH_CU129_INDEX_URL="https://download.pytorch.org/whl/cu129" -ARG VLLM_CU129_INDEX_URL="https://wheels.vllm.ai/ee0da84ab9e04ac7610e28580af62c365e898389/cu129" +ARG VLLM_CU129_WHEEL="vllm @ https://github.com/vllm-project/vllm/releases/download/v0.20.0/vllm-0.20.0%2Bcu129-cp38-abi3-manylinux_2_31_x86_64.whl" # ============================================================================= # uv binary @@ -29,17 +29,13 @@ ARG PYTHON_IMAGE ARG SAFE_SYNTHESIZER_RUNTIME_PACKAGE ARG FLASHINFER_CU129_INDEX_URL ARG PYTORCH_CU129_INDEX_URL -ARG VLLM_CU129_INDEX_URL +ARG VLLM_CU129_WHEEL ARG CONTAINER_VARIANT=cu129 ARG USERNAME=nemo ARG USER_UID=1000 ARG USER_GID=1000 -# Raise uv's HTTP retry budget above the default of 3. uv retries transient -# failures (connection errors and 429/5xx status codes such as the intermittent -# 503s from download.pytorch.org) with exponential backoff between attempts. -ENV UV_HTTP_TIMEOUT=120 \ - UV_HTTP_RETRIES=8 +ENV UV_HTTP_TIMEOUT=120 COPY --from=uv /uv /uvx /usr/local/bin/ @@ -114,10 +110,11 @@ RUN printf '%s\n' \ printf '%s\n' \ "${SAFE_SYNTHESIZER_RUNTIME_PACKAGE}" \ > /tmp/safe-synthesizer-runtime.txt && \ + printf '%s\n' \ + "${VLLM_CU129_WHEEL}" \ + > /tmp/vllm-cu129.txt && \ printf '%s\n' \ wandb==0.27.2 \ - 'cryptography>=48.0.1,<49' \ - 'pyarrow>=23.0.1,<24' \ > /tmp/safe-synthesizer-overrides.txt RUN --mount=type=cache,target=/root/.cache/uv \ @@ -160,12 +157,12 @@ RUN --mount=type=cache,target=/root/.cache/uv \ UV_CACHE_DIR=/root/.cache/uv uv pip install --python /opt/venv/bin/python \ --extra-index-url "${FLASHINFER_CU129_INDEX_URL}" \ --extra-index-url "${PYTORCH_CU129_INDEX_URL}" \ - --extra-index-url "${VLLM_CU129_INDEX_URL}" \ --index-strategy unsafe-best-match \ --torch-backend cu129 \ --overrides /tmp/safe-synthesizer-overrides.txt \ --constraints /tmp/safe-synthesizer-constraints.txt \ - --requirements /tmp/safe-synthesizer-runtime.txt + --requirements /tmp/safe-synthesizer-runtime.txt \ + --requirements /tmp/vllm-cu129.txt COPY pyproject.toml uv.lock ./ COPY packages/ packages/ @@ -182,13 +179,9 @@ RUN mkdir -p docs && \ # Skip the `nemo-platform` wrapper wheel: it bundles every first-party plugin and # service, which drags unrelated source trees into this task image. The runtime # imports come from nemo-platform-sdk, nemo-platform-plugin, and nmp-common. -# The runtime layer above installs CVE-patched cryptography/pyarrow; do not let -# the workspace lock downgrade them during plugin sync. RUN --mount=type=cache,target=/root/.cache/uv \ UV_CACHE_DIR=/root/.cache/uv uv sync --package nemo-safe-synthesizer-plugin --no-dev --no-editable --inexact \ - --no-install-package nemo-platform \ - --no-install-package cryptography \ - --no-install-package pyarrow + --no-install-package nemo-platform COPY docker/scripts/cve-cleanup.sh /bin/ RUN bash /bin/cve-cleanup.sh diff --git a/docker/base/Dockerfile.nmp-workspace b/docker/base/Dockerfile.nmp-workspace index 2839612dda..15c7748a1d 100644 --- a/docker/base/Dockerfile.nmp-workspace +++ b/docker/base/Dockerfile.nmp-workspace @@ -13,3 +13,4 @@ COPY packages/nmp_platform/ entrypoint/ COPY sdk/ sdk/ COPY script/ script/ COPY tools/ tools/ +COPY release/ release/ diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index 3dab764ba6..e0e3244df6 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -896,8 +896,8 @@ safe_synthesizer: container_image_ref: # default: '.nemo/safe-synthesizer-runtime' runtime_venv: .nemo/safe-synthesizer-runtime - # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.7' - runtime_package: nemo-safe-synthesizer[engine,cu129]==0.1.7 + # default: 'nemo-safe-synthesizer[engine,cu129]==0.1.2' + runtime_package: nemo-safe-synthesizer[engine,cu129]==0.1.2 # default: '3.11' runtime_python_version: '3.11' runtime_python: diff --git a/e2e/k8s/scripts/local_build_and_upgrade.sh b/e2e/k8s/scripts/local_build_and_upgrade.sh index 9ab830c991..87ad05d541 100755 --- a/e2e/k8s/scripts/local_build_and_upgrade.sh +++ b/e2e/k8s/scripts/local_build_and_upgrade.sh @@ -8,13 +8,8 @@ # MINIKUBE_PROFILE - minikube profile name (default: minikube) # NMP_REGISTRY - image registry (default: docker.io/my-registry) # IMAGE_TAG - image tag (default: local-) -# BUILD_ARCH - target platform (default: auto-detected from host) -# MINIKUBE_GPU - when 1, start minikube with GPU passthrough -# BUILD_SAFE_SYNTHESIZER - when 1, also build safe-synthesizer-tasks (amd64; -# set BUILD_ARCH=linux/amd64) -# BUILD_GPU - deprecated convenience alias: sets MINIKUBE_GPU=1 and -# BUILD_SAFE_SYNTHESIZER=1 -# HELM_VALUES - values file (default: e2e/k8s/values/minikube.yaml) +# BUILD_ARCH - target platform (default: auto-detected from host) +# HELM_VALUES - values file (default: e2e/k8s/values/minikube.yaml) set -e @@ -23,21 +18,10 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" MINIKUBE_PROFILE="${MINIKUBE_PROFILE:-minikube}" -if [ "${BUILD_GPU:-0}" = "1" ]; then - MINIKUBE_GPU="${MINIKUBE_GPU:-1}" - BUILD_SAFE_SYNTHESIZER="${BUILD_SAFE_SYNTHESIZER:-1}" -fi -MINIKUBE_GPU="${MINIKUBE_GPU:-0}" -BUILD_SAFE_SYNTHESIZER="${BUILD_SAFE_SYNTHESIZER:-0}" - # Check if minikube is running if ! minikube status -p "${MINIKUBE_PROFILE}" &>/dev/null; then echo "Minikube profile ${MINIKUBE_PROFILE} is not running. Starting..." - if [ "${MINIKUBE_GPU}" = "1" ]; then - MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_gpu.sh" - else - MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_cpu.sh" - fi + MINIKUBE_PROFILE="${MINIKUBE_PROFILE}" "$SCRIPT_DIR/setup_local_minikube_cpu.sh" fi # Wait for minikube to be ready @@ -64,22 +48,10 @@ eval "$(minikube -p "${MINIKUBE_PROFILE}" docker-env)" IMAGE_REGISTRY="${NMP_REGISTRY}" \ BUILD_ARCH="$BUILD_ARCH" \ docker buildx bake docker-cpu --set "*.platform=$BUILD_ARCH" - - if [ "${BUILD_SAFE_SYNTHESIZER}" = "1" ]; then - echo "Building safe-synthesizer-tasks (BUILD_SAFE_SYNTHESIZER=1)..." - CI_COMMIT_SHA="$GIT_SHA" \ - BAKE_TAG="$IMAGE_TAG" \ - IMAGE_REGISTRY="${NMP_REGISTRY}" \ - BUILD_ARCH="$BUILD_ARCH" \ - docker buildx bake safe-synthesizer-tasks-docker --set "safe-synthesizer-tasks-docker.platform=$BUILD_ARCH" - fi ) echo "----------------------------------------" echo "Images built with tag: $IMAGE_TAG" -if [ "${BUILD_SAFE_SYNTHESIZER}" = "1" ]; then - echo "Also built: safe-synthesizer-tasks" -fi echo "----------------------------------------" # Delegate helm install to install_helm_e2e.sh diff --git a/e2e/test_safe_synthesizer.py b/e2e/test_safe_synthesizer.py deleted file mode 100644 index 33b2bf8fc3..0000000000 --- a/e2e/test_safe_synthesizer.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Opt-in container E2E for Safe Synthesizer GPU jobs (Docker or Kubernetes). - -These tests exercise the full platform path: plugin job API -> Jobs controller -> -GPU container step -> safe-synthesizer-tasks image -> Files results. - -Excluded from default kind-cpu CI (no GPU, no safe-synthesizer-tasks image). -Run manually against minikube GPU, dev-blue, or a GPU-enabled Docker backend: - - # After nss-k8s-deploy.sh (or MINIKUBE_GPU=1 BUILD_SAFE_SYNTHESIZER=1 local_build_and_upgrade.sh) - NMP_BASE_URL=http://localhost:30080 \ - uv run --frozen pytest e2e/test_safe_synthesizer.py -v --run-e2e --run-slow --feature gpu -""" - -from __future__ import annotations - -import os -import random -import subprocess -from datetime import date -from pathlib import Path - -import pandas as pd -import pytest -from nemo_platform import NeMoPlatform -from nemo_safe_synthesizer_plugin.sdk.job import SafeSynthesizerJob -from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder - -_REPO_ROOT = Path(__file__).resolve().parents[1] -_SETUP_MODEL_FILESETS = _REPO_ROOT / "plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py" - -_MIN_INPUT_ROWS = 200 -_DEFAULT_INPUT_ROWS = 250 -_DEFAULT_NUM_RECORDS = _DEFAULT_INPUT_ROWS - -_ICE_CREAM_FLAVORS = [ - "Vanilla", - "Chocolate", - "Strawberry", - "Mint Chocolate Chip", - "Cookies and Cream", - "Pistachio", - "Rocky Road", - "Butter Pecan", - "Coffee", - "Mango Sorbet", - "Salted Caramel", - "Cookie Dough", -] - -pytestmark = [ - pytest.mark.e2e, - pytest.mark.container_only, - pytest.mark.requires_gpu, - pytest.mark.slow, - pytest.mark.timeout(7200), -] - - -@pytest.fixture(scope="module") -def nss_model_filesets(sdk: NeMoPlatform, _services: str) -> None: - """Register HuggingFace-backed model filesets required by Safe Synthesizer tasks.""" - result = subprocess.run( - [ - "uv", - "run", - "python", - str(_SETUP_MODEL_FILESETS), - "--files-api-url", - _services, - ], - cwd=_REPO_ROOT, - check=False, - capture_output=True, - text=True, - ) - if result.returncode != 0: - pytest.fail( - f"Failed to register Safe Synthesizer model filesets\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) - - -def _synthesis_dataset(rows: int | None = None) -> pd.DataFrame: - """Build a tabular dataset suitable for Safe Synthesizer training (>= 200 rows). - - Schema matches plugins/nemo-safe-synthesizer/tests/e2e/test_local_synthesis.py: - names, dates, and a categorical column with realistic variation. - """ - if rows is None: - rows = int(os.environ.get("NSS_E2E_INPUT_ROWS", str(_DEFAULT_INPUT_ROWS))) - if rows < _MIN_INPUT_ROWS: - raise ValueError(f"Safe Synthesizer container E2E requires at least {_MIN_INPUT_ROWS} input rows, got {rows}") - - faker_mod = pytest.importorskip("faker") - fake = faker_mod.Faker() - faker_mod.Faker.seed(42) - random.seed(42) - - records = [ - { - "name": fake.name(), - "signup_date": fake.date_between_dates( - date_start=date(2020, 1, 1), - date_end=date(2026, 5, 4), - ).isoformat(), - "birthdate": fake.date_between_dates( - date_start=date(1945, 1, 1), - date_end=date(2006, 12, 31), - ).isoformat(), - "favorite_ice_cream_flavor": random.choice(_ICE_CREAM_FLAVORS), - } - for _ in range(rows) - ] - return pd.DataFrame.from_records(records) - - -def test_safe_synthesizer_container_job_completes( - sdk: NeMoPlatform, - workspace: str, - nss_model_filesets: None, -) -> None: - """Submit a GPU container job and verify synthetic data is produced.""" - num_records = int(os.environ.get("NSS_E2E_NUM_RECORDS", str(_DEFAULT_NUM_RECORDS))) - if num_records < _MIN_INPUT_ROWS: - raise ValueError(f"NSS_E2E_NUM_RECORDS must be at least {_MIN_INPUT_ROWS}, got {num_records}") - - job = ( - SafeSynthesizerJobBuilder(sdk, workspace=workspace) - .with_data_source(_synthesis_dataset()) - .synthesize() - .with_generate(num_records=num_records) - .with_evaluate(enabled=True) - .create_job() - ) - - nss_job = SafeSynthesizerJob(job.job_name, sdk, workspace=workspace) - nss_job.wait_for_completion(poll_interval=15, verbose=True) - - summary = nss_job.fetch_summary() - assert summary.timing.training_time_sec is not None - assert summary.timing.generation_time_sec is not None - - synthetic = nss_job.fetch_data() - assert len(synthetic) == num_records diff --git a/packages/nemo_evaluator_sdk/examples/profbench/profbench.py b/packages/nemo_evaluator_sdk/examples/profbench/profbench.py index 888dee118d..84f71e9a07 100644 --- a/packages/nemo_evaluator_sdk/examples/profbench/profbench.py +++ b/packages/nemo_evaluator_sdk/examples/profbench/profbench.py @@ -334,7 +334,7 @@ async def _score(self, input: MetricInput) -> ProfBenchRubricDetails: task = input.row.data.get("task", {}) judge_request = ProfBenchJudgeRequest( task_id=str(task.get("id", "")) if isinstance(task, dict) else "", - prompt=str(inputs.get("instruction", "")) if isinstance(inputs, dict) else "", + prompt=str(inputs.get("prompt", "")) if isinstance(inputs, dict) else "", response=output_text, criterion_id=criterion.id, criterion_description=criterion.description, @@ -447,7 +447,7 @@ def load_profbench( task = AgentEvalTask( id=task_id, intent=str(row["prompt"]), - inputs={"instruction": row["prompt"], "domain": row.get("domain")}, + inputs={"prompt": row["prompt"], "domain": row.get("domain")}, metrics=[ProfBenchRubricMetric(criteria=criteria, judge=judge, evidence_dir=evidence_dir)], metadata={ "benchmark": "ProfBench", diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py index f47e160c24..037bdadd37 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/evaluator.py @@ -628,12 +628,10 @@ def _default_prompt_template(target: Model | Agent) -> dict[str, Any]: def _task_row(task: AgentEvalTask) -> dict[str, Any]: - # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}` - # template): a dataset `prompt` column or the agent `instruction`. return { **task.inputs, "task_id": task.id, - "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"), + "prompt": task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent, } diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py index 4e7fb6ce04..c75d402f9d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/codex/runtime.py @@ -68,7 +68,7 @@ def __init__( self._work_root = Path(work_root).expanduser() if work_root is not None else None self._codex_bin = codex_bin self._timeout_s = timeout_s - self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt + self._prompt_builder = prompt_builder or default_codex_prompt self._process_factory = process_factory or asyncio.create_subprocess_exec self._runtime_name = runtime_name @@ -95,17 +95,15 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC evidence_dir.mkdir(parents=True, exist_ok=True) workspace_dir.mkdir(parents=True, exist_ok=True) + prompt = self._prompt_builder(task) prompt_path = evidence_dir / "prompt.txt" task_path = evidence_dir / "task.json" stdout_path = evidence_dir / "stdout.jsonl" stderr_path = evidence_dir / "stderr.txt" final_output_path = evidence_dir / "final_output.txt" - # Persist the task for debugging, but never the grader-only fields: the docker variant mounts - # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired - # behavior) or `reference` (held-out ground truth) here would let the agent read them back out - # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes. - task_path.write_text(task.model_dump_json(indent=2, exclude={"intent", "reference"}), encoding="utf-8") + prompt_path.write_text(prompt, encoding="utf-8") + task_path.write_text(task.model_dump_json(indent=2), encoding="utf-8") command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path) process: Any | None = None @@ -115,10 +113,6 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Build the prompt after seeding and inside the guarded block: an instruction-less task - # raises here, failing just this task instead of aborting the run (and seeding wins if both). - prompt = self._prompt_builder(task) - prompt_path.write_text(prompt, encoding="utf-8") process = await self._process_factory( *command, stdin=subprocess.PIPE, @@ -392,6 +386,30 @@ def print_codex_agent_models(*, codex_bin: str = "codex") -> None: print(slug) +def default_codex_prompt(task: AgentEvalTask) -> str: + """Frame a task for Codex as an agent that works in its current directory. + + Task-agnostic: it states the intent and inputs and invites the agent to read/create/edit files, + rather than constraining the answer to a single text reply. Seed files (``inputs[SEED_FILES_INPUT_KEY]``) + are listed by name instead of dumped inline — the agent finds them already in its workspace. Pass a + custom :data:`CodexPromptBuilder` to the runtime to override this framing for a specific benchmark. + """ + body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY} + lines = [f"Task id: {task.id}", f"Intent: {task.intent}"] + if body_inputs: + lines += ["", "Inputs:", json.dumps(body_inputs, indent=2, default=str)] + seeded = task.inputs.get(SEED_FILES_INPUT_KEY) + if isinstance(seeded, Mapping) and seeded: + lines += ["", "These files are already in your working directory:"] + lines += [f" - {path}" for path in seeded] + lines += [ + "", + "Complete the task by working in your current directory. You may read, create, and edit files " + "as needed. When you are done, briefly summarize what you changed.", + ] + return "\n".join(lines) + "\n" + + def _failed_codex_trial( task: AgentEvalTask, evidence_dir: Path, diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py index 489db70355..43354d00b9 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/docker_sandbox.py @@ -136,15 +136,13 @@ async def _run_task( ) -> AgentEvalTrial: evidence_dir = self._evidence_dir(index, task, config) evidence_dir.mkdir(parents=True, exist_ok=True) + prompt = _task_prompt(task) + manifest = self._build_manifest(task, sdk) + agent = self._build_agent(manifest, sdk) client = self._build_client(sdk) sandbox = None try: - # Build the prompt inside the guarded block: an instruction-less task raises here and fails - # just this task rather than aborting the whole run. - prompt = task.agent_prompt() - manifest = self._build_manifest(task, sdk) - agent = self._build_agent(manifest, sdk) sandbox = await client.create( manifest=manifest, options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE), @@ -165,7 +163,7 @@ def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. entries: dict[str, Any] = { - "instruction.md": sdk.File(content=task.agent_prompt().encode("utf-8")), + "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), "output": sdk.Dir(), } workspace_dir = task.inputs.get("workspace_dir") @@ -297,6 +295,10 @@ def _validated_workspace_dir(workspace_dir: Any) -> Path: return resolved +def _task_prompt(task: AgentEvalTask) -> str: + return str(task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent) + + async def _maybe_await(value: Awaitable[Any] | Any) -> Any: if inspect.isawaitable(value): return await value diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py index 5d21d3fc9c..cc41c5d99d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/runtimes/fabric/runtime.py @@ -7,11 +7,7 @@ NeMo Fabric Python SDK and adapts each normalized Fabric ``RunResult`` into an :class:`AgentEvalTrial`. The harness is chosen by the supplied Fabric config's ``harness.adapter_id`` (never inferred from a model); an optional ``model`` slug -is applied as the config's default model, mirroring Fabric's own Harbor integration. - -Per-task settings (workspace, model, trajectory capture) are composed directly onto -a copy of the supplied config via the SDK's config helpers (``model_copy`` + -``enable_relay`` + ``environment``), rather than layered as profile overlays. +is applied as a final profile overlay, mirroring Fabric's own Harbor integration. Every task runs in its own fresh workspace: the runtime seeds it from ``inputs['files']`` (a no-op when there are none), runs the harness in it (via @@ -42,7 +38,6 @@ CandidateEvidence, EvidenceDescriptor, ) -from pydantic import JsonValue if TYPE_CHECKING: # Annotations use nemo_fabric's real types (single source of truth). nemo_fabric is an optional @@ -50,10 +45,9 @@ # loaded lazily at runtime (see ``run_tasks``). Drop the ty:ignore once nemo-fabric is a # resolvable dependency and the checker can see it. from nemo_fabric import ( # ty: ignore[unresolved-import] - Fabric, + FabricClient, FabricConfig, FabricProfileConfig, - RunOutput, RunResult, ) @@ -72,18 +66,18 @@ # Per-task workspace: where seed files are staged and where the harness reads/writes. We # create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence. _WORKSPACE_SUBDIR = "workspace" +_WORKSPACE_PROFILE_NAME = "eval_workspace" # Evidence key + descriptor kind for the staged workspace, consumed by the # workspace-reading metrics. _WORKSPACE_EVIDENCE_KEY = "workspace" _WORKSPACE_EVIDENCE_KIND = "filesystem" -# File-exporter output names we choose for the Relay ATIF/ATOF trajectory (Relay accepts these as inputs). +# Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as inputs). +_TRAJECTORY_PROFILE_NAME = "eval_trajectory" _ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" _ATOF_FILENAME = "events.atof.jsonl" -# Names for the trailing overlays that re-assert the evaluator-owned per-task settings (see -# ``_eval_lock_profiles``): Fabric applies caller profiles over the config, so these must trail them. -_WORKSPACE_PROFILE_NAME = "eval_workspace" -_MODEL_PROFILE_NAME = "eval_model" -_ARTIFACTS_PROFILE_NAME = "eval_artifacts" +# Fabric telemetry-profile selectors (file exporter, no OTLP endpoint). +_TELEMETRY_PROVIDER = "relay" +_TELEMETRY_MODE = "sdk" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" @@ -127,77 +121,69 @@ async def run_tasks( try: # nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported # lazily here rather than at module load. - from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import FabricClient, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_FABRIC_MSG) from exc resolved_config = config or AgentEvalRunConfig() agent_config = FabricConfig.from_mapping(self._config) - # Fail fast (once) if trajectory capture is requested but the nemo-relay gateway isn't - # importable, rather than failing every task the same way inside the per-task guard. - if self._capture_trajectory: - try: - import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_RELAY_MSG) from exc - # Caller-supplied profile overlays pass through as-is; this runtime's per-task workspace, model, - # and trajectory settings are composed directly onto a copy of the config (config-first), not - # layered as profiles. - base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles] + base_profiles = self._build_profiles(FabricProfileConfig) semaphore = asyncio.Semaphore(resolved_config.parallelism) - # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle - # context manager — so it is created once and reused across tasks with no cleanup. - client = Fabric() + async with FabricClient() as client: - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(client, agent_config, base_profiles, index, task, resolved_config) + async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + return await self._run_task( + client, agent_config, base_profiles, FabricProfileConfig, index, task, resolved_config + ) - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) + return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) async def _run_task( self, - client: Fabric, + client: FabricClient, agent_config: FabricConfig, base_profiles: list[FabricProfileConfig], + profile_cls: type[FabricProfileConfig], index: int, task: AgentEvalTask, config: AgentEvalRunConfig, ) -> AgentEvalTrial: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import FabricProfileConfig, RunRequest # ty: ignore[unresolved-import] - evidence_dir = self._evidence_dir(index, task, config) evidence_dir.mkdir(parents=True, exist_ok=True) + profiles = list(base_profiles) + if self._capture_trajectory: + # Enable Relay's ATIF file exporter, writing the trajectory under this task's durable + # evidence dir; Fabric promotes the resulting file into RunResult.artifacts. Both the + # Fabric artifact root and the relay output dir must exist and be durable. + relay_dir = evidence_dir / _RELAY_SUBDIR + artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR + relay_dir.mkdir(parents=True, exist_ok=True) + artifacts_dir.mkdir(parents=True, exist_ok=True) + profiles.append(self._trajectory_profile(profile_cls, relay_dir=relay_dir, artifacts_dir=artifacts_dir)) + # Every task runs in its own fresh workspace: seed any ``inputs['files']`` into it (a no-op when - # there are none), point the harness at it, and expose it as ``workspace`` filesystem evidence — - # a uniform per-task dir that maps cleanly onto a per-task container volume later. Seeding runs - # inside the guarded block so a bad seed (a path escaping the workspace, an unresolvable fileset) - # fails just this task, not the whole run; it is synchronous and may block (a fileset handler - # downloads), so it is offloaded off the shared event loop. + # there are none), point the harness at it via ``environment.workspace``, and expose it as + # ``workspace`` filesystem evidence — a uniform per-task dir that maps cleanly onto a per-task + # container volume later. Seeding runs inside the guarded block so a bad seed (a path escaping + # the workspace, an unresolvable fileset) fails just this task, not the whole run; it is + # synchronous and may block (a fileset handler downloads), so it is offloaded off the shared + # event loop. workspace_dir = evidence_dir / _WORKSPACE_SUBDIR workspace_dir.mkdir(parents=True, exist_ok=True) try: - # Stage seed files into the workspace for their on-disk side effect; the prompt is the task - # instruction only, so the returned paths are unused. - await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - task_config = self._compose_config(agent_config, evidence_dir, workspace_dir) - # Caller ``base_profiles`` are applied by Fabric over the config; the evaluator-owned - # settings are re-asserted as trailing overlays so they win over any caller profile. - lock_profiles = self._eval_lock_profiles( - FabricProfileConfig, workspace_dir=workspace_dir, evidence_dir=evidence_dir - ) + seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) + profiles.append(self._workspace_profile(profile_cls, workspace_dir=workspace_dir)) result = await asyncio.wait_for( - # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``. client.run( - task_config, - profiles=[*base_profiles, *lock_profiles], + agent_config, + profiles=profiles, + input=_fabric_input(task, seeded_files), + request_id=task.id, base_dir=self._base_dir, - request=RunRequest(input=task.agent_prompt(), request_id=task.id), ), timeout=self._timeout_s, ) @@ -228,17 +214,13 @@ def _to_trial( if result.status != "succeeded": return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) - # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), - # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the - # trial's ``JsonValue``-typed response. - output = _normalize_output(result.output) return AgentEvalTrial( id=f"{task.id}:fabric", task_id=task.id, status=AgentEvalTrialStatus.COMPLETED, output=AgentOutput( - output_text=_extract_output_text(output), - response=output, + output_text=_extract_output_text(result.output), + response=result.output, metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, ), evidence=self._evidence(result, result_path, workspace_dir), @@ -315,90 +297,31 @@ def _failed_trial( }, ) - def _compose_config( - self, - agent_config: FabricConfig, - evidence_dir: Path, - workspace_dir: Path, - ) -> FabricConfig: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import EnvironmentConfig # ty: ignore[unresolved-import] - - # Config-first composition (the SDK's recommended in-memory pattern): copy the base config and - # apply this task's workspace, model, and trajectory settings directly onto it, rather than - # layering FabricProfileConfig overlays. - cfg = agent_config.model_copy(deep=True) - - # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from - # it). ``provider="local"`` is required by the native planner. Any config-supplied - # environment.workspace is overridden per task. - environment = cfg.environment or EnvironmentConfig(provider="local") - environment.provider = environment.provider or "local" - environment.workspace = str(workspace_dir) - cfg.environment = environment - - # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor). + def _build_profiles(self, profile_cls: type[FabricProfileConfig]) -> list[FabricProfileConfig]: + profiles = [profile_cls.from_mapping(profile) for profile in self._profiles] if self._model: + # Apply the model as a final profile overlay (mirrors nemo_fabric.integrations.harbor). provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - cfg.models["default"] = {"provider": provider, "model": self._model} - - if self._capture_trajectory: - # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the - # Fabric artifact root so the promoted ``trajectory-*.atif.json`` persists. Requires the - # ``nemo-relay`` gateway on PATH in the runtime. - relay_dir = evidence_dir / _RELAY_SUBDIR - artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR - relay_dir.mkdir(parents=True, exist_ok=True) - artifacts_dir.mkdir(parents=True, exist_ok=True) - cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir)) - cfg.runtime.artifacts = str(artifacts_dir) - cfg.environment.artifacts = str(artifacts_dir) - - return cfg - - def _eval_lock_profiles( - self, - profile_cls: type[FabricProfileConfig], - *, - workspace_dir: Path, - evidence_dir: Path, - ) -> list[FabricProfileConfig]: - # ``_compose_config`` composes the evaluator's per-task settings onto the config, but Fabric - # applies caller-supplied profiles OVER the config (last-wins), so a caller profile could - # otherwise override them. Re-assert the evaluator-owned settings here as trailing overlays — - # applied after the caller profiles — so the per-task workspace (isolation + ``workspace`` - # evidence integrity), the model under evaluation, and the trajectory artifact location stay - # authoritative and non-overridable. - overlays = [ - profile_cls.from_mapping( - {"name": _WORKSPACE_PROFILE_NAME, "environment": {"workspace": str(workspace_dir)}} - ) - ] - if self._model: - provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - overlays.append( - profile_cls.from_mapping( - {"name": _MODEL_PROFILE_NAME, "models": {"default": {"provider": provider, "model": self._model}}} + profiles.append( + profile_cls( + name="eval_model", + models={"default": {"provider": provider, "model": self._model}}, ) ) - if self._capture_trajectory: - artifacts_dir = str(evidence_dir / _ARTIFACTS_SUBDIR) - overlays.append( - profile_cls.from_mapping( - { - "name": _ARTIFACTS_PROFILE_NAME, - "runtime": {"artifacts": artifacts_dir}, - "environment": {"artifacts": artifacts_dir}, - } - ) - ) - return overlays - - def _relay_config(self, relay_dir: Path) -> dict[str, Any]: + return profiles + + def _trajectory_profile( + self, profile_cls: type[FabricProfileConfig], *, relay_dir: Path, artifacts_dir: Path + ) -> FabricProfileConfig: + # Relay ATIF/ATOF file exporter (mode=sdk): the harness emits its trajectory to a local + # nemo-relay gateway, which writes ``trajectory-*.atif.json`` under ``relay_dir``. No OTLP + # collector endpoint is involved. Requires the ``nemo-relay`` gateway on PATH in the runtime. + # The Fabric artifact root is pinned to a durable dir so the promoted trajectory persists. + # # The observability component is built from nemo_relay's own typed config objects so Relay owns # its schema (no hand-maintained dict that silently drifts when Relay changes it); imported - # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. + # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. ``schema_version`` + # is omitted — ``FabricProfileConfig`` defaults it. try: from nemo_relay.observability import ( # ty: ignore[unresolved-import] AtifConfig, @@ -410,6 +333,7 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]: raise RuntimeError(_MISSING_RELAY_MSG) from exc relay_dir_str = str(relay_dir) + artifacts_dir_str = str(artifacts_dir) observability = ComponentSpec( config=ObservabilityConfig( atif=AtifConfig( @@ -427,7 +351,33 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]: ), ) ) - return {"version": 1, "components": [observability.to_dict()]} + return profile_cls.from_mapping( + { + "name": _TRAJECTORY_PROFILE_NAME, + "description": "Capture the agent trajectory as ATIF via the NeMo Relay file exporter.", + "runtime": {"artifacts": artifacts_dir_str}, + "environment": {"artifacts": artifacts_dir_str}, + "telemetry": { + "enabled": True, + "provider": _TELEMETRY_PROVIDER, + "mode": _TELEMETRY_MODE, + "output_dir": relay_dir_str, + "config": {"version": 1, "components": [observability.to_dict()]}, + }, + } + ) + + def _workspace_profile(self, profile_cls: type[FabricProfileConfig], *, workspace_dir: Path) -> FabricProfileConfig: + # Point the harness at this task's staged workspace via ``environment.workspace`` (the codex-cli + # adapter resolves its cwd from it). Set as a final profile overlay, the same mechanism the + # trajectory profile uses for ``environment.artifacts``. + return profile_cls.from_mapping( + { + "name": _WORKSPACE_PROFILE_NAME, + "description": "Run the harness in the per-task evaluation workspace seeded with the task inputs.", + "environment": {"workspace": str(workspace_dir)}, + } + ) def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root @@ -438,16 +388,22 @@ def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunCon return Path(root) / task_dir -def _normalize_output(output: RunOutput | JsonValue) -> JsonValue: - """Unwrap a Fabric ``RunResult.output`` into the plain JSON value the trial response stores. +def _fabric_input(task: AgentEvalTask, seeded_files: Sequence[str] = ()) -> str: + """Frame the task as the harness's input text. - Newer Fabric wraps output in a ``RunOutput`` (the RunOutput response contract), which is a - ``Mapping``; copy it into a plain dict (equivalent to its ``to_mapping()``). Raw/older JSON outputs - are already JSON values and pass through unchanged. + When files were staged into the workspace, list them by name and invite the agent to work in its + current directory rather than dumping their contents inline; the seed-files key is dropped from + the echoed inputs since those files are already on disk. """ - if isinstance(output, Mapping): - return dict(output) - return output + body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY} + lines = [f"Task id: {task.id}", f"Intent: {task.intent}"] + if body_inputs: + lines += ["", f"Inputs: {body_inputs}"] + if seeded_files: + lines += ["", "These files are already in your working directory:"] + lines += [f" - {path}" for path in seeded_files] + lines += ["", "Complete the task by reading, creating, and editing files in your current directory."] + return "\n".join(lines) + "\n" def _extract_output_text(output: object) -> str | None: diff --git a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py index 83e5d64520..7d1a23ee7d 100644 --- a/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py +++ b/packages/nemo_evaluator_sdk/src/nemo_evaluator_sdk/agent_eval/tasks.py @@ -97,23 +97,6 @@ def _id_must_not_be_empty(cls, value: str) -> str: raise ValueError("task id must not be empty") return value - def agent_prompt(self) -> str: - """The intent-free prompt handed to the agent under evaluation. - - Exactly the task's natural-language instruction (``inputs["instruction"]``), with no - runtime-added framing. ``intent`` is deliberately never used: it is the eval-side description - of the desired behavior (what the grader checks for), so exposing it to the agent is a - reward-hacking hole. - - Raises ``ValueError`` when ``inputs["instruction"]`` is missing or empty; a task with no - instruction cannot be evaluated, so the runner fails that task rather than running an agent on - an empty prompt. - """ - instruction = self.inputs.get("instruction") - if instruction: - return str(instruction) - raise ValueError(f"task {self.id!r} has no instruction: set inputs['instruction']") - @field_serializer("metrics", when_used="json") def _serialize_metrics(self, metrics: list[Metric]) -> list[dict[str, Any]]: """Serialize local metric instances as descriptors for run bundles.""" diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py index cc00788024..c78f50f7c9 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_codex_runtime.py @@ -111,10 +111,9 @@ def __init__(self, command: tuple[str, ...]) -> None: self.command = command async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - # Default prompt is exactly the instruction from inputs — no runtime framing — and never - # leaks the eval-side `intent`. - assert input == b"Question?" - assert b"Answer." not in input # intent stays eval-side + # Default prompt is task-agnostic: it states the task and invites workspace edits. + assert b"Task id: task/1" in input + assert b"Intent:" in input final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) final_output_path.write_text("codex answer", encoding="utf-8") return b'{"type":"event"}\n', b"" @@ -129,7 +128,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: work_root=tmp_path / "codex", process_factory=fake_process_factory, ) - task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Question?"}) + task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"prompt": "Question?"}) trials = await runtime.run_tasks([task]) @@ -151,44 +150,6 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: assert (tmp_path / "codex" / "000000-task-1" / "stdout.jsonl").read_text(encoding="utf-8") == '{"type":"event"}\n' -@pytest.mark.asyncio -async def test_codex_task_json_omits_grader_only_fields(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - # The docker variant mounts the evidence dir into the sandbox (danger-full-access), so the persisted - # task.json must never carry grader-only fields — otherwise the agent could read `intent` (desired - # behavior) or the held-out `reference` back out of /evidence and reward-hack. Enforced on the shared - # base runtime so both the local and docker variants persist an agent-safe task.json. - class FakeProcess: - returncode = 0 - - def __init__(self, command: tuple[str, ...]) -> None: - self.command = command - - async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) - final_output_path.write_text("ok", encoding="utf-8") - return b"", b"" - - async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: - return FakeProcess(command) - - monkeypatch.setattr(codex_runtime.shutil, "which", lambda value: f"/bin/{value}") - runtime = codex_runtime.CodexCliAgentRuntime(work_root=tmp_path / "codex", process_factory=fake_process_factory) - task = AgentEvalTask( - id="task/1", - intent="SECRET_GRADER_INTENT", - inputs={"instruction": "do the thing"}, - reference={"expected": "HELD_OUT_GROUND_TRUTH"}, - ) - - await runtime.run_tasks([task]) - - task_json = (tmp_path / "codex" / "000000-task-1" / "task.json").read_text(encoding="utf-8") - assert "SECRET_GRADER_INTENT" not in task_json # intent is eval-side desired-behavior metadata - assert "HELD_OUT_GROUND_TRUTH" not in task_json # reference is grader-only ground truth - assert '"intent"' not in task_json and '"reference"' not in task_json # dropped entirely, not just empty - assert "do the thing" in task_json # agent-safe fields (id, inputs) are still persisted - - @pytest.mark.asyncio async def test_codex_docker_cli_agent_runtime_runs_codex_in_container_and_writes_evidence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -204,7 +165,7 @@ def __init__(self, command: tuple[str, ...]) -> None: self.command = command async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - assert input == b"Question?" # prompt is the instruction verbatim + assert b"Task id: task/1" in input evidence_mount = self.command[self.command.index(f"{auth_path.resolve()}:/root/.codex/auth.json:ro") + 4] evidence_dir = Path(evidence_mount.split(":/evidence", maxsplit=1)[0]) (evidence_dir / "final_output.txt").write_text("docker codex answer", encoding="utf-8") @@ -221,7 +182,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: auth_path=auth_path, process_factory=fake_process_factory, ) - task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Question?"}) + task = AgentEvalTask(id="task/1", intent="Answer.", inputs={"prompt": "Question?"}) trials = await runtime.run_tasks([task]) @@ -277,7 +238,7 @@ async def fake_wait_for(awaitable: Any, timeout: float) -> Any: work_root=tmp_path / "codex", process_factory=fake_process_factory, ) - task = AgentEvalTask(id="task-timeout", intent="Answer.", inputs={"instruction": "Q?"}) + task = AgentEvalTask(id="task-timeout", intent="Answer.", inputs={"prompt": "Q?"}) trials = await runtime.run_tasks([task]) @@ -305,7 +266,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: work_root=tmp_path / "codex", process_factory=fake_process_factory, ) - task = AgentEvalTask(id="task-2", intent="Answer.", inputs={"instruction": "Q?"}) + task = AgentEvalTask(id="task-2", intent="Answer.", inputs={"prompt": "Q?"}) trials = await runtime.run_tasks([task]) @@ -327,9 +288,10 @@ def __init__(self, command: tuple[str, ...]) -> None: self.command = command async def communicate(self, input: bytes) -> tuple[bytes, bytes]: - # Seed files are staged into the workspace before the agent runs. + # Seed files are staged before the agent runs and listed (by name) in the prompt. workspace_dir = Path(self.command[self.command.index("--cd") + 1]) assert (workspace_dir / "buggy.py").read_text(encoding="utf-8") == "def add(a, b)\n return a + b\n" + assert b"buggy.py" in input final_output_path = Path(self.command[self.command.index("--output-last-message") + 1]) final_output_path.write_text("fixed it", encoding="utf-8") return b"", b"" @@ -345,7 +307,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: task = AgentEvalTask( id="fix-bug", intent="Fix the syntax error.", - inputs={"instruction": "fix the bug", "files": {"buggy.py": "def add(a, b)\n return a + b\n"}}, + inputs={"files": {"buggy.py": "def add(a, b)\n return a + b\n"}}, ) trials = await runtime.run_tasks([task]) @@ -399,9 +361,7 @@ async def fake_process_factory(*command: str, **kwargs: Any) -> FakeProcess: work_root=tmp_path / "codex", process_factory=fake_process_factory, ) - task = AgentEvalTask( - id="probe", intent="probe", inputs={"instruction": "run", "files": {"p.txt": {"kind": "thread_probe"}}} - ) + task = AgentEvalTask(id="probe", intent="probe", inputs={"files": {"p.txt": {"kind": "thread_probe"}}}) trials = await runtime.run_tasks([task]) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py index ad62495708..54d41c3431 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_docker_sandbox_runtime.py @@ -165,10 +165,13 @@ def _fake_sdk() -> SandboxSDK: def _task( *, task_id: str = "task-1", - instruction: str | None = "Instruction text.", + prompt: str | None = "Prompt text.", + instruction: str | None = None, workspace_dir: Path | None = None, ) -> AgentEvalTask: inputs: dict[str, Any] = {} + if prompt is not None: + inputs["prompt"] = prompt if instruction is not None: inputs["instruction"] = instruction if workspace_dir is not None: @@ -200,23 +203,22 @@ def fake_import( docker_sandbox._load_agents_sdk() -def test_manifest_uses_instruction_and_never_leaks_intent() -> None: - # The instruction is surfaced verbatim; `task.intent` is eval-side metadata and must never leak to - # the agent (reward-hacking hole), so it is never a fallback. +@pytest.mark.parametrize( + ("task", "expected_prompt"), + [ + (_task(prompt="Prompt text.", instruction="Instruction text."), "Prompt text."), + (_task(prompt=None, instruction="Instruction text."), "Instruction text."), + (_task(prompt=None, instruction=None), "Intent text."), + ], +) +def test_manifest_uses_prompt_instruction_intent_fallback( + task: AgentEvalTask, + expected_prompt: str, +) -> None: runtime = DockerSandboxAgentRuntime() - manifest = runtime._build_manifest(_task(instruction="Instruction text."), _fake_sdk()) - - content = manifest.entries["instruction.md"].content.decode("utf-8") - assert content == "Instruction text." - assert "Intent text." not in content - + manifest = runtime._build_manifest(task, _fake_sdk()) -def test_manifest_raises_when_task_has_no_instruction() -> None: - # A task with no instruction cannot be evaluated; building its manifest raises rather than - # producing an empty prompt (and `task.intent` must never leak as a fallback). - runtime = DockerSandboxAgentRuntime() - with pytest.raises(ValueError, match="no instruction"): - runtime._build_manifest(_task(instruction=None), _fake_sdk()) + assert manifest.entries["instruction.md"].content.decode("utf-8") == expected_prompt def test_manifest_maps_workspace_dir_to_local_dir(tmp_path: Path) -> None: @@ -236,7 +238,7 @@ def test_manifest_omits_serialized_task_to_avoid_leaking_grader_fields() -> None task = AgentEvalTask( id="task-1", intent="Intent text.", - inputs={"instruction": "Instruction text."}, + inputs={"prompt": "Prompt text."}, reference={"test_calculator.py": "def test_add(): assert add(2, 3) == 5"}, ) diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py index 64af6626a1..39870b6020 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_integration.py @@ -14,7 +14,6 @@ from __future__ import annotations -import copy import importlib.util import json import os @@ -69,45 +68,11 @@ def _task() -> AgentEvalTask: # --- hermetic: fake nemo_fabric so CI exercises the runner+evaluator+metric+evidence chain ---------- -class _FakeEnvironment: - def __init__(self, *, provider: str = "local", workspace: str | None = None, artifacts: str | None = None) -> None: - self.provider = provider - self.workspace = workspace - self.artifacts = artifacts - - -class _FakeRuntimeCfg: - def __init__(self, artifacts: str | None = None) -> None: - self.artifacts = artifacts - - class _FakeConfig: - """Stand-in for nemo_fabric.FabricConfig supporting the config-first helpers the runtime uses.""" - - def __init__(self) -> None: - self.environment: _FakeEnvironment | None = None - self.runtime = _FakeRuntimeCfg() - self.models: dict[str, Any] = {} - self.relay: dict[str, Any] | None = None - @classmethod def from_mapping(cls, mapping: dict[str, Any]) -> _FakeConfig: return cls() - def model_copy(self, *, deep: bool = False) -> _FakeConfig: - clone = _FakeConfig() - clone.environment = copy.deepcopy(self.environment) - clone.runtime = _FakeRuntimeCfg(self.runtime.artifacts) - clone.models = copy.deepcopy(self.models) - clone.relay = copy.deepcopy(self.relay) - return clone - - def enable_relay( - self, *, project: str | None = None, output_dir: str | None = None, config: Any = None - ) -> _FakeConfig: - self.relay = {"project": project, "output_dir": output_dir, "config": config} - return self - class _FakeProfile: def __init__(self, **kwargs: Any) -> None: @@ -159,20 +124,19 @@ async def test_fabric_runner_eval_exposes_trajectory_to_metric(tmp_path: Path, m (tmp_path / "out.txt").write_text("DONE\n", encoding="utf-8") class _FakeClient: - # Fabric is a plain reusable facade (not an async context manager). + async def __aenter__(self) -> _FakeClient: + return self + + async def __aexit__(self, *exc: Any) -> bool: + return False + async def run(self, agent: Any, **kwargs: Any) -> _FakeResult: return _FakeResult(artifacts) - class _FakeRunRequest: - def __init__(self, **kwargs: Any) -> None: - self.__dict__.update(kwargs) - module = types.ModuleType("nemo_fabric") - module.Fabric = _FakeClient # type: ignore[attr-defined] + module.FabricClient = _FakeClient # type: ignore[attr-defined] module.FabricConfig = _FakeConfig # type: ignore[attr-defined] module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined] - module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined] - module.RunRequest = _FakeRunRequest # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "nemo_fabric", module) # Trajectory capture builds the profile from nemo_relay's typed config objects (lazy import); stub diff --git a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py index 5805cc9425..cc5d97b759 100644 --- a/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py +++ b/packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_runtime.py @@ -5,11 +5,9 @@ from __future__ import annotations -import copy import json import sys import types -from collections.abc import Iterator, Mapping from pathlib import Path from typing import Any @@ -19,48 +17,14 @@ from nemo_evaluator_sdk.values.evidence import EVIDENCE_FORMAT_ATIF, EVIDENCE_TRACE -class _FakeEnvironment: - """Stand-in for nemo_fabric.EnvironmentConfig (the runtime sets workspace/provider/artifacts).""" - - def __init__(self, *, provider: str = "local", workspace: str | None = None, artifacts: str | None = None) -> None: - self.provider = provider - self.workspace = workspace - self.artifacts = artifacts - - -class _FakeRuntimeCfg: - def __init__(self, artifacts: str | None = None) -> None: - self.artifacts = artifacts - - class _FakeConfig: - """Stand-in for nemo_fabric.FabricConfig with the config-first helpers the runtime composes onto.""" - def __init__(self, mapping: dict[str, Any]) -> None: self.mapping = mapping - self.environment: _FakeEnvironment | None = None - self.runtime = _FakeRuntimeCfg() - self.models: dict[str, Any] = dict(mapping.get("models", {})) - self.relay: dict[str, Any] | None = None # records enable_relay(...) @classmethod def from_mapping(cls, mapping: dict[str, Any]) -> _FakeConfig: return cls(mapping) - def model_copy(self, *, deep: bool = False) -> _FakeConfig: - clone = _FakeConfig(self.mapping) - clone.environment = copy.deepcopy(self.environment) - clone.runtime = _FakeRuntimeCfg(self.runtime.artifacts) - clone.models = copy.deepcopy(self.models) - clone.relay = copy.deepcopy(self.relay) - return clone - - def enable_relay( - self, *, project: str | None = None, output_dir: str | None = None, config: Any = None - ) -> _FakeConfig: - self.relay = {"project": project, "output_dir": output_dir, "config": config} - return self - class _FakeProfile: def __init__(self, *, name: str | None = None, models: Any = None, mapping: Any = None) -> None: @@ -73,14 +37,6 @@ def from_mapping(cls, mapping: dict[str, Any]) -> _FakeProfile: return cls(name=mapping.get("name"), mapping=mapping) -class _FakeRunRequest: - """Stand-in for nemo_fabric.RunRequest (Fabric.run folds input + request id into it).""" - - def __init__(self, *, input: Any = None, request_id: str | None = None) -> None: - self.input = input - self.request_id = request_id - - class _FakeRelayConfig: """Stand-in for nemo_relay.observability's typed config objects (AtifConfig/AtofConfig/...).""" @@ -163,20 +119,23 @@ def _install_fake_fabric(monkeypatch: pytest.MonkeyPatch, handler: Any) -> type: """Inject a fake ``nemo_fabric`` module (the runtime imports it lazily); return the client class.""" class _FakeClient: - # Fabric is a plain reusable facade (not an async context manager). recorded: list[dict[str, Any]] = [] + async def __aenter__(self) -> _FakeClient: + return self + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool: + return False + async def run(self, agent: Any, **kwargs: Any) -> Any: _FakeClient.recorded.append({"agent": agent, **kwargs}) return handler(agent, kwargs) _FakeClient.recorded = [] module = types.ModuleType("nemo_fabric") - module.Fabric = _FakeClient # type: ignore[attr-defined] + module.FabricClient = _FakeClient # type: ignore[attr-defined] module.FabricConfig = _FakeConfig # type: ignore[attr-defined] module.FabricProfileConfig = _FakeProfile # type: ignore[attr-defined] - module.EnvironmentConfig = _FakeEnvironment # type: ignore[attr-defined] - module.RunRequest = _FakeRunRequest # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, "nemo_fabric", module) # The runtime builds the trajectory profile from nemo_relay's typed config objects (lazy import); @@ -193,7 +152,7 @@ async def run(self, agent: Any, **kwargs: Any) -> Any: return _FakeClient -_TASK = AgentEvalTask(id="task/1", intent="Answer.", inputs={"instruction": "Ping?"}) +_TASK = AgentEvalTask(id="task/1", intent="Answer.", inputs={"prompt": "Ping?"}) _CONFIG = {"metadata": {"name": "a"}, "harness": {"adapter_id": "nvidia.fabric.codex.cli"}} @@ -231,44 +190,19 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: assert trial.evidence.descriptors["stdout"].ref == str(tmp_path / "stdout.txt") result_file = tmp_path / "fabric" / "000000-task-1" / "fabric_result.json" assert json.loads(result_file.read_text(encoding="utf-8"))["status"] == "succeeded" - # Config-first: the model is set on the config's default model and relay (ATIF trajectory) is - # enabled on the config, rather than layered as profile overlays. - composed = client_cls.recorded[0]["agent"] - assert composed.models["default"] == {"provider": "openai", "model": "openai/gpt-5.4"} - assert composed.relay is not None # capture_trajectory defaults on -> enable_relay(...) called - assert client_cls.recorded[0]["request"].request_id == "task/1" + # Model applied as a profile overlay (Harbor pattern) + the Relay ATIF trajectory overlay. + profiles = client_cls.recorded[0]["profiles"] + names = [p.name for p in profiles] + assert "eval_model" in names + assert "eval_trajectory" in names # capture_trajectory defaults on + model_profile = next(p for p in profiles if p.name == "eval_model") + assert model_profile.models == {"default": {"provider": "openai", "model": "openai/gpt-5.4"}} + assert client_cls.recorded[0]["request_id"] == "task/1" # Telemetry reference is preserved end-to-end (uri + trace_id), not just provider/kind. assert trial.evidence.metadata["telemetry"][0]["uri"] == "file:///relay" assert trial.evidence.metadata["telemetry"][0]["trace_id"] == "tid-1" -@pytest.mark.asyncio -async def test_fabric_runtime_prompt_excludes_intent_and_frames_inputs( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # The prompt is exactly the task instruction — no runtime framing. `task.intent` is eval-side - # "desired behavior" metadata and must never reach the agent (reward-hacking hole); other inputs - # keys are not templated into the prompt either. - task = AgentEvalTask( - id="task/2", - intent="SECRET_GRADER_INTENT", - inputs={"instruction": "Ping?", "files": {"data.csv": "s3://seed"}, "hint": "be terse"}, - ) - - def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: - return _FakeResult(status="succeeded", output="ok") - - client_cls = _install_fake_fabric(monkeypatch, handler) - runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric") - - await runtime.run_tasks([task]) - - prompt = client_cls.recorded[0]["request"].input - assert prompt == "Ping?" # the instruction verbatim, nothing else - assert "SECRET_GRADER_INTENT" not in prompt # intent stays eval-side - assert "be terse" not in prompt # non-instruction inputs are not templated in - - @pytest.mark.asyncio async def test_fabric_runtime_maps_atif_artifact_to_trace_evidence( tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -292,60 +226,10 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: assert trace.ref == str(tmp_path / "trajectory.atif.json") -def _workspace_from_config(config: Any) -> Path: - """Pull the staged workspace path out of the composed per-task config.""" - return Path(config.environment.workspace) - - -def _resolve_like_fabric(config: Any, profiles: list[Any], section: str, key: str) -> Any: - """Mirror Fabric's resolver: start from the config, then apply each profile as a winning overlay in - order (last wins). Used to assert what value actually reaches the harness for a config/profile key. - """ - if section == "environment": - value = getattr(config.environment, key, None) if config.environment is not None else None - elif section == "models": - value = config.models.get(key) - else: # pragma: no cover - only the two sections above are exercised - raise ValueError(section) - for profile in profiles: - overlay = getattr(profile, "mapping", None) - if isinstance(overlay, Mapping) and isinstance(overlay.get(section), Mapping): - if overlay[section].get(key) is not None: - value = overlay[section][key] - return value - - -@pytest.mark.asyncio -async def test_caller_profiles_cannot_override_evaluator_owned_settings( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - # Fabric applies caller-supplied profiles over the config (last-wins), so the evaluator's per-task - # workspace (isolation + `workspace` evidence integrity) and model-under-eval must remain the final, - # authoritative layer. A caller profile that sets these must NOT win. - caller_profile = { - "name": "caller", - "environment": {"workspace": "/caller/hijacked-workspace"}, - "models": {"default": {"provider": "openai", "model": "caller/rogue-model"}}, - } - - def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: - return _FakeResult(status="succeeded", output="ok") - - client_cls = _install_fake_fabric(monkeypatch, handler) - runtime = fabric_runtime.FabricAgentRuntime( - config=_CONFIG, model="openai/gpt-5.4", work_root=tmp_path / "fabric", profiles=[caller_profile] - ) - - await runtime.run_tasks([_TASK]) - - config = client_cls.recorded[0]["agent"] - profiles = client_cls.recorded[0]["profiles"] - eval_workspace = config.environment.workspace # the per-task dir the evaluator composed - eval_model = config.models["default"] - - # After Fabric applies the caller profile, the evaluator's workspace + model must still win. - assert _resolve_like_fabric(config, profiles, "environment", "workspace") == eval_workspace - assert _resolve_like_fabric(config, profiles, "models", "default") == eval_model +def _workspace_from_profiles(profiles: list[Any]) -> Path: + """Pull the staged workspace path out of the ``eval_workspace`` profile overlay.""" + profile = next(p for p in profiles if p.name == fabric_runtime._WORKSPACE_PROFILE_NAME) + return Path(profile.mapping["environment"]["workspace"]) @pytest.mark.asyncio @@ -360,7 +244,7 @@ async def test_fabric_runtime_seeds_workspace_and_exposes_workspace_evidence( def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: # The harness runs in the staged workspace; simulate an edit it leaves behind. - workspace = _workspace_from_config(agent) + workspace = _workspace_from_profiles(kwargs["profiles"]) (workspace / "result.txt").write_text("done", encoding="utf-8") return _FakeResult(status="succeeded", output={"response": "ok"}) @@ -371,10 +255,10 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: trial = trials[0] assert trial.status == "completed" - # The composed config carries environment.workspace (the harness's cwd) with provider=local. - composed = client_cls.recorded[0]["agent"] - assert composed.environment.provider == "local" - workspace = _workspace_from_config(composed) + # A dedicated workspace profile overlay carries environment.workspace (the harness's cwd). + profiles = client_cls.recorded[0]["profiles"] + assert fabric_runtime._WORKSPACE_PROFILE_NAME in [p.name for p in profiles] + workspace = _workspace_from_profiles(profiles) # The seed file is staged and the agent's edit is present in the same dir. assert (workspace / "calc.py").read_text(encoding="utf-8") == "value = 1\n" assert (workspace / "result.txt").read_text(encoding="utf-8") == "done" @@ -383,8 +267,9 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: workspace_evidence = trial.evidence.descriptors["workspace"] assert workspace_evidence.kind == "filesystem" assert workspace_evidence.ref == str(workspace) - # Seed-file contents are not inlined into the prompt (they are already on disk in the workspace). - harness_input = client_cls.recorded[0]["request"].input + # Seed-file contents are listed by name in the input, not dumped inline (they are already on disk). + harness_input = client_cls.recorded[0]["input"] + assert "calc.py" in harness_input assert "value = 1" not in harness_input @@ -402,7 +287,9 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: trials = await runtime.run_tasks([_TASK]) # _TASK has no 'files' input - workspace = _workspace_from_config(client_cls.recorded[0]["agent"]) + profiles = client_cls.recorded[0]["profiles"] + assert fabric_runtime._WORKSPACE_PROFILE_NAME in [p.name for p in profiles] + workspace = _workspace_from_profiles(profiles) assert workspace.is_dir() assert trials[0].evidence is not None assert trials[0].evidence.descriptors["workspace"].ref == str(workspace) @@ -430,7 +317,7 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: @pytest.mark.asyncio -async def test_fabric_runtime_capture_trajectory_false_skips_relay( +async def test_fabric_runtime_capture_trajectory_false_skips_relay_overlay( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: @@ -441,7 +328,8 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: await runtime.run_tasks([_TASK]) - assert client_cls.recorded[0]["agent"].relay is None + names = [p.name for p in client_cls.recorded[0]["profiles"]] + assert "eval_trajectory" not in names @pytest.mark.asyncio @@ -552,34 +440,3 @@ def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: assert result.outputs[0].name == "agent_phase_success" assert result.outputs[0].value is True - - -@pytest.mark.asyncio -async def test_fabric_runtime_normalizes_runoutput_response(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - # Newer Fabric wraps RunResult.output in a RunOutput Mapping (not a plain JSON value); the runtime - # must copy it into a plain dict so it round-trips through the trial's JsonValue response field. - class _FakeRunOutput(Mapping): - def __init__(self, data: dict[str, Any]) -> None: - self._data = dict(data) - - def __getitem__(self, key: str) -> Any: - return self._data[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._data) - - def __len__(self) -> int: - return len(self._data) - - def handler(agent: Any, kwargs: dict[str, Any]) -> _FakeResult: - return _FakeResult(status="succeeded", output=_FakeRunOutput({"response": "PONG", "returncode": 0})) - - _install_fake_fabric(monkeypatch, handler) - runtime = fabric_runtime.FabricAgentRuntime(config=_CONFIG, work_root=tmp_path / "fabric") - - trial = (await runtime.run_tasks([_TASK]))[0] - - assert trial.status == "completed" - assert trial.output is not None - assert trial.output.output_text == "PONG" # extracted from the normalized mapping - assert trial.output.response == {"response": "PONG", "returncode": 0} # plain dict, not RunOutput diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index d7bb978d9f..1147a71a50 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -351,7 +351,7 @@ nemo-safe-synthesizer-plugin = [ "gunicorn>=23.0.0", "httpx>=0.27.2", "nemo-platform-plugin", - "nemo-safe-synthesizer==0.1.7", + "nemo-safe-synthesizer==0.1.2", "pydantic[email]>=2.9.2", "pydantic-settings>=2.2.1", "python-multipart~=0.0.9", diff --git a/plugins/nemo-safe-synthesizer/constraints.txt b/plugins/nemo-safe-synthesizer/constraints.txt index ae8172735b..6f487f70d6 100644 --- a/plugins/nemo-safe-synthesizer/constraints.txt +++ b/plugins/nemo-safe-synthesizer/constraints.txt @@ -1,17 +1,17 @@ -# Security floor constraints -- generated from [tool.uv] constraint-dependencies in pyproject.toml -# Vendored from https://github.com/NVIDIA-NeMo/Safe-Synthesizer/blob/v0.1.7/constraints.txt -# on 2026-07-10. Local additions keep pyarrow/cryptography on CVE-fixed -# versions and the AWS SDK packages aligned with NeMo Platform's -# aiobotocore-compatible range. +# Vendored from https://github.com/NVIDIA-NeMo/Safe-Synthesizer/blob/v0.1.2/constraints.txt +# on 2026-06-30. Local additions keep the AWS SDK packages aligned with +# NeMo Platform's aiobotocore-compatible range. # # Pass to pip/uv with: pip install -c constraints.txt # uv pip install -c constraints.txt + +# Security floor constraints -- generated from [tool.uv] constraint-dependencies in pyproject.toml GitPython>=3.1.50 -PyJWT>=2.13.0 +PyJWT>=2.12.0 Pygments>=2.20.0 -aiohttp>=3.14.0 +aiohttp>=3.13.4 cbor2>=5.9.0 -cryptography>=48.0.1,<49 +cryptography>=46.0.7 grpcio>=1.80.0 idna>=3.15 jsonpath-ng>=1.8.0 @@ -21,14 +21,11 @@ mistune>=3.2.1 nbconvert>=7.17.1 pandas<3 pillow>=12.2.0 -pyarrow>=23.0.1,<24 pymdown-extensions>=10.21.3 python-dotenv>=1.2.2 python-multipart>=0.0.27 requests>=2.33.0 -starlette>=1.0.1 tornado>=6.5.5 -urllib3>=2.7.0 # Runtime compatibility constraints. boto3>=1.40.46,<1.40.62 diff --git a/plugins/nemo-safe-synthesizer/openapi/openapi.yaml b/plugins/nemo-safe-synthesizer/openapi/openapi.yaml index a2624974c2..0cd08d3d7a 100644 --- a/plugins/nemo-safe-synthesizer/openapi/openapi.yaml +++ b/plugins/nemo-safe-synthesizer/openapi/openapi.yaml @@ -821,11 +821,45 @@ components: description: The fraction of invalid records that will stop generation after the ``patience`` limit is reached. Must be in [0, 1]. default: 0.8 - structured_generation: - allOf: - - $ref: '#/components/schemas/StructuredGenerationParameters' - description: Structured generation parameters controlling schema-constrained - output. + use_structured_generation: + type: boolean + title: use_structured_generation + description: Whether to use structured generation for better format control. + default: false + structured_generation_backend: + type: string + enum: + - auto + - xgrammar + - guidance + - outlines + - lm-format-enforcer + title: structured_generation_backend + description: 'The backend used by vLLM when ``use_structured_generation`` + is ``True``. Supported backends: ''outlines'', ''guidance'', ''xgrammar'', + ''lm-format-enforcer''. ''auto'' will allow vLLM to choose the backend.' + default: auto + structured_generation_schema_method: + type: string + enum: + - auto + - regex + - json_schema + - structural_tag + title: structured_generation_schema_method + description: The method used to generate the schema from your dataset and + pass it to the generation backend. 'auto' picks 'structural_tag' on xgrammar-capable + backends and 'regex' otherwise. 'regex' uses a custom regex construction + method that tends to be more comprehensive than 'json_schema' at the cost + of speed. 'structural_tag' uses XGrammar Structural Tag to compose schema-constrained + JSONL output. + default: auto + structured_generation_use_single_sequence: + type: boolean + title: structured_generation_use_single_sequence + description: Whether to use a regex that matches exactly one sequence or + record if ``max_sequences_per_example`` is 1. + default: false enforce_timeseries_fidelity: type: boolean title: enforce_timeseries_fidelity @@ -1788,57 +1822,6 @@ components: type: array title: StringFilter type: object - StructuredGenerationParameters: - properties: - enabled: - type: boolean - title: enabled - description: Whether to use structured generation for better format control. - default: false - backend: - type: string - enum: - - auto - - xgrammar - - guidance - - outlines - - lm-format-enforcer - title: backend - description: 'The backend used by vLLM when structured generation is enabled. - Supported backends: ''outlines'', ''guidance'', ''xgrammar'', ''lm-format-enforcer''. - ''auto'' will allow vLLM to choose the backend.' - default: auto - schema_method: - type: string - enum: - - auto - - regex - - json_schema - - structural_tag - title: schema_method - description: The method used to generate the schema from your dataset and - pass it to the generation backend. 'auto' picks 'structural_tag' on xgrammar-capable - backends and 'regex' otherwise. 'regex' uses a custom regex construction - method that tends to be more comprehensive than 'json_schema' at the cost - of speed. 'structural_tag' uses XGrammar Structural Tag to compose schema-constrained - JSONL output. - default: auto - use_single_sequence: - type: boolean - title: use_single_sequence - description: Whether to use a regex that matches exactly one sequence or - record if ``max_sequences_per_example`` is 1. - default: false - type: object - title: StructuredGenerationParameters - description: 'Configuration for vLLM structured generation. - - - These parameters control whether generation is constrained to schema-shaped - - output, which backend enforces the constraint, and how the constraint schema - - is built.' TimeSeriesParameters: properties: is_timeseries: @@ -1856,8 +1839,8 @@ components: type: string timestamp_interval_seconds: title: Timestamp Interval Seconds - description: Positive whole-number interval in seconds between timestamps. - If not provided, the timestamp column will be used to infer the interval. + description: Interval in seconds between timestamps. If not provided, the + timestamp column will be used to infer the interval. type: integer timestamp_format: title: Timestamp Format diff --git a/plugins/nemo-safe-synthesizer/pyproject.toml b/plugins/nemo-safe-synthesizer/pyproject.toml index 10a3806c8d..2d0e3eba57 100644 --- a/plugins/nemo-safe-synthesizer/pyproject.toml +++ b/plugins/nemo-safe-synthesizer/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "httpx>=0.27.2", "nemo-platform", "nemo-platform-plugin", - "nemo-safe-synthesizer==0.1.7", + "nemo-safe-synthesizer==0.1.2", "pydantic[email]>=2.9.2", "pydantic-settings>=2.2.1", "python-multipart~=0.0.9", diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py index 3bfba8263c..1c69204ea4 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/config.py @@ -31,7 +31,7 @@ class SafeSynthesizerConfig(NemoConfig): ), ) runtime_venv: str = ".nemo/safe-synthesizer-runtime" - runtime_package: str = "nemo-safe-synthesizer[engine,cu129]==0.1.7" + runtime_package: str = "nemo-safe-synthesizer[engine,cu129]==0.1.2" runtime_python_version: str = "3.11" runtime_python: str | None = None default_job_resource_memory_request: str = "16G" diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py index 82b5a893f3..5f1183ec81 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/runtime.py @@ -5,6 +5,7 @@ from __future__ import annotations +import platform import shutil import subprocess from pathlib import Path @@ -22,7 +23,21 @@ RUNTIME_CONSTRAINTS_FILE = Path("plugins/nemo-safe-synthesizer/constraints.txt") FLASHINFER_CU129_INDEX_URL = "https://flashinfer.ai/whl/cu129" PYTORCH_CU129_INDEX_URL = "https://download.pytorch.org/whl/cu129" -VLLM_CU129_INDEX_URL = "https://wheels.vllm.ai/ee0da84ab9e04ac7610e28580af62c365e898389/cu129" +VLLM_CU129_VERSION = "0.20.0" + + +def vllm_cu129_wheel() -> str: + """Return the direct vLLM CUDA 12.9 wheel URL for the current host.""" + machine = platform.machine().lower() + if machine in {"amd64", "x86_64"}: + arch = "x86_64" + elif machine in {"aarch64", "arm64"}: + arch = "aarch64" + else: + raise RuntimeError(f"Unsupported architecture for vLLM CUDA 12.9 wheel: {platform.machine()}") + + wheel = f"vllm-{VLLM_CU129_VERSION}%2Bcu129-cp38-abi3-manylinux_2_31_{arch}.whl" + return f"vllm @ https://github.com/vllm-project/vllm/releases/download/v{VLLM_CU129_VERSION}/{wheel}" def runtime_package_index_options(runtime_package: str) -> list[str]: @@ -34,8 +49,6 @@ def runtime_package_index_options(runtime_package: str) -> list[str]: FLASHINFER_CU129_INDEX_URL, "--extra-index-url", PYTORCH_CU129_INDEX_URL, - "--extra-index-url", - VLLM_CU129_INDEX_URL, ] @@ -43,9 +56,7 @@ def runtime_package_extra_requirements(runtime_package: str) -> list[str]: """Return direct requirements needed by the selected runtime package.""" if "cu129" not in runtime_package: return [] - # Safe Synthesizer 0.1.7 declares its cu129 vLLM dependency directly; the - # runtime only needs to add the vLLM wheel index above. - return [] + return [vllm_cu129_wheel()] def repo_root() -> Path: diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py b/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py index 95f6cb68ef..6c40851e56 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py @@ -51,10 +51,8 @@ def test_cuda_runtime_package_adds_cu129_sources(): runtime.FLASHINFER_CU129_INDEX_URL, "--extra-index-url", runtime.PYTORCH_CU129_INDEX_URL, - "--extra-index-url", - runtime.VLLM_CU129_INDEX_URL, ] - assert runtime.runtime_package_extra_requirements(runtime_package) == [] + assert runtime.runtime_package_extra_requirements(runtime_package) == [runtime.vllm_cu129_wheel()] def test_non_cu129_runtime_package_does_not_add_cu129_sources(): @@ -92,7 +90,7 @@ def fake_run(command, **kwargs): assert "--extra-index-url" in calls[2][0] assert runtime.FLASHINFER_CU129_INDEX_URL in calls[2][0] assert runtime.PYTORCH_CU129_INDEX_URL in calls[2][0] - assert runtime.VLLM_CU129_INDEX_URL in calls[2][0] + assert runtime.vllm_cu129_wheel() in calls[2][0] assert str(tmp_path / "plugins/nemo-safe-synthesizer") not in calls[2][0] assert "nemo-safe-synthesizer[engine,cu129]" in calls[2][0] assert calls[3][0] == [ diff --git a/pytest.ini b/pytest.ini index bf7f15fbf4..cc60fa0df8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -66,7 +66,6 @@ markers = e2e_config(*layers, harness=...): Ordered list of repo-root-relative config paths and/or inline dict overlays; harness config stays separate from platform config subprocess_only: Test only works in subprocess mode (not on Kubernetes); skipped when NMP_BASE_URL is set container_only: Test requires a container backend (Docker or Kubernetes); skipped unless NMP_BASE_URL is set - requires_gpu: Container e2e requiring GPU job scheduling; skipped unless --feature gpu is passed regression: Regression tests - test individual functional microservices for baseline functionality infrastructure: Infrastructure tests - ensure services are compatible with customer infrastructure canary: Canary tests - test deployed integration environments like top of tree diff --git a/release/assets.yaml b/release/assets.yaml new file mode 100644 index 0000000000..ec1f8adbc0 --- /dev/null +++ b/release/assets.yaml @@ -0,0 +1,15 @@ +sdk: + - id: nemo-platform + - id: nemo-platform-plugin + +# Container images eligible for release publishing. This list is the single +# source of truth: the release-consumer repository reads it at the release ref +# to decide which images to stage. Each id must match an image name pushed to +# the dev registry tagged by this repository's commit SHA, and should have a +# catalog metadata entry on the consumer side. +container: + - id: nmp-automodel-tasks + - id: nmp-automodel-training + - id: nmp-unsloth-training + - id: auditor-tasks + - id: safe-synthesizer-tasks diff --git a/script/dev-install-fabric.sh b/script/dev-install-fabric.sh index 6c966fc324..cb20509872 100755 --- a/script/dev-install-fabric.sh +++ b/script/dev-install-fabric.sh @@ -53,7 +53,7 @@ if [ ! -d "$FABRIC_REPO" ]; then fi echo "Building + installing nemo-fabric[codex,relay] from $FABRIC_REPO into $VENV_PY ..." uv pip install --python "$VENV_PY" "${FABRIC_REPO}[codex,relay]" -"$VENV_PY" -c "import nemo_fabric; from nemo_fabric import Fabric, RunResult; print('nemo_fabric OK:', nemo_fabric.__file__)" +"$VENV_PY" -c "import nemo_fabric; from nemo_fabric import FabricClient, RunResult; print('nemo_fabric OK:', nemo_fabric.__file__)" # 2. nemo-relay gateway binary (codex -> OTLP -> gateway -> trajectory-*.atif.json). Required for # trajectory capture; the pip `nemo-relay` package does NOT ship this executable. diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py index c86b06fdb9..adda911e12 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/evaluator.py @@ -628,12 +628,10 @@ def _default_prompt_template(target: Model | Agent) -> dict[str, Any]: def _task_row(task: AgentEvalTask) -> dict[str, Any]: - # `prompt` is what the target under evaluation is prompted with (via the `{{item.prompt}}` - # template): a dataset `prompt` column or the agent `instruction`. return { **task.inputs, "task_id": task.id, - "prompt": task.inputs.get("prompt") or task.inputs.get("instruction"), + "prompt": task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent, } diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py index 9a904d8952..1443f7146a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/codex/runtime.py @@ -68,7 +68,7 @@ def __init__( self._work_root = Path(work_root).expanduser() if work_root is not None else None self._codex_bin = codex_bin self._timeout_s = timeout_s - self._prompt_builder = prompt_builder or AgentEvalTask.agent_prompt + self._prompt_builder = prompt_builder or default_codex_prompt self._process_factory = process_factory or asyncio.create_subprocess_exec self._runtime_name = runtime_name @@ -95,17 +95,15 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC evidence_dir.mkdir(parents=True, exist_ok=True) workspace_dir.mkdir(parents=True, exist_ok=True) + prompt = self._prompt_builder(task) prompt_path = evidence_dir / "prompt.txt" task_path = evidence_dir / "task.json" stdout_path = evidence_dir / "stdout.jsonl" stderr_path = evidence_dir / "stderr.txt" final_output_path = evidence_dir / "final_output.txt" - # Persist the task for debugging, but never the grader-only fields: the docker variant mounts - # this evidence dir into the sandbox (danger-full-access), so serializing `intent` (desired - # behavior) or `reference` (held-out ground truth) here would let the agent read them back out - # of `/evidence/task.json` — the same reward-hacking leak the intent-free prompt closes. - task_path.write_text(task.model_dump_json(indent=2, exclude={"intent", "reference"}), encoding="utf-8") + prompt_path.write_text(prompt, encoding="utf-8") + task_path.write_text(task.model_dump_json(indent=2), encoding="utf-8") command = self._command(workspace_dir=workspace_dir, final_output_path=final_output_path) process: Any | None = None @@ -115,10 +113,6 @@ async def _run_task(self, index: int, task: AgentEvalTask, config: AgentEvalRunC # synchronous (a handler may do blocking I/O, e.g. the plugin's fileset download), and this # runs on the event loop shared by every concurrent task, so a blocking seed would stall them all. seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - # Build the prompt after seeding and inside the guarded block: an instruction-less task - # raises here, failing just this task instead of aborting the run (and seeding wins if both). - prompt = self._prompt_builder(task) - prompt_path.write_text(prompt, encoding="utf-8") process = await self._process_factory( *command, stdin=subprocess.PIPE, @@ -392,6 +386,30 @@ def print_codex_agent_models(*, codex_bin: str = "codex") -> None: print(slug) +def default_codex_prompt(task: AgentEvalTask) -> str: + """Frame a task for Codex as an agent that works in its current directory. + + Task-agnostic: it states the intent and inputs and invites the agent to read/create/edit files, + rather than constraining the answer to a single text reply. Seed files (``inputs[SEED_FILES_INPUT_KEY]``) + are listed by name instead of dumped inline — the agent finds them already in its workspace. Pass a + custom :data:`CodexPromptBuilder` to the runtime to override this framing for a specific benchmark. + """ + body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY} + lines = [f"Task id: {task.id}", f"Intent: {task.intent}"] + if body_inputs: + lines += ["", "Inputs:", json.dumps(body_inputs, indent=2, default=str)] + seeded = task.inputs.get(SEED_FILES_INPUT_KEY) + if isinstance(seeded, Mapping) and seeded: + lines += ["", "These files are already in your working directory:"] + lines += [f" - {path}" for path in seeded] + lines += [ + "", + "Complete the task by working in your current directory. You may read, create, and edit files " + "as needed. When you are done, briefly summarize what you changed.", + ] + return "\n".join(lines) + "\n" + + def _failed_codex_trial( task: AgentEvalTask, evidence_dir: Path, diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py index 2fcdff1982..be39b8e1f8 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/docker_sandbox.py @@ -136,15 +136,13 @@ async def _run_task( ) -> AgentEvalTrial: evidence_dir = self._evidence_dir(index, task, config) evidence_dir.mkdir(parents=True, exist_ok=True) + prompt = _task_prompt(task) + manifest = self._build_manifest(task, sdk) + agent = self._build_agent(manifest, sdk) client = self._build_client(sdk) sandbox = None try: - # Build the prompt inside the guarded block: an instruction-less task raises here and fails - # just this task rather than aborting the whole run. - prompt = task.agent_prompt() - manifest = self._build_manifest(task, sdk) - agent = self._build_agent(manifest, sdk) sandbox = await client.create( manifest=manifest, options=sdk.DockerSandboxClientOptions(image=self._image or sdk.DEFAULT_PYTHON_SANDBOX_IMAGE), @@ -165,7 +163,7 @@ def _build_manifest(self, task: AgentEvalTask, sdk: SandboxSDK) -> Any: # workspace — nothing in the runtime consumes it, and dumping the whole DTO would expose # grader-only fields (e.g. ``reference`` held-out ground truth) to the agent. entries: dict[str, Any] = { - "instruction.md": sdk.File(content=task.agent_prompt().encode("utf-8")), + "instruction.md": sdk.File(content=_task_prompt(task).encode("utf-8")), "output": sdk.Dir(), } workspace_dir = task.inputs.get("workspace_dir") @@ -297,6 +295,10 @@ def _validated_workspace_dir(workspace_dir: Any) -> Path: return resolved +def _task_prompt(task: AgentEvalTask) -> str: + return str(task.inputs.get("prompt") or task.inputs.get("instruction") or task.intent) + + async def _maybe_await(value: Awaitable[Any] | Any) -> Any: if inspect.isawaitable(value): return await value diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py index 7e7f02468c..c0cd728cfd 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/runtimes/fabric/runtime.py @@ -7,11 +7,7 @@ NeMo Fabric Python SDK and adapts each normalized Fabric ``RunResult`` into an :class:`AgentEvalTrial`. The harness is chosen by the supplied Fabric config's ``harness.adapter_id`` (never inferred from a model); an optional ``model`` slug -is applied as the config's default model, mirroring Fabric's own Harbor integration. - -Per-task settings (workspace, model, trajectory capture) are composed directly onto -a copy of the supplied config via the SDK's config helpers (``model_copy`` + -``enable_relay`` + ``environment``), rather than layered as profile overlays. +is applied as a final profile overlay, mirroring Fabric's own Harbor integration. Every task runs in its own fresh workspace: the runtime seeds it from ``inputs['files']`` (a no-op when there are none), runs the harness in it (via @@ -42,7 +38,6 @@ CandidateEvidence, EvidenceDescriptor, ) -from pydantic import JsonValue if TYPE_CHECKING: # Annotations use nemo_fabric's real types (single source of truth). nemo_fabric is an optional @@ -50,10 +45,9 @@ # loaded lazily at runtime (see ``run_tasks``). Drop the ty:ignore once nemo-fabric is a # resolvable dependency and the checker can see it. from nemo_fabric import ( # ty: ignore[unresolved-import] - Fabric, + FabricClient, FabricConfig, FabricProfileConfig, - RunOutput, RunResult, ) @@ -72,18 +66,18 @@ # Per-task workspace: where seed files are staged and where the harness reads/writes. We # create it, point Fabric's ``environment.workspace`` at it, and expose it as ``workspace`` evidence. _WORKSPACE_SUBDIR = "workspace" +_WORKSPACE_PROFILE_NAME = "eval_workspace" # Evidence key + descriptor kind for the staged workspace, consumed by the # workspace-reading metrics. _WORKSPACE_EVIDENCE_KEY = "workspace" _WORKSPACE_EVIDENCE_KIND = "filesystem" -# File-exporter output names we choose for the Relay ATIF/ATOF trajectory (Relay accepts these as inputs). +# Trajectory profile identity + the file-exporter output names we choose (Relay accepts these as inputs). +_TRAJECTORY_PROFILE_NAME = "eval_trajectory" _ATIF_FILENAME_TEMPLATE = "trajectory-{session_id}.atif.json" _ATOF_FILENAME = "events.atof.jsonl" -# Names for the trailing overlays that re-assert the evaluator-owned per-task settings (see -# ``_eval_lock_profiles``): Fabric applies caller profiles over the config, so these must trail them. -_WORKSPACE_PROFILE_NAME = "eval_workspace" -_MODEL_PROFILE_NAME = "eval_model" -_ARTIFACTS_PROFILE_NAME = "eval_artifacts" +# Fabric telemetry-profile selectors (file exporter, no OTLP endpoint). +_TELEMETRY_PROVIDER = "relay" +_TELEMETRY_MODE = "sdk" # ``kind`` Fabric stamps on the promoted Relay ATIF artifact; used to surface it as trace evidence. _ATIF_ARTIFACT_KIND = "atif" @@ -127,77 +121,69 @@ async def run_tasks( try: # nemo_fabric ships a native (pyo3) core and is an optional dependency, so it is imported # lazily here rather than at module load. - from nemo_fabric import Fabric, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] + from nemo_fabric import FabricClient, FabricConfig, FabricProfileConfig # ty: ignore[unresolved-import] except ImportError as exc: raise RuntimeError(_MISSING_FABRIC_MSG) from exc resolved_config = config or AgentEvalRunConfig() agent_config = FabricConfig.from_mapping(self._config) - # Fail fast (once) if trajectory capture is requested but the nemo-relay gateway isn't - # importable, rather than failing every task the same way inside the per-task guard. - if self._capture_trajectory: - try: - import nemo_relay.observability # noqa: F401 # ty: ignore[unresolved-import] - except ImportError as exc: - raise RuntimeError(_MISSING_RELAY_MSG) from exc - # Caller-supplied profile overlays pass through as-is; this runtime's per-task workspace, model, - # and trajectory settings are composed directly onto a copy of the config (config-first), not - # layered as profiles. - base_profiles = [FabricProfileConfig.from_mapping(profile) for profile in self._profiles] + base_profiles = self._build_profiles(FabricProfileConfig) semaphore = asyncio.Semaphore(resolved_config.parallelism) - # ``Fabric`` (formerly ``FabricClient``) is a lightweight, reusable facade — not a lifecycle - # context manager — so it is created once and reused across tasks with no cleanup. - client = Fabric() + async with FabricClient() as client: - async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: - async with semaphore: - return await self._run_task(client, agent_config, base_profiles, index, task, resolved_config) + async def run_one(index: int, task: AgentEvalTask) -> AgentEvalTrial: + async with semaphore: + return await self._run_task( + client, agent_config, base_profiles, FabricProfileConfig, index, task, resolved_config + ) - return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) + return await asyncio.gather(*(run_one(index, task) for index, task in enumerate(tasks))) async def _run_task( self, - client: Fabric, + client: FabricClient, agent_config: FabricConfig, base_profiles: list[FabricProfileConfig], + profile_cls: type[FabricProfileConfig], index: int, task: AgentEvalTask, config: AgentEvalRunConfig, ) -> AgentEvalTrial: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import FabricProfileConfig, RunRequest # ty: ignore[unresolved-import] - evidence_dir = self._evidence_dir(index, task, config) evidence_dir.mkdir(parents=True, exist_ok=True) + profiles = list(base_profiles) + if self._capture_trajectory: + # Enable Relay's ATIF file exporter, writing the trajectory under this task's durable + # evidence dir; Fabric promotes the resulting file into RunResult.artifacts. Both the + # Fabric artifact root and the relay output dir must exist and be durable. + relay_dir = evidence_dir / _RELAY_SUBDIR + artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR + relay_dir.mkdir(parents=True, exist_ok=True) + artifacts_dir.mkdir(parents=True, exist_ok=True) + profiles.append(self._trajectory_profile(profile_cls, relay_dir=relay_dir, artifacts_dir=artifacts_dir)) + # Every task runs in its own fresh workspace: seed any ``inputs['files']`` into it (a no-op when - # there are none), point the harness at it, and expose it as ``workspace`` filesystem evidence — - # a uniform per-task dir that maps cleanly onto a per-task container volume later. Seeding runs - # inside the guarded block so a bad seed (a path escaping the workspace, an unresolvable fileset) - # fails just this task, not the whole run; it is synchronous and may block (a fileset handler - # downloads), so it is offloaded off the shared event loop. + # there are none), point the harness at it via ``environment.workspace``, and expose it as + # ``workspace`` filesystem evidence — a uniform per-task dir that maps cleanly onto a per-task + # container volume later. Seeding runs inside the guarded block so a bad seed (a path escaping + # the workspace, an unresolvable fileset) fails just this task, not the whole run; it is + # synchronous and may block (a fileset handler downloads), so it is offloaded off the shared + # event loop. workspace_dir = evidence_dir / _WORKSPACE_SUBDIR workspace_dir.mkdir(parents=True, exist_ok=True) try: - # Stage seed files into the workspace for their on-disk side effect; the prompt is the task - # instruction only, so the returned paths are unused. - await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) - task_config = self._compose_config(agent_config, evidence_dir, workspace_dir) - # Caller ``base_profiles`` are applied by Fabric over the config; the evaluator-owned - # settings are re-asserted as trailing overlays so they win over any caller profile. - lock_profiles = self._eval_lock_profiles( - FabricProfileConfig, workspace_dir=workspace_dir, evidence_dir=evidence_dir - ) + seeded_files = await asyncio.to_thread(seed_workspace, workspace_dir, task.inputs.get(SEED_FILES_INPUT_KEY)) + profiles.append(self._workspace_profile(profile_cls, workspace_dir=workspace_dir)) result = await asyncio.wait_for( - # ``Fabric.run`` folds the per-invocation input + request id into a ``RunRequest``. client.run( - task_config, - profiles=[*base_profiles, *lock_profiles], + agent_config, + profiles=profiles, + input=_fabric_input(task, seeded_files), + request_id=task.id, base_dir=self._base_dir, - request=RunRequest(input=task.agent_prompt(), request_id=task.id), ), timeout=self._timeout_s, ) @@ -228,17 +214,13 @@ def _to_trial( if result.status != "succeeded": return self._failed_trial(task, evidence_dir, _result_error(result), extra_metadata=base_metadata) - # Fabric wraps the output in a ``RunOutput`` mapping (RunOutput response contract, #52), - # which is not itself a JSON value; normalize it to a plain mapping so it round-trips through the - # trial's ``JsonValue``-typed response. - output = _normalize_output(result.output) return AgentEvalTrial( id=f"{task.id}:fabric", task_id=task.id, status=AgentEvalTrialStatus.COMPLETED, output=AgentOutput( - output_text=_extract_output_text(output), - response=output, + output_text=_extract_output_text(result.output), + response=result.output, metadata={**base_metadata, "evidence_dir": str(evidence_dir)}, ), evidence=self._evidence(result, result_path, workspace_dir), @@ -315,90 +297,31 @@ def _failed_trial( }, ) - def _compose_config( - self, - agent_config: FabricConfig, - evidence_dir: Path, - workspace_dir: Path, - ) -> FabricConfig: - # nemo_fabric is already imported+validated in ``run_tasks``; this is a cached sys.modules - # lookup, not a re-load, so the type is used where it's constructed instead of threaded down. - from nemo_fabric import EnvironmentConfig # ty: ignore[unresolved-import] - - # Config-first composition (the SDK's recommended in-memory pattern): copy the base config and - # apply this task's workspace, model, and trajectory settings directly onto it, rather than - # layering FabricProfileConfig overlays. - cfg = agent_config.model_copy(deep=True) - - # Point the harness at this task's staged workspace (the codex-cli adapter resolves its cwd from - # it). ``provider="local"`` is required by the native planner. Any config-supplied - # environment.workspace is overridden per task. - environment = cfg.environment or EnvironmentConfig(provider="local") - environment.provider = environment.provider or "local" - environment.workspace = str(workspace_dir) - cfg.environment = environment - - # Apply the model as the config's default (mirrors nemo_fabric.integrations.harbor). + def _build_profiles(self, profile_cls: type[FabricProfileConfig]) -> list[FabricProfileConfig]: + profiles = [profile_cls.from_mapping(profile) for profile in self._profiles] if self._model: + # Apply the model as a final profile overlay (mirrors nemo_fabric.integrations.harbor). provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - cfg.models["default"] = {"provider": provider, "model": self._model} - - if self._capture_trajectory: - # Enable Relay's ATIF/ATOF file exporter under this task's durable evidence dir, and pin the - # Fabric artifact root so the promoted ``trajectory-*.atif.json`` persists. Requires the - # ``nemo-relay`` gateway on PATH in the runtime. - relay_dir = evidence_dir / _RELAY_SUBDIR - artifacts_dir = evidence_dir / _ARTIFACTS_SUBDIR - relay_dir.mkdir(parents=True, exist_ok=True) - artifacts_dir.mkdir(parents=True, exist_ok=True) - cfg.enable_relay(output_dir=str(relay_dir), config=self._relay_config(relay_dir)) - cfg.runtime.artifacts = str(artifacts_dir) - cfg.environment.artifacts = str(artifacts_dir) - - return cfg - - def _eval_lock_profiles( - self, - profile_cls: type[FabricProfileConfig], - *, - workspace_dir: Path, - evidence_dir: Path, - ) -> list[FabricProfileConfig]: - # ``_compose_config`` composes the evaluator's per-task settings onto the config, but Fabric - # applies caller-supplied profiles OVER the config (last-wins), so a caller profile could - # otherwise override them. Re-assert the evaluator-owned settings here as trailing overlays — - # applied after the caller profiles — so the per-task workspace (isolation + ``workspace`` - # evidence integrity), the model under evaluation, and the trajectory artifact location stay - # authoritative and non-overridable. - overlays = [ - profile_cls.from_mapping( - {"name": _WORKSPACE_PROFILE_NAME, "environment": {"workspace": str(workspace_dir)}} - ) - ] - if self._model: - provider = self._model.split("/", maxsplit=1)[0] if "/" in self._model else "openai" - overlays.append( - profile_cls.from_mapping( - {"name": _MODEL_PROFILE_NAME, "models": {"default": {"provider": provider, "model": self._model}}} + profiles.append( + profile_cls( + name="eval_model", + models={"default": {"provider": provider, "model": self._model}}, ) ) - if self._capture_trajectory: - artifacts_dir = str(evidence_dir / _ARTIFACTS_SUBDIR) - overlays.append( - profile_cls.from_mapping( - { - "name": _ARTIFACTS_PROFILE_NAME, - "runtime": {"artifacts": artifacts_dir}, - "environment": {"artifacts": artifacts_dir}, - } - ) - ) - return overlays - - def _relay_config(self, relay_dir: Path) -> dict[str, Any]: + return profiles + + def _trajectory_profile( + self, profile_cls: type[FabricProfileConfig], *, relay_dir: Path, artifacts_dir: Path + ) -> FabricProfileConfig: + # Relay ATIF/ATOF file exporter (mode=sdk): the harness emits its trajectory to a local + # nemo-relay gateway, which writes ``trajectory-*.atif.json`` under ``relay_dir``. No OTLP + # collector endpoint is involved. Requires the ``nemo-relay`` gateway on PATH in the runtime. + # The Fabric artifact root is pinned to a durable dir so the promoted trajectory persists. + # # The observability component is built from nemo_relay's own typed config objects so Relay owns # its schema (no hand-maintained dict that silently drifts when Relay changes it); imported - # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. + # lazily since nemo-relay, like nemo-fabric, is an optional native dependency. ``schema_version`` + # is omitted — ``FabricProfileConfig`` defaults it. try: from nemo_relay.observability import ( # ty: ignore[unresolved-import] AtifConfig, @@ -410,6 +333,7 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]: raise RuntimeError(_MISSING_RELAY_MSG) from exc relay_dir_str = str(relay_dir) + artifacts_dir_str = str(artifacts_dir) observability = ComponentSpec( config=ObservabilityConfig( atif=AtifConfig( @@ -427,7 +351,33 @@ def _relay_config(self, relay_dir: Path) -> dict[str, Any]: ), ) ) - return {"version": 1, "components": [observability.to_dict()]} + return profile_cls.from_mapping( + { + "name": _TRAJECTORY_PROFILE_NAME, + "description": "Capture the agent trajectory as ATIF via the NeMo Relay file exporter.", + "runtime": {"artifacts": artifacts_dir_str}, + "environment": {"artifacts": artifacts_dir_str}, + "telemetry": { + "enabled": True, + "provider": _TELEMETRY_PROVIDER, + "mode": _TELEMETRY_MODE, + "output_dir": relay_dir_str, + "config": {"version": 1, "components": [observability.to_dict()]}, + }, + } + ) + + def _workspace_profile(self, profile_cls: type[FabricProfileConfig], *, workspace_dir: Path) -> FabricProfileConfig: + # Point the harness at this task's staged workspace via ``environment.workspace`` (the codex-cli + # adapter resolves its cwd from it). Set as a final profile overlay, the same mechanism the + # trajectory profile uses for ``environment.artifacts``. + return profile_cls.from_mapping( + { + "name": _WORKSPACE_PROFILE_NAME, + "description": "Run the harness in the per-task evaluation workspace seeded with the task inputs.", + "environment": {"workspace": str(workspace_dir)}, + } + ) def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunConfig) -> Path: root = self._work_root @@ -438,16 +388,22 @@ def _evidence_dir(self, index: int, task: AgentEvalTask, config: AgentEvalRunCon return Path(root) / task_dir -def _normalize_output(output: RunOutput | JsonValue) -> JsonValue: - """Unwrap a Fabric ``RunResult.output`` into the plain JSON value the trial response stores. +def _fabric_input(task: AgentEvalTask, seeded_files: Sequence[str] = ()) -> str: + """Frame the task as the harness's input text. - Newer Fabric wraps output in a ``RunOutput`` (the RunOutput response contract), which is a - ``Mapping``; copy it into a plain dict (equivalent to its ``to_mapping()``). Raw/older JSON outputs - are already JSON values and pass through unchanged. + When files were staged into the workspace, list them by name and invite the agent to work in its + current directory rather than dumping their contents inline; the seed-files key is dropped from + the echoed inputs since those files are already on disk. """ - if isinstance(output, Mapping): - return dict(output) - return output + body_inputs = {key: value for key, value in task.inputs.items() if key != SEED_FILES_INPUT_KEY} + lines = [f"Task id: {task.id}", f"Intent: {task.intent}"] + if body_inputs: + lines += ["", f"Inputs: {body_inputs}"] + if seeded_files: + lines += ["", "These files are already in your working directory:"] + lines += [f" - {path}" for path in seeded_files] + lines += ["", "Complete the task by reading, creating, and editing files in your current directory."] + return "\n".join(lines) + "\n" def _extract_output_text(output: object) -> str | None: diff --git a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py index 6c0da4ac21..445af23e24 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py +++ b/sdk/python/nemo-platform/src/nemo_platform/beta/evaluator/agent_eval/tasks.py @@ -97,23 +97,6 @@ def _id_must_not_be_empty(cls, value: str) -> str: raise ValueError("task id must not be empty") return value - def agent_prompt(self) -> str: - """The intent-free prompt handed to the agent under evaluation. - - Exactly the task's natural-language instruction (``inputs["instruction"]``), with no - runtime-added framing. ``intent`` is deliberately never used: it is the eval-side description - of the desired behavior (what the grader checks for), so exposing it to the agent is a - reward-hacking hole. - - Raises ``ValueError`` when ``inputs["instruction"]`` is missing or empty; a task with no - instruction cannot be evaluated, so the runner fails that task rather than running an agent on - an empty prompt. - """ - instruction = self.inputs.get("instruction") - if instruction: - return str(instruction) - raise ValueError(f"task {self.id!r} has no instruction: set inputs['instruction']") - @field_serializer("metrics", when_used="json") def _serialize_metrics(self, metrics: list[Metric]) -> list[dict[str, Any]]: """Serialize local metric instances as descriptors for run bundles.""" diff --git a/services/studio/src/nmp/studio/env_mappings.py b/services/studio/src/nmp/studio/env_mappings.py index c46a6f68f1..5ff3fe3f17 100644 --- a/services/studio/src/nmp/studio/env_mappings.py +++ b/services/studio/src/nmp/studio/env_mappings.py @@ -82,7 +82,7 @@ class EnvMapping: EnvMapping( marker="STUDIO_UI_VITE_FF_DATA_DESIGNER_ENABLED", config_path="studio.feature_flags.data_designer_enabled", - default="true", + default="false", ), EnvMapping( marker="STUDIO_UI_VITE_FF_DATASETS_ENABLED", diff --git a/tests/unit/release/test_write_release_bundle_metadata.py b/tests/unit/release/test_write_release_bundle_metadata.py new file mode 100644 index 0000000000..1096d67f00 --- /dev/null +++ b/tests/unit/release/test_write_release_bundle_metadata.py @@ -0,0 +1,441 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import importlib.util +import json +import zipfile +from pathlib import Path +from types import ModuleType + +import pytest + + +def load_bundle_metadata_module() -> ModuleType: + script_path = Path(__file__).parents[3] / ".github/scripts/write_release_bundle_metadata.py" + spec = importlib.util.spec_from_file_location("write_release_bundle_metadata", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +bundle_metadata = load_bundle_metadata_module() +BundleMetadataError = bundle_metadata.BundleMetadataError + + +def selected_artifacts(*artifacts: dict[str, str]) -> str: + return json.dumps(list(artifacts), separators=(",", ":")) + + +def write_wheel( + sdk_artifacts_dir: Path, + sdk_id: str, + *, + filename: str = "nemo_platform-1.0.0-py3-none-any.whl", + version: str = "1.0.0", + metadata_files: list[str] | None = None, +) -> Path: + artifact_dir = sdk_artifacts_dir / f"release-sdk-{sdk_id}" + artifact_dir.mkdir(parents=True, exist_ok=True) + wheel_path = artifact_dir / filename + metadata_files = metadata_files if metadata_files is not None else ["nemo_platform-1.0.0.dist-info/METADATA"] + + with zipfile.ZipFile(wheel_path, "w") as wheel: + for metadata_file in metadata_files: + wheel.writestr( + metadata_file, + f"""Metadata-Version: 2.1 +Name: nemo-platform +Version: {version} +""", + ) + wheel.writestr("nemo_platform-1.0.0.dist-info/WHEEL", "Wheel-Version: 1.0\n") + + return wheel_path + + +def write_metadata( + tmp_path: Path, + *, + selected: str | None = None, + cadence: str = "release", + release_label: str = "1.0.0", + release_date_json: str = '"2026-06-30"', + source_sha: str = "a" * 40, +) -> tuple[Path, Path]: + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + bundle_dir = tmp_path / "release-bundle" + write_wheel(sdk_artifacts_dir, "nemo-platform") + + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=bundle_dir, + selected_artifacts_json=selected or selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence=cadence, + release_label=release_label, + release_date_json=release_date_json, + source_sha=source_sha, + ) + return sdk_artifacts_dir, bundle_dir + + +def read_manifest(bundle_dir: Path) -> dict[str, object]: + return json.loads((bundle_dir / "release-manifest.json").read_text(encoding="utf-8")) + + +def parse_checksums(bundle_dir: Path) -> dict[str, str]: + entries = {} + for line in (bundle_dir / "checksums.txt").read_text(encoding="utf-8").splitlines(): + digest, path = line.split(" ", 1) + entries[path] = digest + return entries + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_one_sdk_wheel_writes_manifest_and_checksums(tmp_path: Path): + sdk_artifacts_dir, bundle_dir = write_metadata(tmp_path) + source_wheel = sdk_artifacts_dir / "release-sdk-nemo-platform/nemo_platform-1.0.0-py3-none-any.whl" + bundled_wheel = bundle_dir / "wheels/nemo_platform-1.0.0-py3-none-any.whl" + + assert bundled_wheel.read_bytes() == source_wheel.read_bytes() + assert read_manifest(bundle_dir) == { + "cadence": "release", + "release_label": "1.0.0", + "release_date": "2026-06-30", + "source_sha": "a" * 40, + "artifacts": [ + { + "type": "sdk", + "id": "nemo-platform", + "version": "1.0.0", + "path": "wheels/nemo_platform-1.0.0-py3-none-any.whl", + } + ], + } + + assert parse_checksums(bundle_dir) == { + "release-manifest.json": sha256(bundle_dir / "release-manifest.json"), + "wheels/nemo_platform-1.0.0-py3-none-any.whl": sha256(bundled_wheel), + } + + +def test_one_sdk_wheel_can_be_downloaded_directly_to_artifacts_dir(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + bundle_dir = tmp_path / "release-bundle" + nested_wheel = write_wheel(sdk_artifacts_dir, "nemo-platform") + direct_wheel = sdk_artifacts_dir / nested_wheel.name + nested_wheel.rename(direct_wheel) + nested_wheel.parent.rmdir() + + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=bundle_dir, + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + bundled_wheel = bundle_dir / "wheels/nemo_platform-1.0.0-py3-none-any.whl" + assert bundled_wheel.read_bytes() == direct_wheel.read_bytes() + assert read_manifest(bundle_dir)["artifacts"][0]["path"] == ( # type: ignore[index] + "wheels/nemo_platform-1.0.0-py3-none-any.whl" + ) + + +def test_release_date_json_null_becomes_manifest_null(tmp_path: Path): + _, bundle_dir = write_metadata(tmp_path, release_date_json="null") + + assert read_manifest(bundle_dir)["release_date"] is None + + +def test_checksums_only_include_manifest_and_wheels(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + bundle_dir = tmp_path / "release-bundle" + write_wheel(sdk_artifacts_dir, "nemo-platform") + bundle_dir.mkdir() + (bundle_dir / "stale.txt").write_text("old file\n", encoding="utf-8") + + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=bundle_dir, + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + assert set(parse_checksums(bundle_dir)) == { + "release-manifest.json", + "wheels/nemo_platform-1.0.0-py3-none-any.whl", + } + + +def test_rc_label_stays_release_label_and_wheel_version_comes_from_metadata(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + bundle_dir = tmp_path / "release-bundle" + write_wheel( + sdk_artifacts_dir, + "nemo-platform", + filename="nemo_platform-1.0.0rc0-py3-none-any.whl", + version="1.0.0rc0", + ) + + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=bundle_dir, + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="rc", + release_label="1.0.0-rc0", + release_date_json="null", + source_sha="b" * 40, + ) + + manifest = read_manifest(bundle_dir) + assert manifest["release_label"] == "1.0.0-rc0" + assert manifest["artifacts"][0]["version"] == "1.0.0rc0" # type: ignore[index] + + +def test_missing_sdk_artifact_directory_fails_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="missing downloaded SDK artifact directory"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_zero_wheels_fails_clearly(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + (sdk_artifacts_dir / "release-sdk-nemo-platform").mkdir(parents=True) + + with pytest.raises(BundleMetadataError, match="expected exactly one wheel"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_multiple_wheels_for_one_sdk_fails_clearly(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + write_wheel(sdk_artifacts_dir, "nemo-platform") + write_wheel( + sdk_artifacts_dir, + "nemo-platform", + filename="nemo_platform-1.0.1-py3-none-any.whl", + version="1.0.1", + ) + + with pytest.raises(BundleMetadataError, match="expected exactly one wheel"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_multiple_directly_downloaded_wheels_fail_clearly(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + nested_wheel = write_wheel(sdk_artifacts_dir, "nemo-platform") + direct_wheel = sdk_artifacts_dir / nested_wheel.name + nested_wheel.rename(direct_wheel) + nested_wheel.parent.rmdir() + write_wheel( + sdk_artifacts_dir, + "extra", + filename="nemo_platform-1.0.1-py3-none-any.whl", + version="1.0.1", + ).rename( + sdk_artifacts_dir / "nemo_platform-1.0.1-py3-none-any.whl", + ) + + with pytest.raises(BundleMetadataError, match="expected exactly one wheel"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_unsupported_artifact_type_fails_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="unsupported artifact type"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts({"type": "helm", "id": "platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_container_artifacts_become_metadata_only_entries(tmp_path: Path): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + bundle_dir = tmp_path / "release-bundle" + write_wheel(sdk_artifacts_dir, "nemo-platform") + + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=bundle_dir, + selected_artifacts_json=selected_artifacts( + {"type": "sdk", "id": "nemo-platform"}, + {"type": "container", "id": "nmp-automodel-tasks"}, + {"type": "container", "id": "nmp-unsloth-training"}, + ), + cadence="rc", + release_label="1.0.0-rc1", + release_date_json="null", + source_sha="c" * 40, + ) + + artifacts = read_manifest(bundle_dir)["artifacts"] + assert artifacts[1:] == [ # type: ignore[index] + {"type": "container", "id": "nmp-automodel-tasks", "version": "1.0.0-rc1"}, + {"type": "container", "id": "nmp-unsloth-training", "version": "1.0.0-rc1"}, + ] + # Container entries are metadata-only: no path, and nothing extra in checksums. + assert set(parse_checksums(bundle_dir)) == { + "release-manifest.json", + "wheels/nemo_platform-1.0.0-py3-none-any.whl", + } + + +def test_container_only_selection_is_valid(tmp_path: Path): + bundle_dir = tmp_path / "release-bundle" + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=bundle_dir, + selected_artifacts_json=selected_artifacts( + {"type": "container", "id": "nmp-automodel-tasks"}, + {"type": "container", "id": "nmp-unsloth-training"}, + ), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + # Container-only bundle: only container entries, no wheels, checksums = manifest only. + assert read_manifest(bundle_dir)["artifacts"] == [ + {"type": "container", "id": "nmp-automodel-tasks", "version": "1.0.0"}, + {"type": "container", "id": "nmp-unsloth-training", "version": "1.0.0"}, + ] + assert set(parse_checksums(bundle_dir)) == {"release-manifest.json"} + + +def test_empty_selection_fails_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="non-empty list"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json="[]", + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_duplicate_container_ids_fail_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="duplicate container id: nmp-automodel-tasks"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts( + {"type": "sdk", "id": "nemo-platform"}, + {"type": "container", "id": "nmp-automodel-tasks"}, + {"type": "container", "id": "nmp-automodel-tasks"}, + ), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_unsafe_container_id_fails_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="container id must be a safe single path segment"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts( + {"type": "sdk", "id": "nemo-platform"}, + {"type": "container", "id": "../evil"}, + ), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_duplicate_selected_sdk_ids_fail_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="duplicate sdk id: nemo-platform"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=tmp_path / "downloaded-artifacts", + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts( + {"type": "sdk", "id": "nemo-platform"}, + {"type": "sdk", "id": "nemo-platform"}, + ), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) + + +def test_malformed_release_date_json_fails_clearly(tmp_path: Path): + with pytest.raises(BundleMetadataError, match="release_date_json must be valid JSON"): + write_metadata(tmp_path, release_date_json="2026-06-30") + + +@pytest.mark.parametrize( + "metadata_files", + [ + [], + [ + "nemo_platform-1.0.0.dist-info/METADATA", + "other-1.0.0.dist-info/METADATA", + ], + ], +) +def test_missing_or_duplicate_wheel_metadata_fails_clearly(tmp_path: Path, metadata_files: list[str]): + sdk_artifacts_dir = tmp_path / "downloaded-artifacts" + write_wheel(sdk_artifacts_dir, "nemo-platform", metadata_files=metadata_files) + + with pytest.raises(BundleMetadataError, match="expected exactly one METADATA file"): + bundle_metadata.write_release_bundle_metadata( + sdk_artifacts_dir=sdk_artifacts_dir, + bundle_dir=tmp_path / "release-bundle", + selected_artifacts_json=selected_artifacts({"type": "sdk", "id": "nemo-platform"}), + cadence="release", + release_label="1.0.0", + release_date_json="null", + source_sha="a" * 40, + ) diff --git a/uv.lock b/uv.lock index 0bbb03ade0..a708953e70 100644 --- a/uv.lock +++ b/uv.lock @@ -5412,10 +5412,10 @@ requires-dist = [ { name = "nemo-platform-sdk", marker = "extra == 'nmp-common'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'plugins'", editable = "sdk/python/nemo-platform" }, { name = "nemo-platform-sdk", marker = "extra == 'services'", editable = "sdk/python/nemo-platform" }, - { name = "nemo-safe-synthesizer", marker = "extra == 'all'", specifier = "==0.1.7" }, - { name = "nemo-safe-synthesizer", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = "==0.1.7" }, - { name = "nemo-safe-synthesizer", marker = "extra == 'plugins'", specifier = "==0.1.7" }, - { name = "nemo-safe-synthesizer", marker = "extra == 'services'", specifier = "==0.1.7" }, + { name = "nemo-safe-synthesizer", marker = "extra == 'all'", specifier = "==0.1.2" }, + { name = "nemo-safe-synthesizer", marker = "extra == 'nemo-safe-synthesizer-plugin'", specifier = "==0.1.2" }, + { name = "nemo-safe-synthesizer", marker = "extra == 'plugins'", specifier = "==0.1.2" }, + { name = "nemo-safe-synthesizer", marker = "extra == 'services'", specifier = "==0.1.2" }, { name = "nemoguardrails", extras = ["tracing"], marker = "extra == 'all'", specifier = "==0.23.0" }, { name = "nemoguardrails", extras = ["tracing"], marker = "extra == 'guardrails-service'", specifier = "==0.23.0" }, { name = "nemoguardrails", extras = ["tracing"], marker = "extra == 'nemo-guardrails-plugin'", specifier = "==0.23.0" }, @@ -6050,7 +6050,7 @@ dev = [ [[package]] name = "nemo-safe-synthesizer" -version = "0.1.7" +version = "0.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -6066,10 +6066,9 @@ dependencies = [ { name = "setuptools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "structlog", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "tqdm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, - { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/31/573f66c9b87e663d23f061fb1f65ef5b8dc72a0597bff7d4b044e88c2a23/nemo_safe_synthesizer-0.1.7-py3-none-any.whl", hash = "sha256:07ad037e6ded8f7020039fa88efdab4aaaf805ea52fb8238380086e0811eaaf0", size = 586219, upload-time = "2026-07-10T18:34:02.825Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f8/2ec77a08595640b331c420c6978cb626be6f28c770481c0e5a3e900a27ee/nemo_safe_synthesizer-0.1.2-py3-none-any.whl", hash = "sha256:fbf6f9179052d0ac27ad4493238009a5e1c7ae7775354f062d5e90725890b0ce", size = 561741, upload-time = "2026-06-04T20:03:40.12Z" }, ] [[package]] @@ -6127,7 +6126,7 @@ requires-dist = [ { name = "lark", marker = "extra == 'nemo-platform-plugin'", specifier = ">=1.1.0" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, - { name = "nemo-safe-synthesizer", specifier = "==0.1.7" }, + { name = "nemo-safe-synthesizer", specifier = "==0.1.2" }, { name = "openai", marker = "extra == 'nemo-platform-plugin'", specifier = ">=1.109.1" }, { name = "pydantic", marker = "extra == 'nemo-platform-plugin'", specifier = ">=2.10.3" }, { name = "pydantic", extras = ["email"], specifier = ">=2.9.2" }, diff --git a/web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts b/web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts new file mode 100644 index 0000000000..a13495c487 --- /dev/null +++ b/web/packages/sdk/generated/agents/schema/AgentsGetDeploymentLogsParams.ts @@ -0,0 +1,16 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * agents (plugin) + */ + +export type AgentsGetDeploymentLogsParams = { + /** + * @minimum 0 + * @maximum 10000 + */ + tail?: number; +}; diff --git a/web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts b/web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts new file mode 100644 index 0000000000..b2dd176038 --- /dev/null +++ b/web/packages/sdk/generated/agents/schema/DeploymentLogsResponse.ts @@ -0,0 +1,20 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * agents (plugin) + */ +import type { LogLine } from './LogLine.ts'; + +/** + * Response body for ``GET /deployments/{name}/logs``. + */ +export interface DeploymentLogsResponse { + data: LogLine[]; + /** Number of lines actually returned. */ + total_lines: number; + /** Byte offset just past the returned tail; pass as Last-Event-ID to resume the stream without gaps. */ + next_offset: number; +} diff --git a/web/packages/sdk/generated/agents/schema/LogLine.ts b/web/packages/sdk/generated/agents/schema/LogLine.ts new file mode 100644 index 0000000000..1a16b61a43 --- /dev/null +++ b/web/packages/sdk/generated/agents/schema/LogLine.ts @@ -0,0 +1,24 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * agents (plugin) + */ + +/** + * One line shaped to match ``PlatformJobLog`` so Studio's LogViewer renders it as-is. + */ +export interface LogLine { + /** ISO-8601 timestamp parsed from the line; empty when absent. */ + timestamp: string; + /** Empty — kept for shape compatibility with jobs logs. */ + job?: string; + /** Empty — kept for shape compatibility. */ + job_step?: string; + /** Empty — kept for shape compatibility. */ + job_task?: string; + /** The raw log line minus any parsed timestamp prefix. */ + message: string; +} diff --git a/web/packages/studio/env/.env.dev.local.sample b/web/packages/studio/env/.env.dev.local.sample index 10dce482bf..10d1819ee3 100644 --- a/web/packages/studio/env/.env.dev.local.sample +++ b/web/packages/studio/env/.env.dev.local.sample @@ -18,7 +18,7 @@ VITE_FF_BASE_MODELS_ENABLED='true' VITE_FF_CODING_AGENT_STUDIO_ENABLED='false' VITE_FF_CUSTOMIZER_ENABLED='false' VITE_FF_DASHBOARD_ENABLED='false' -VITE_FF_DATA_DESIGNER_ENABLED='true' +VITE_FF_DATA_DESIGNER_ENABLED='false' VITE_FF_DATASETS_ENABLED='true' VITE_FF_DEPLOYMENTS_ENABLED='false' VITE_FF_EVALUATOR_BENCHMARKS_ENABLED='false' diff --git a/web/packages/studio/src/constants/featureFlags/featureFlags.ts b/web/packages/studio/src/constants/featureFlags/featureFlags.ts index d60dc66a46..b02d4c8b1c 100644 --- a/web/packages/studio/src/constants/featureFlags/featureFlags.ts +++ b/web/packages/studio/src/constants/featureFlags/featureFlags.ts @@ -59,7 +59,7 @@ export const flagDefinitions = { codingAgentStudioEnabled: previewFlag('VITE_FF_CODING_AGENT_STUDIO_ENABLED', false), customizerEnabled: previewFlag('VITE_FF_CUSTOMIZER_ENABLED', false), dashboardEnabled: previewFlag('VITE_FF_DASHBOARD_ENABLED', false), - dataDesignerEnabled: previewFlag('VITE_FF_DATA_DESIGNER_ENABLED', true), + dataDesignerEnabled: previewFlag('VITE_FF_DATA_DESIGNER_ENABLED'), datasetsEnabled: previewFlag('VITE_FF_DATASETS_ENABLED', true), deploymentsEnabled: previewFlag('VITE_FF_DEPLOYMENTS_ENABLED'), evaluatorBenchmarksEnabled: previewFlag('VITE_FF_EVALUATOR_BENCHMARKS_ENABLED', false), diff --git a/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx b/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx index ce4b2c26bd..2bda8c418f 100644 --- a/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx +++ b/web/packages/studio/src/routes/DashboardLandingRoute/index.tsx @@ -65,7 +65,6 @@ const LandingComposer = ({ data-testid="dashboard-landing-composer" onSubmit={handleSubmit} className="w-full rounded-lg border border-base bg-surface-base p-2 shadow-xl" - data-tour="dashboard-get-started" >