From 50dd92cfa9ca985cd5e2128e59f9f40c313f9b04 Mon Sep 17 00:00:00 2001 From: Nico Tonozzi Date: Thu, 30 Jul 2026 13:33:18 -0700 Subject: [PATCH] feat(experimentalist): add Tau3 Airline CLI smoke workflow Provide a bounded official-dataset workflow that customers can run through the NeMo CLI while recording Platform traces. Signed-off-by: Nico Tonozzi --- docs/get-started/example-agent.mdx | 50 ++++++++++++- .../experimentalist-smoke.yaml | 21 ++++++ .../tau3-nooa-agent/harbor_wrapper.py | 5 +- .../tau3-nooa-agent/prepare-airline-smoke.sh | 75 +++++++++++++++++++ 4 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml create mode 100755 plugins/nemo-experimentalist/examples/tau3-nooa-agent/prepare-airline-smoke.sh diff --git a/docs/get-started/example-agent.mdx b/docs/get-started/example-agent.mdx index 4f4569bf66..226f334cb9 100644 --- a/docs/get-started/example-agent.mdx +++ b/docs/get-started/example-agent.mdx @@ -1,6 +1,6 @@ --- title: "Get started with an example agent" -description: "" +description: "Run the optimization loop from end to end on τ-Bench" --- This provides a guide for how you can quickly load agent traces into your NeMo Platform, run the analyst agent to discover issues with that agent, and launch an experimentalist to fix those issues. This is meant as an example to show how the whole platform works. You can also jump straight to setting up your agent with NeMo Platform. @@ -78,3 +78,51 @@ uv run --frozen nemo insights analyze \ ``` This will take a few minutes to run. Once it's done, you can navigate to the insights page to see the issues the analyst discovered in the τ-Bench Airline Agent. + +## 5. Prepare a Tau3 Airline smoke dataset + +The next step is to improve the τ-Bench agent using the experimentalist. The +experimentalist will run your evals, debug failures using trace data, understand +the root cause of the failure and attempt to fix it. After it makes the change, +it will run the evals again to validate whether the change improved the +performance of your agent on the evalaution. + +This is a shortened example that only uses a few tasks, but it can still take up to an hour to finish. First, set your environment variables and set up the task dataset: + +```bash +export OPENAI_API_KEY="$INFERENCE_API_KEY" +export OPENAI_BASE_URL=https://inference-api.nvidia.com/v1 +export TAU2_USER_MODEL=openai/openai/openai/gpt-5-mini +export TAU2_NL_ASSERTIONS_MODEL=openai/openai/openai/gpt-5-mini +export AUT_MODEL_NAME=openai/openai/openai/gpt-5-mini +export EXPERIMENTALIST_SMART_MODEL_NAME=openai/openai/openai/gpt-5-mini +export EXPERIMENTALIST_MID_MODEL_NAME=openai/openai/openai/gpt-5-mini +export EXPERIMENTALIST_FAST_MODEL_NAME=openai/openai/openai/gpt-5-mini + +uv run --frozen nemo workspaces create canonical-tau3-airline \ + --description "Tau3 Airline Experimentalist runs" \ + --exist-ok + +plugins/nemo-experimentalist/examples/tau3-nooa-agent/prepare-airline-smoke.sh +``` + +Now we're ready to run the experimentalist! + +```bash +uv run --frozen nemo experimentalist run \ + --no-insight \ + --agent plugins/nemo-experimentalist/examples/tau3-nooa-agent \ + --agent-spec plugins/nemo-experimentalist/examples/tau3-nooa-agent/AGENT-SPEC.md \ + --train-dataset plugins/nemo-experimentalist/tmp/tau3-airline-smoke/train \ + --validation-dataset plugins/nemo-experimentalist/tmp/tau3-airline-smoke/validation \ + --workspace canonical-tau3-airline \ + --framework-skills plugins/nemo-experimentalist/framework-skills/nooa \ + --config plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml \ + --experiment-dir plugins/nemo-experimentalist/tmp/tau3-airline-experimentalist \ + --base-url "$NMP_BASE_URL" +``` + +The trace records include the Experimentalist evaluation ID and Tau3 task ID. +After the run completes, inspect `eval-and-optimize/run.json` for the selected +winner and compare the `agent-0` and `agent-1` directories to review the code +change that was evaluated. diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml new file mode 100644 index 0000000000..2dfa4f9721 --- /dev/null +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +max_rounds: 1 +min_rounds_before_stopping: 1 +max_survivors: 1 +max_candidates: 1 +max_trajectory_tasks: 2 +max_train_batch_tasks: 4 +train_batch_seed: 20260727 +disable_trajectory_scoring: true +disable_convergence_check: true +evaluator: + n_attempts: 1 + n_concurrent_trials: 1 + quiet: true + agent_setup_timeout_multiplier: 2.0 + environment_build_timeout_multiplier: 3.0 +eval_author: + max_traces: 3 + max_validation_repair_attempts: 2 diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/harbor_wrapper.py b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/harbor_wrapper.py index e8c364cf4d..6b1914d9c9 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/harbor_wrapper.py +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/harbor_wrapper.py @@ -137,8 +137,9 @@ async def run( context: AgentContext, ) -> None: env = {name: value for name in MODEL_CREDENTIAL_VARS if (value := os.environ.get(name))} - if self.model_name: - env["NEMO_AGENT_MODEL"] = self.model_name + model_name = self.model_name or os.environ.get("AUT_MODEL_NAME") + if model_name: + env["NEMO_AGENT_MODEL"] = model_name proc = await environment.exec( f"cd /app && uv run --frozen python main.py --prompt {shlex.quote(instruction.strip())}", env=env, diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/prepare-airline-smoke.sh b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/prepare-airline-smoke.sh new file mode 100755 index 0000000000..f57f437e6a --- /dev/null +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/prepare-airline-smoke.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +shopt -s nullglob + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)" +OUTPUT_ROOT="${1:-${PLUGIN_ROOT}/tmp/tau3-airline-smoke}" +SOURCE_ROOT="${OUTPUT_ROOT}/source" +DATASET_ROOT="${SOURCE_ROOT}/tau3-bench" +TRAIN_ROOT="${OUTPUT_ROOT}/train" +VALIDATION_ROOT="${OUTPUT_ROOT}/validation" +DATASET_REF="sierra-research/tau3-bench@1" + +TRAIN_TASKS=( + "tau3-bench__tau3-airline-0" + "tau3-bench__tau3-airline-20" + "tau3-bench__tau3-airline-39" +) +VALIDATION_TASKS=( + "tau3-bench__tau3-airline-3" + "tau3-bench__tau3-airline-36" +) + +if [[ ! -f "${DATASET_ROOT}/${TRAIN_TASKS[0]}/task.toml" ]]; then + mkdir -p "${SOURCE_ROOT}" + ( + cd "${PLUGIN_ROOT}" + uv run --frozen harbor download "${DATASET_REF}" \ + --output-dir "${SOURCE_ROOT}" \ + --export \ + --overwrite + ) +fi + +prepare_split() { + local split_root="$1" + shift + local task_names=("$@") + + if [[ -d "${split_root}" ]]; then + local existing_tasks=("${split_root}"/*/task.toml) + if [[ ${#existing_tasks[@]} -eq ${#task_names[@]} ]]; then + for task_name in "${task_names[@]}"; do + if [[ ! -f "${split_root}/${task_name}/task.toml" ]]; then + echo "Existing split is not the expected dataset: ${split_root}" >&2 + exit 1 + fi + done + echo "Reusing ${split_root}" + return + fi + echo "Existing split is incomplete: ${split_root}" >&2 + exit 1 + fi + + mkdir -p "${split_root}" + for task_name in "${task_names[@]}"; do + local source_task="${DATASET_ROOT}/${task_name}" + if [[ ! -f "${source_task}/task.toml" ]]; then + echo "Downloaded dataset is missing ${task_name}" >&2 + exit 1 + fi + cp -R "${source_task}" "${split_root}/${task_name}" + done +} + +prepare_split "${TRAIN_ROOT}" "${TRAIN_TASKS[@]}" +prepare_split "${VALIDATION_ROOT}" "${VALIDATION_TASKS[@]}" + +echo "Tau3 Airline smoke datasets are ready:" +echo " train: ${TRAIN_ROOT}" +echo " validation: ${VALIDATION_ROOT}"