diff --git a/.agents/skills/skippy-bench/SKILL.md b/.agents/skills/skippy-bench/SKILL.md index 41500b50d..04ef333df 100644 --- a/.agents/skills/skippy-bench/SKILL.md +++ b/.agents/skills/skippy-bench/SKILL.md @@ -106,6 +106,10 @@ Optional future packs are intentionally not wired yet: Keep `sync`/`install` opt-in. Do not make normal `just build` or `cargo build` download external harnesses, datasets, or Docker images. +`eval sync` checks out the fetched upstream ref directly, and `eval run` records +the resolved harness SHA as `harness_commit` in `run.json`. Preserve both +behaviors so benchmark evidence remains reproducible even when definitions use +floating upstream refs. Terminal-Bench should be installed with `uv tool install --python 3.12 terminal-bench`; Python 3.14 currently breaks the `tb` Typer CLI. Treat Docker @@ -149,8 +153,9 @@ path, not a Skippy dataset or harness rewrite. For TTFT/FTTT, use metrics-server correlation rather than harness-only timing. `skippy-bench eval run` and `skippy-bench chat-corpus` create/finalize a -metrics-server run and fail if the metrics report cannot be exported. The -target endpoint must be emitting OTLP for the same run id. Debug telemetry is -required for per-token spans such as `stage.openai_decode_token`; without it, -the JSON report will still include a telemetry block explaining why TTFT/FTTT -was unavailable. +metrics-server run. `eval run` keeps harness success independent from a +finalization/export failure and records telemetry as unavailable; `chat-corpus` +still fails when its metrics report cannot be exported. The target endpoint +must be emitting OTLP for the same run id. Debug telemetry is required for +per-token spans such as `stage.openai_decode_token`; without it, the JSON report +will still include a telemetry block explaining why TTFT/FTTT was unavailable. diff --git a/crates/skippy-bench/README.md b/crates/skippy-bench/README.md index bc2c8737b..c223eed0f 100644 --- a/crates/skippy-bench/README.md +++ b/crates/skippy-bench/README.md @@ -143,6 +143,10 @@ fails before launching the native harness. `~/.cache/mesh-llm/skippy-bench/harnesses/` by default. Use `--cache-root` to override that location. Use `--dry-run` with `sync` or `run` to inspect the commands without cloning, pulling Docker images, or launching a benchmark. +For an existing harness clone, `sync` fetches the configured upstream ref and +checks out the fetched commit directly, so a stale local branch cannot leave the +cache behind upstream. Each run records that resolved commit in +`run.json` as `harness_commit` for reproducible benchmark evidence. Before launching native harness traffic, `eval run` enforces the same required tool checks as `eval doctor`, including Docker container-start readiness for Docker-backed evals. @@ -194,10 +198,13 @@ local-model path. Set `SWE_BENCH_PRO_USE_LOCAL_DOCKER=0` when running the full harness in a different environment such as Modal. Every `eval run` writes `run.json` under the run directory with command status, -raw artifact paths, wall-clock duration, and normalized metrics where the -harness exposes them. `speed-bench` records request counts, latency, +the resolved harness commit, raw artifact paths, wall-clock duration, and +normalized metrics where the harness exposes them. `speed-bench` records request counts, latency, prompt/completion/total token counts, prompt and completion tok/s, and draft -acceptance rate when the server returns llama.cpp-compatible `timings`. +acceptance rate when the server returns llama.cpp-compatible `timings`. Because +the upstream SPEED-Bench script does not expose an authorization argument, +SkippyBench launches it through a small adapter that adds the bearer token from +`--api-key` without modifying the upstream harness. SWE-Bench Pro records OpenAI usage tokens and client-side tok/s when the upstream flow produces them. Terminal-Bench records pass rate, resolved/unresolved task counts, token totals @@ -214,9 +221,11 @@ run id. SkippyBench finalizes and fetches adds a `telemetry` block to `run.json`. When the target emits debug telemetry, SkippyBench derives TTFT/FTTT from the first request span to the first `stage.openai_decode_token` span, plus request and generation latency -aggregates. If the target endpoint is not emitting the requested run id, or if -debug token spans are disabled, the telemetry block records that status rather -than filling misleading values. +aggregates. A finalization or report-fetch failure marks telemetry unavailable +without changing the native harness result in `report.success`. If the target +endpoint is not emitting the requested run id, or if debug token spans are +disabled, the telemetry block records that status rather than filling +misleading values. Optional packs intentionally not wired yet: diff --git a/crates/skippy-bench/src/evals.rs b/crates/skippy-bench/src/evals.rs index 33d41dca2..f2b4ca268 100644 --- a/crates/skippy-bench/src/evals.rs +++ b/crates/skippy-bench/src/evals.rs @@ -26,13 +26,19 @@ const CORE_EVALS: [EvalId; 4] = [ EvalId::SweBenchPro, EvalId::McpAtlas, ]; +mod adapters; +mod doctor; +mod registry; +mod run; +mod sync; + pub fn eval_command(args: EvalArgs) -> Result<()> { match args.command { - EvalCommandKind::List(args) => list_evals(args), - EvalCommandKind::Info(args) => info_eval(args), - EvalCommandKind::Sync(args) | EvalCommandKind::Install(args) => sync_evals(args), - EvalCommandKind::Doctor(args) => doctor_evals(args), - EvalCommandKind::Run(args) => run_eval(args), + EvalCommandKind::List(args) => registry::list_evals(args), + EvalCommandKind::Info(args) => registry::info_eval(args), + EvalCommandKind::Sync(args) | EvalCommandKind::Install(args) => sync::sync_evals(args), + EvalCommandKind::Doctor(args) => doctor::doctor_evals(args), + EvalCommandKind::Run(args) => run::run_eval(args), } } @@ -89,6 +95,7 @@ struct RunReport { base_url: String, endpoint_concurrency: usize, run_dir: String, + harness_commit: Option, dry_run: bool, command: String, exit_status: Option, @@ -133,6 +140,7 @@ struct CommandSpec { args: Vec, cwd: Option, envs: Vec<(String, String)>, + secret_envs: Vec<(String, String)>, } impl CommandSpec { @@ -142,6 +150,7 @@ impl CommandSpec { args: Vec::new(), cwd: None, envs: Vec::new(), + secret_envs: Vec::new(), } } @@ -160,11 +169,21 @@ impl CommandSpec { self } + fn secret_env(mut self, key: impl Into, value: impl Into) -> Self { + self.secret_envs.push((key.into(), value.into())); + self + } + fn display(&self) -> String { let envs = self .envs .iter() .map(|(key, value)| format!("{key}={}", shell_quote(value))) + .chain( + self.secret_envs + .iter() + .map(|(key, _)| format!("{key}=")), + ) .collect::>() .join(" "); let command = std::iter::once(shell_quote(&self.program)) @@ -195,224 +214,12 @@ impl CommandSpec { for (key, value) in &self.envs { command.env(key, value); } - command - } -} - -fn list_evals(args: EvalListArgs) -> Result<()> { - let root = cache_root(args.cache_root.clone())?; - let views = selected_evals(&[], EvalPack::Core) - .into_iter() - .map(|definition| eval_view(definition, &root)) - .collect::>(); - if args.json { - println!("{}", serde_json::to_string_pretty(&views)?); - return Ok(()); - } - - println!("SkippyBench external evals"); - println!("cache: {}", root.display()); - for view in views { - let status = if view.installed { - "installed" - } else { - "not installed" - }; - println!( - " {:<16} {:<13} {}", - view.id, - format!("[{status}]"), - view.description - ); - } - Ok(()) -} - -fn info_eval(args: EvalInfoArgs) -> Result<()> { - let root = cache_root(args.cache_root.clone())?; - let definition = definition(args.eval); - let view = eval_view(definition, &root); - if args.json { - println!("{}", serde_json::to_string_pretty(&view)?); - return Ok(()); - } - - println!("{} ({})", view.name, view.id); - println!("description: {}", view.description); - println!("repo: {} @ {}", view.repo_url, view.repo_ref); - println!("pack: {}", view.pack); - println!("disk: {}", view.disk_estimate); - println!("installed: {}", view.installed); - println!("harness: {}", view.harness_dir); - println!("requires: {}", view.required_tools.join(", ")); - print_notes("sync", view.sync_notes); - print_notes("run", view.run_notes); - Ok(()) -} - -fn sync_evals(args: EvalSyncArgs) -> Result<()> { - let root = cache_root(args.cache_root)?; - fs::create_dir_all(harness_root(&root)).with_context(|| { - format!( - "create eval harness cache {}", - harness_root(&root).display() - ) - })?; - - for definition in selected_evals(&args.evals, args.pack) { - println!("sync {}", definition.id.as_str()); - sync_repo(definition, &root, args.dry_run)?; - for step in sync_steps(definition, &root) { - run_step(&step, args.dry_run)?; - } - } - Ok(()) -} - -fn doctor_evals(args: EvalDoctorArgs) -> Result<()> { - let root = cache_root(args.cache_root)?; - let views = selected_evals(&args.evals, args.pack) - .into_iter() - .map(|definition| doctor_view(definition, &root)) - .collect::>(); - if args.json { - println!("{}", serde_json::to_string_pretty(&views)?); - return Ok(()); - } - - println!("SkippyBench eval doctor"); - println!("cache: {}", root.display()); - for view in views { - println!(); - println!("{}:", view.eval_id); - println!(" installed: {}", view.installed); - println!(" harness: {}", view.harness_dir); - for check in view.checks { - let status = if check.ok { "ok" } else { "missing" }; - println!(" {status:<7} {} - {}", check.name, check.detail); - } - } - Ok(()) -} - -fn run_eval(args: EvalRunArgs) -> Result<()> { - if args.endpoint_concurrency == 0 { - bail!("--endpoint-concurrency must be greater than zero"); - } - let root = absolute_path(cache_root(args.cache_root.clone())?)?; - let definition = definition(args.eval); - let run_dir = absolute_path(output_dir(args.output_dir.clone(), args.eval)?)?; - let run_id = args - .run_id - .clone() - .unwrap_or_else(|| eval_run_id(args.eval)); - let metrics_run_id = args - .metrics_run_id - .clone() - .unwrap_or_else(|| run_id.clone()); - let metrics_http = args.metrics_http.trim_end_matches('/').to_string(); - - if !args.dry_run { - let harness = harness_dir(&root, definition); - if !harness.exists() { - bail!( - "{} is not installed at {}; run `skippy-bench eval sync {}` first", - definition.id.as_str(), - harness.display(), - definition.id.as_str() - ); - } - preflight_eval_run(definition)?; - } - - fs::create_dir_all(run_dir.join("raw")) - .with_context(|| format!("create eval run dir {}", run_dir.display()))?; - - let command = run_command(definition, &args, &root, &run_dir)?; - let display = command.display(); - println!("{display}"); - - let mut report = RunReport { - run_id: run_id.clone(), - eval_id: definition.id.as_str(), - model: args.model.clone(), - base_url: args.base_url.clone(), - endpoint_concurrency: args.endpoint_concurrency, - run_dir: run_dir.display().to_string(), - dry_run: args.dry_run, - command: display, - exit_status: None, - success: args.dry_run, - timed_out: false, - timeout_secs: args.timeout_secs, - harness_timeout_secs: args.harness_timeout_secs, - stdout_path: None, - stderr_path: None, - metrics: EvalMetrics::default(), - telemetry: telemetry_report::pending(&metrics_http, &metrics_run_id), - artifacts: run_artifacts(definition, &run_dir), - }; - - if !args.dry_run { - create_metrics_run(&args, &run_id, &metrics_run_id)?; - let started = Instant::now(); - let stdout_path = run_dir.join("raw").join("stdout.log"); - let stderr_path = run_dir.join("raw").join("stderr.log"); - let outcome = run_command_with_timeout( - &command, - args.harness_timeout_secs.map(Duration::from_secs), - &stdout_path, - &stderr_path, - ) - .with_context(|| format!("run {}", definition.id.as_str()))?; - let duration_ms = started.elapsed().as_secs_f64() * 1000.0; - report.exit_status = outcome.exit_status; - report.success = outcome.success; - report.timed_out = outcome.timed_out; - report.stdout_path = Some(stdout_path.display().to_string()); - report.stderr_path = Some(stderr_path.display().to_string()); - report.metrics = collect_metrics(definition, &run_dir, duration_ms); - if let Err(error) = collect_telemetry(&metrics_http, &metrics_run_id, &run_dir) - .map(|telemetry| report.telemetry = telemetry) - { - report.telemetry = - telemetry_report::unavailable(&metrics_http, &metrics_run_id, &error); - report.success = false; + for (key, value) in &self.secret_envs { + command.env(key, value); } + command } - - let report_path = run_dir.join("run.json"); - fs::write(&report_path, serde_json::to_vec_pretty(&report)?) - .with_context(|| format!("write {}", report_path.display()))?; - if !report.success { - bail!( - "{} failed; see {}", - definition.id.as_str(), - report_path.display() - ); - } - Ok(()) -} - -fn preflight_eval_run(definition: EvalDefinition) -> Result<()> { - let failed = definition - .required_tools - .iter() - .map(|tool| tool_check(tool)) - .filter(|check| !check.ok) - .map(|check| format!("{} - {}", check.name, check.detail)) - .collect::>(); - if failed.is_empty() { - return Ok(()); - } - bail!( - "{} prerequisites failed; run `skippy-bench eval doctor {}` for details:\n {}", - definition.id.as_str(), - definition.id.as_str(), - failed.join("\n ") - ) } - struct CommandOutcome { exit_status: Option, success: bool, @@ -504,612 +311,6 @@ fn terminate_child_signal(child: &mut Child, signal: &str) { } } -fn definition(id: EvalId) -> EvalDefinition { - match id { - EvalId::SpeedBench => EvalDefinition { - id, - name: "llama.cpp SPEED-Bench", - repo_url: "https://github.com/ggml-org/llama.cpp.git", - repo_ref: "master", - cache_name: "llama.cpp", - description: "OpenAI-compatible serving latency and throughput benchmark.", - disk_estimate: "1-3GB including clone and Python dataset cache", - required_tools: &["git", "uv", "python3"], - sync_notes: &["Clones llama.cpp; SPEED-Bench data is fetched by the Python runner."], - run_notes: &[ - "Runs the upstream SPEED-Bench client with category=all and no sample limit.", - ], - }, - EvalId::TerminalBench => EvalDefinition { - id, - name: "Terminal-Bench", - repo_url: "https://github.com/harbor-framework/terminal-bench.git", - repo_ref: "main", - cache_name: "terminal-bench", - description: "Agent benchmark for real terminal tasks in Docker sandboxes.", - disk_estimate: "5-20GB depending on Docker images", - required_tools: &["git", "uv", "python3.12", "tb", "docker"], - sync_notes: &["Clones task repo and installs the terminal-bench uv tool."], - run_notes: &["Runs the Terminal-Bench CLI against the full selected dataset."], - }, - EvalId::SweBenchPro => EvalDefinition { - id, - name: "SWE-Bench Pro", - repo_url: "https://github.com/scaleapi/SWE-bench_Pro-os.git", - repo_ref: "main", - cache_name: "swe-bench-pro", - description: "Long-horizon software-engineering patch benchmark.", - disk_estimate: "10GB+ before task Docker images", - required_tools: &["git", "uv", "python3", "docker"], - sync_notes: &["Clones the official repo and initializes submodules."], - run_notes: &[ - "Generates SWE-agent instances from the full SWE-Bench Pro test split.", - "Runs the synced SWE-agent scaffold, gathers .pred patches, then invokes swe_bench_pro_eval.py.", - ], - }, - EvalId::McpAtlas => EvalDefinition { - id, - name: "MCP-Atlas", - repo_url: "https://github.com/scaleapi/mcp-atlas.git", - repo_ref: "main", - cache_name: "mcp-atlas", - description: "Tool-use benchmark over real MCP servers and tasks.", - disk_estimate: "10GB+ including Docker image", - required_tools: &["git", "uv", "python3", "docker", "make", "curl"], - sync_notes: &["Clones repo and pulls the prebuilt MCP-Atlas Docker image."], - run_notes: &[ - "Starts the MCP environment and completion service when they are not already running.", - "Runs the MCP-Atlas completion and scoring scripts with --no-filter and without --num-tasks or tool_choice overrides.", - ], - }, - } -} - -fn selected_evals(requested: &[EvalId], pack: EvalPack) -> Vec { - let ids = if requested.is_empty() { - match pack { - EvalPack::Core => CORE_EVALS.to_vec(), - } - } else { - requested.to_vec() - }; - ids.into_iter().map(definition).collect() -} - -fn eval_view(definition: EvalDefinition, root: &Path) -> EvalView { - let harness_dir = harness_dir(root, definition); - EvalView { - id: definition.id.as_str(), - name: definition.name, - pack: "core", - repo_url: definition.repo_url, - repo_ref: definition.repo_ref, - installed: harness_dir.exists(), - harness_dir: harness_dir.display().to_string(), - description: definition.description, - disk_estimate: definition.disk_estimate, - required_tools: definition.required_tools, - sync_notes: definition.sync_notes, - run_notes: definition.run_notes, - } -} - -fn doctor_view(definition: EvalDefinition, root: &Path) -> DoctorView { - let harness = harness_dir(root, definition); - let mut checks = definition - .required_tools - .iter() - .map(|tool| tool_check(tool)) - .collect::>(); - if definition.id == EvalId::McpAtlas { - checks.push(port_check("mcp-atlas agent environment", 1984)); - checks.push(port_check("mcp-atlas completion service", 3000)); - } - DoctorView { - eval_id: definition.id.as_str(), - installed: harness.exists(), - harness_dir: harness.display().to_string(), - checks, - } -} - -fn tool_check(tool: &str) -> DoctorCheck { - if tool == "docker" { - return docker_check(); - } - let ok = command_exists(tool); - DoctorCheck { - name: tool.to_string(), - ok, - detail: if ok { - "found on PATH".to_string() - } else { - "not found on PATH".to_string() - }, - } -} - -fn docker_check() -> DoctorCheck { - let installed = command_exists("docker"); - if !installed { - return DoctorCheck { - name: "docker".to_string(), - ok: false, - detail: "docker CLI not found on PATH".to_string(), - }; - } - - let Some(info) = run_probe_command( - CommandSpec::new("docker").args(["info"]), - "docker-info", - Duration::from_secs(10), - ) else { - return DoctorCheck { - name: "docker".to_string(), - ok: false, - detail: "docker CLI found, but daemon did not answer within 10s".to_string(), - }; - }; - if !info.outcome.success { - return DoctorCheck { - name: "docker".to_string(), - ok: false, - detail: format!( - "docker CLI found, but daemon is not reachable{}", - probe_detail_suffix(&info) - ), - }; - } - - let container_name = format!("skippybench-doctor-{}", unix_millis().unwrap_or_default()); - let start = run_probe_command( - CommandSpec::new("docker").args([ - "run", - "--rm", - "--name", - &container_name, - "--pull=missing", - "hello-world", - ]), - "docker-run", - Duration::from_secs(60), - ); - cleanup_docker_container(&container_name); - let Some(start) = start else { - return DoctorCheck { - name: "docker".to_string(), - ok: false, - detail: - "docker daemon is reachable, but a hello-world container did not finish within 60s" - .to_string(), - }; - }; - let ok = start.outcome.success; - DoctorCheck { - name: "docker".to_string(), - ok, - detail: if ok { - "docker daemon is reachable and can start containers".to_string() - } else { - format!( - "docker daemon is reachable, but hello-world container start failed{}", - probe_detail_suffix(&start) - ) - }, - } -} - -struct ProbeCommandResult { - outcome: CommandOutcome, - stdout: String, - stderr: String, -} - -fn run_probe_command( - spec: CommandSpec, - label: &str, - timeout: Duration, -) -> Option { - let run_dir = temp_probe_dir(label)?; - let stdout_path = run_dir.join("stdout.log"); - let stderr_path = run_dir.join("stderr.log"); - let outcome = - run_command_with_timeout(&spec, Some(timeout), &stdout_path, &stderr_path).ok()?; - let stdout = fs::read_to_string(&stdout_path).unwrap_or_default(); - let stderr = fs::read_to_string(&stderr_path).unwrap_or_default(); - let _ = fs::remove_dir_all(run_dir); - Some(ProbeCommandResult { - outcome, - stdout, - stderr, - }) -} - -fn temp_probe_dir(label: &str) -> Option { - let millis = unix_millis().ok()?; - let dir = env::temp_dir().join(format!( - "skippy-bench-{label}-{}-{millis}", - std::process::id() - )); - fs::create_dir_all(&dir).ok()?; - Some(dir) -} - -fn cleanup_docker_container(name: &str) { - let _ = Command::new("docker").args(["rm", "-f", name]).status(); -} - -fn probe_detail_suffix(result: &ProbeCommandResult) -> String { - if result.outcome.timed_out { - return " (timed out)".to_string(); - } - let detail = - first_nonempty_line(&result.stderr).or_else(|| first_nonempty_line(&result.stdout)); - detail.map(|line| format!(": {line}")).unwrap_or_default() -} - -fn first_nonempty_line(text: &str) -> Option { - text.lines() - .map(str::trim) - .find(|line| !line.is_empty()) - .map(str::to_string) -} - -fn port_check(name: &str, port: u16) -> DoctorCheck { - let ok = localhost_port_ready(port); - DoctorCheck { - name: name.to_string(), - ok, - detail: if ok { - format!("localhost:{port} is reachable") - } else { - format!("expected on localhost:{port} when running this eval") - }, - } -} - -fn localhost_port_ready(port: u16) -> bool { - let timeout = Duration::from_millis(500); - ("127.0.0.1", port) - .to_socket_addrs() - .ok() - .into_iter() - .flatten() - .any(|addr| TcpStream::connect_timeout(&addr, timeout).is_ok()) -} - -fn sync_repo(definition: EvalDefinition, root: &Path, dry_run: bool) -> Result<()> { - let target = harness_dir(root, definition); - if target.exists() { - run_step( - &CommandSpec::new("git").args(["-C", &target.display().to_string(), "fetch", "origin"]), - dry_run, - )?; - run_step( - &CommandSpec::new("git").args([ - "-C", - &target.display().to_string(), - "checkout", - definition.repo_ref, - ]), - dry_run, - )?; - return Ok(()); - } - - run_step( - &CommandSpec::new("git").args([ - "clone", - "--recurse-submodules", - "--branch", - definition.repo_ref, - definition.repo_url, - &target.display().to_string(), - ]), - dry_run, - ) -} - -fn sync_steps(definition: EvalDefinition, root: &Path) -> Vec { - let harness = harness_dir(root, definition); - match definition.id { - EvalId::SpeedBench => Vec::new(), - EvalId::TerminalBench => { - vec![CommandSpec::new("uv").args([ - "tool", - "install", - "--python", - "3.12", - "terminal-bench", - ])] - } - EvalId::SweBenchPro => vec![ - CommandSpec::new("git") - .args(["submodule", "update", "--init", "--recursive"]) - .cwd(harness), - ], - EvalId::McpAtlas => { - vec![CommandSpec::new("docker").args(["pull", "ghcr.io/scaleapi/mcp-atlas:1.2.5"])] - } - } -} - -fn run_artifacts(definition: EvalDefinition, run_dir: &Path) -> Vec { - let mut artifacts = vec![ - RunArtifact { - kind: "stdout", - path: run_dir.join("raw/stdout.log").display().to_string(), - }, - RunArtifact { - kind: "stderr", - path: run_dir.join("raw/stderr.log").display().to_string(), - }, - ]; - match definition.id { - EvalId::SpeedBench => artifacts.push(RunArtifact { - kind: "speed-bench-json", - path: speed_bench_output_path(run_dir).display().to_string(), - }), - EvalId::SweBenchPro => artifacts.extend([ - RunArtifact { - kind: "swe-bench-pro-sweagent-results", - path: swe_bench_pro_sweagent_output_path(run_dir) - .display() - .to_string(), - }, - RunArtifact { - kind: "swe-bench-pro-patches-json", - path: swe_bench_pro_patches_path(run_dir).display().to_string(), - }, - RunArtifact { - kind: "swe-bench-pro-eval-json", - path: swe_bench_pro_output_path(run_dir).display().to_string(), - }, - ]), - EvalId::McpAtlas => artifacts.extend([ - RunArtifact { - kind: "mcp-atlas-completion-results-csv", - path: mcp_atlas_output_path(run_dir).display().to_string(), - }, - RunArtifact { - kind: "mcp-atlas-score-dir", - path: mcp_atlas_score_dir(run_dir).display().to_string(), - }, - ]), - EvalId::TerminalBench => artifacts.push(RunArtifact { - kind: "terminal-bench-results", - path: terminal_bench_output_path(run_dir).display().to_string(), - }), - } - artifacts -} - -fn collect_metrics(definition: EvalDefinition, run_dir: &Path, duration_ms: f64) -> EvalMetrics { - let mut metrics = match definition.id { - EvalId::SpeedBench => speed_bench_metrics(run_dir).unwrap_or_default(), - EvalId::SweBenchPro => swe_bench_pro_metrics(run_dir).unwrap_or_default(), - EvalId::McpAtlas => mcp_atlas_metrics(run_dir).unwrap_or_default(), - EvalId::TerminalBench => terminal_bench_metrics(run_dir).unwrap_or_default(), - }; - metrics.duration_ms = Some(duration_ms); - fill_client_rates(&mut metrics, duration_ms); - metrics -} - -fn create_metrics_run(args: &EvalRunArgs, eval_run_id: &str, metrics_run_id: &str) -> Result<()> { - let config = json!({ - "eval_run_id": eval_run_id, - "mode": "skippy-bench-external-eval", - "eval_id": args.eval.as_str(), - "model": args.model, - "base_url": args.base_url, - }); - telemetry_report::create_run(&args.metrics_http, metrics_run_id, &config) -} - -fn collect_telemetry( - metrics_http: &str, - metrics_run_id: &str, - run_dir: &Path, -) -> Result { - telemetry_report::finalize_and_collect( - metrics_http, - metrics_run_id, - &metrics_report_path(run_dir), - ) -} - -fn speed_bench_metrics(run_dir: &Path) -> Result { - let value = read_json(&speed_bench_output_path(run_dir))?; - let overall = value - .get("summary") - .and_then(Value::as_array) - .and_then(|rows| { - rows.iter() - .find(|row| string_field(row, "category") == Some("overall")) - }); - let results = value - .get("results") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - let mut metrics = EvalMetrics { - request_count: value.get("completed_samples").and_then(Value::as_u64), - failed_count: value.get("failed_samples").and_then(Value::as_u64), - ..EvalMetrics::default() - }; - metrics.prompt_tokens = sum_u64_field(results, "prompt_tokens"); - metrics.completion_tokens = sum_u64_field(results, "completion_tokens"); - metrics.total_tokens = sum_u64_field(results, "total_tokens"); - metrics.draft_tokens = sum_u64_field(results, "draft_n"); - metrics.draft_accepted_tokens = sum_u64_field(results, "draft_n_accepted"); - if let Some(overall) = overall { - metrics.prompt_tok_s = numeric_field(overall, "avg_prompt_t_s"); - metrics.completion_tok_s = numeric_field(overall, "avg_pred_t_s"); - metrics.avg_latency_ms = - numeric_field(overall, "avg_latency").map(|seconds| seconds * 1000.0); - metrics.draft_accept_rate = numeric_field(overall, "accept_rate"); - } - Ok(metrics) -} - -fn swe_bench_pro_metrics(run_dir: &Path) -> Result { - let value = read_json(&swe_bench_pro_output_path(run_dir))?; - let rows = value - .get("rows") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - let mut metrics = EvalMetrics { - request_count: swe_bench_pro_request_count(&value, rows), - failed_count: swe_bench_pro_failed_count(&value), - pass_rate: swe_bench_pro_pass_rate(&value), - ..EvalMetrics::default() - }; - metrics.prompt_tokens = sum_nested_usage(rows, "prompt_tokens"); - metrics.completion_tokens = sum_nested_usage(rows, "completion_tokens"); - metrics.total_tokens = sum_nested_usage(rows, "total_tokens"); - Ok(metrics) -} - -fn swe_bench_pro_request_count(value: &Value, rows: &[Value]) -> Option { - value - .get("total_instances") - .or_else(|| value.get("total")) - .or_else(|| value.get("n_total")) - .and_then(Value::as_u64) - .or_else(|| (!rows.is_empty()).then_some(rows.len() as u64)) -} - -fn swe_bench_pro_failed_count(value: &Value) -> Option { - value - .get("failed_instances") - .or_else(|| value.get("unresolved_instances")) - .or_else(|| value.get("n_unresolved")) - .and_then(Value::as_u64) -} - -fn swe_bench_pro_pass_rate(value: &Value) -> Option { - if let Some(rate) = value - .get("pass_rate") - .or_else(|| value.get("resolved_rate")) - .or_else(|| value.get("accuracy")) - .and_then(Value::as_f64) - { - return Some(rate); - } - let resolved = value - .get("resolved_instances") - .or_else(|| value.get("n_resolved")) - .and_then(Value::as_u64)?; - let total = value - .get("total_instances") - .or_else(|| value.get("total")) - .or_else(|| value.get("n_total")) - .and_then(Value::as_u64)?; - (total > 0).then_some(resolved as f64 / total as f64) -} - -fn terminal_bench_metrics(run_dir: &Path) -> Result { - let value = read_json(&terminal_bench_results_path(run_dir)?)?; - let results = value - .get("results") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - let mut metrics = EvalMetrics { - request_count: Some(results.len() as u64), - failed_count: value.get("n_unresolved").and_then(Value::as_u64), - pass_rate: value.get("accuracy").and_then(Value::as_f64), - ..EvalMetrics::default() - }; - metrics.prompt_tokens = sum_u64_field(results, "total_input_tokens"); - metrics.completion_tokens = sum_u64_field(results, "total_output_tokens"); - metrics.total_tokens = match (metrics.prompt_tokens, metrics.completion_tokens) { - (Some(prompt), Some(completion)) => Some(prompt + completion), - _ => None, - }; - Ok(metrics) -} - -fn terminal_bench_results_path(run_dir: &Path) -> Result { - let root = terminal_bench_output_path(run_dir); - for entry in fs::read_dir(&root).with_context(|| format!("read {}", root.display()))? { - let entry = entry?; - let path = entry.path().join("results.json"); - if path.is_file() { - return Ok(path); - } - } - bail!("no Terminal-Bench results.json under {}", root.display()) -} - -fn mcp_atlas_metrics(run_dir: &Path) -> Result { - let mut reader = csv::Reader::from_path(mcp_atlas_output_path(run_dir))?; - let data_rows = reader.records().filter(|record| record.is_ok()).count(); - Ok(EvalMetrics { - request_count: Some(data_rows as u64), - ..EvalMetrics::default() - }) -} - -fn fill_client_rates(metrics: &mut EvalMetrics, duration_ms: f64) { - if duration_ms <= 0.0 { - return; - } - let seconds = duration_ms / 1000.0; - if metrics.prompt_tok_s.is_none() { - metrics.prompt_tok_s = metrics.prompt_tokens.map(|tokens| tokens as f64 / seconds); - } - if metrics.completion_tok_s.is_none() { - metrics.completion_tok_s = metrics - .completion_tokens - .map(|tokens| tokens as f64 / seconds); - } - metrics.total_tok_s = metrics.total_tokens.map(|tokens| tokens as f64 / seconds); -} - -fn read_json(path: &Path) -> Result { - let bytes = fs::read(path).with_context(|| format!("read {}", path.display()))?; - serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display())) -} - -fn sum_u64_field(rows: &[Value], key: &str) -> Option { - let mut total = 0; - let mut found = false; - for row in rows { - if let Some(value) = row.get(key).and_then(Value::as_u64) { - total += value; - found = true; - } - } - found.then_some(total) -} - -fn sum_nested_usage(rows: &[Value], key: &str) -> Option { - let mut total = 0; - let mut found = false; - for row in rows { - if let Some(value) = row - .get("usage") - .and_then(|usage| usage.get(key)) - .and_then(Value::as_u64) - { - total += value; - found = true; - } - } - found.then_some(total) -} - -fn numeric_field(value: &Value, key: &str) -> Option { - value.get(key).and_then(Value::as_f64) -} - -fn string_field<'a>(value: &'a Value, key: &str) -> Option<&'a str> { - value.get(key).and_then(Value::as_str) -} - fn env_concurrency_matching_endpoint( var_name: &str, endpoint_concurrency: usize, @@ -1134,610 +335,6 @@ fn env_concurrency_matching_endpoint( } } -fn run_command( - definition: EvalDefinition, - args: &EvalRunArgs, - root: &Path, - run_dir: &Path, -) -> Result { - Ok(match definition.id { - EvalId::SpeedBench => speed_bench_command(definition, args, root, run_dir), - EvalId::TerminalBench => terminal_bench_command(args, run_dir), - EvalId::SweBenchPro => swe_bench_pro_command(args, root, run_dir)?, - EvalId::McpAtlas => mcp_atlas_command(args, root, run_dir)?, - }) -} - -fn speed_bench_command( - definition: EvalDefinition, - args: &EvalRunArgs, - root: &Path, - run_dir: &Path, -) -> CommandSpec { - let harness = harness_dir(root, definition); - let requirements = harness.join("tools/server/bench/speed-bench/requirements.txt"); - let script = harness.join("tools/server/bench/speed-bench/speed_bench.py"); - let cache_root = env::temp_dir().join("skippy-bench-speed-cache"); - CommandSpec::new("uv") - .args([ - "run".to_string(), - "--with-requirements".to_string(), - requirements.display().to_string(), - "python".to_string(), - script.display().to_string(), - "--url".to_string(), - args.base_url.clone(), - "--model".to_string(), - args.model.clone(), - "--bench".to_string(), - "qualitative".to_string(), - "--category".to_string(), - "all".to_string(), - "--osl".to_string(), - "1024".to_string(), - "--concurrency".to_string(), - args.endpoint_concurrency.to_string(), - "--timeout".to_string(), - args.timeout_secs.to_string(), - "--output".to_string(), - speed_bench_output_path(run_dir).display().to_string(), - ]) - .env( - "XDG_CACHE_HOME", - cache_root.join("xdg").display().to_string(), - ) - .env("HF_HOME", cache_root.join("hf").display().to_string()) - .env( - "HF_DATASETS_CACHE", - cache_root.join("hf-datasets").display().to_string(), - ) - .env("UV_CACHE_DIR", cache_root.join("uv").display().to_string()) -} - -fn terminal_bench_command(args: &EvalRunArgs, run_dir: &Path) -> CommandSpec { - let model = litellm_model_name(&args.model); - CommandSpec::new("tb") - .args([ - "run".to_string(), - "--dataset".to_string(), - "terminal-bench-core==0.1.1".to_string(), - "--agent".to_string(), - "terminus".to_string(), - "--model".to_string(), - model, - "--n-concurrent".to_string(), - args.endpoint_concurrency.to_string(), - "--output-path".to_string(), - terminal_bench_output_path(run_dir).display().to_string(), - "--global-agent-timeout-sec".to_string(), - args.timeout_secs.to_string(), - "--global-test-timeout-sec".to_string(), - "60".to_string(), - "--no-upload-results".to_string(), - "--no-livestream".to_string(), - ]) - .env("OPENAI_BASE_URL", args.base_url.clone()) - .env("OPENAI_API_KEY", args.api_key.clone()) -} - -fn swe_bench_pro_command(args: &EvalRunArgs, root: &Path, run_dir: &Path) -> Result { - let harness = harness_dir(root, definition(EvalId::SweBenchPro)); - let script = run_dir.join("raw/swe-bench-pro-run.sh"); - write_swe_bench_pro_run_script(&script, args, root, &harness, run_dir)?; - Ok(CommandSpec::new("zsh").args([script.display().to_string()])) -} - -fn mcp_atlas_command(args: &EvalRunArgs, root: &Path, run_dir: &Path) -> Result { - let harness = harness_dir(root, definition(EvalId::McpAtlas)); - let script = run_dir.join("raw/mcp-atlas-run.sh"); - write_mcp_atlas_run_script(&script, args, root, &harness, run_dir)?; - Ok(CommandSpec::new("zsh").args([script.display().to_string()])) -} - -fn speed_bench_output_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/speed-bench.json") -} - -fn swe_bench_pro_output_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/swe-bench-pro/eval/eval_results.json") -} - -fn swe_bench_pro_sweagent_output_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/swe-bench-pro/sweagent-results") -} - -fn swe_bench_pro_patches_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/swe-bench-pro/patches.json") -} - -fn mcp_atlas_output_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/mcp-atlas-completion-results.csv") -} - -fn mcp_atlas_score_dir(run_dir: &Path) -> PathBuf { - run_dir.join("raw/mcp-atlas-evaluation-results") -} - -fn terminal_bench_output_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/terminal-bench") -} - -fn metrics_report_path(run_dir: &Path) -> PathBuf { - run_dir.join("raw/metrics-report.json") -} - -fn eval_run_id(eval: EvalId) -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis(); - format!("skippy-eval-{}-{millis}", eval.as_str()) -} - -fn write_mcp_atlas_run_script( - path: &Path, - args: &EvalRunArgs, - cache_root: &Path, - harness: &Path, - run_dir: &Path, -) -> Result<()> { - let raw_dir = run_dir.join("raw"); - let completion_dir = harness.join("services/mcp_eval"); - let default_output_name = format!("skippybench-{}-completion-results.csv", unix_millis()?); - let model_label = model_label(&args.model); - let score_dir = mcp_atlas_score_dir(run_dir); - let completion_concurrency = env_concurrency_matching_endpoint( - "MCP_ATLAS_COMPLETION_CONCURRENCY", - args.endpoint_concurrency, - )?; - let script = format!( - r#"#!/usr/bin/env zsh -set -euo pipefail - -HARNESS={harness} -COMPLETION_DIR={completion_dir} -RAW_DIR={raw_dir} -BASE_URL={base_url} -API_KEY={api_key} -MODEL={model} -EVAL_MODEL=${{EVAL_LLM_MODEL:-$MODEL}} -OUTPUT={output} -OUTPUT_NAME=${{MCP_ATLAS_COMPLETION_OUTPUT_NAME:-{output_name}}} -MODEL_LABEL={model_label} -SCORE_DIR={score_dir} -COMPLETION_CONCURRENCY={completion_concurrency} -SCORE_CONCURRENCY=${{MCP_ATLAS_SCORE_CONCURRENCY:-5}} -HF_HOME_DIR={hf_home} -HF_DATASETS_CACHE_DIR={hf_datasets_cache} -UV_CACHE_DIR_LOCAL={uv_cache_dir} -XDG_CACHE_HOME_DIR={xdg_cache_home} -agent_started=0 -completion_started=0 -completion_pid="" - -mkdir -p \ - "$HF_HOME_DIR" \ - "$HF_DATASETS_CACHE_DIR" \ - "$UV_CACHE_DIR_LOCAL" \ - "$XDG_CACHE_HOME_DIR" -export HF_HOME="$HF_HOME_DIR" -export HF_DATASETS_CACHE="$HF_DATASETS_CACHE_DIR" -export UV_CACHE_DIR="$UV_CACHE_DIR_LOCAL" -export XDG_CACHE_HOME="$XDG_CACHE_HOME_DIR" - -port_ready() {{ - python3 - "$1" <<'PY' -import socket -import sys - -port = int(sys.argv[1]) -sock = socket.socket() -sock.settimeout(0.5) -try: - sock.connect(("127.0.0.1", port)) -except OSError: - sys.exit(1) -finally: - sock.close() -PY -}} - -wait_url() {{ - local name="$1" - local url="$2" - local log="$3" - for _ in {{1..90}}; do - if curl -fsS --max-time 5 "$url" >/dev/null 2>&1; then - return 0 - fi - tail -20 "$log" 2>/dev/null || true - sleep 2 - done - echo "timed out waiting for $name at $url" >&2 - return 1 -}} - -cleanup() {{ - if [[ "$completion_started" == "1" && -n "$completion_pid" ]]; then - kill "$completion_pid" >/dev/null 2>&1 || true - wait "$completion_pid" >/dev/null 2>&1 || true - fi - if [[ "$agent_started" == "1" ]]; then - docker rm -f skippy-bench-mcp-atlas-agent-env >/dev/null 2>&1 || true - fi -}} -trap cleanup EXIT - -mkdir -p "$RAW_DIR" "$SCORE_DIR" -cd "$HARNESS" -cp -n env.template .env >/dev/null 2>&1 || true -if ! docker image inspect agent-environment:latest >/dev/null 2>&1; then - docker tag ghcr.io/scaleapi/mcp-atlas:1.2.5 agent-environment:latest -fi - -if ! port_ready 1984; then - docker rm -f skippy-bench-mcp-atlas-agent-env >/dev/null 2>&1 || true - docker run --rm \ - --name skippy-bench-mcp-atlas-agent-env \ - -p 1984:1984 \ - --env-file .env \ - agent-environment:latest \ - > "$RAW_DIR/mcp-agent-env.log" 2>&1 & - agent_started=1 -fi -wait_url "MCP-Atlas agent environment" \ - "http://localhost:1984/enabled-servers" \ - "$RAW_DIR/mcp-agent-env.log" - -if ! port_ready 3000; then - ( - cd "$COMPLETION_DIR" - LLM_BASE_URL="$BASE_URL" \ - LLM_API_KEY="$API_KEY" \ - OPENAI_BASE_URL="$BASE_URL" \ - OPENAI_API_KEY="$API_KEY" \ - uv run python -m mcp_completion.main - ) > "$RAW_DIR/mcp-completion.log" 2>&1 & - completion_pid="$!" - completion_started=1 -fi -wait_url "MCP-Atlas completion service" \ - "http://localhost:3000/docs" \ - "$RAW_DIR/mcp-completion.log" - -cd "$COMPLETION_DIR" -LLM_BASE_URL="$BASE_URL" \ - LLM_API_KEY="$API_KEY" \ - OPENAI_BASE_URL="$BASE_URL" \ - OPENAI_API_KEY="$API_KEY" \ -uv run python mcp_completion_script.py \ - --model "$MODEL" \ - --input_huggingface ScaleAI/MCP-Atlas \ - --output "$OUTPUT_NAME" \ - --no-filter \ - --concurrency "$COMPLETION_CONCURRENCY" -cp "completion_results/$OUTPUT_NAME" "$OUTPUT" -EVAL_LLM_BASE_URL="${{EVAL_LLM_BASE_URL:-$BASE_URL}}" \ - EVAL_LLM_API_KEY="${{EVAL_LLM_API_KEY:-$API_KEY}}" \ -uv run python mcp_evals_scores.py \ - --input-file "completion_results/$OUTPUT_NAME" \ - --model-label "$MODEL_LABEL" \ - --evaluator-model "$EVAL_MODEL" \ - --output-dir "$SCORE_DIR" \ - --concurrency "$SCORE_CONCURRENCY" -"#, - harness = shell_quote(&harness.display().to_string()), - completion_dir = shell_quote(&completion_dir.display().to_string()), - raw_dir = shell_quote(&raw_dir.display().to_string()), - base_url = shell_quote(&args.base_url), - api_key = shell_quote(&args.api_key), - model = shell_quote(&litellm_model_name(&args.model)), - output = shell_quote(&mcp_atlas_output_path(run_dir).display().to_string()), - output_name = shell_quote(&default_output_name), - model_label = shell_quote(&model_label), - score_dir = shell_quote(&score_dir.display().to_string()), - completion_concurrency = shell_quote(&completion_concurrency), - hf_home = shell_quote(&cache_root.join("hf").display().to_string()), - hf_datasets_cache = shell_quote(&cache_root.join("hf-datasets").display().to_string()), - uv_cache_dir = shell_quote(&cache_root.join("uv").display().to_string()), - xdg_cache_home = shell_quote(&cache_root.join("xdg").display().to_string()), - ); - fs::write(path, script).with_context(|| format!("write {}", path.display())) -} - -fn write_swe_bench_pro_run_script( - path: &Path, - args: &EvalRunArgs, - cache_root: &Path, - harness: &Path, - run_dir: &Path, -) -> Result<()> { - let raw_dir = run_dir.join("raw/swe-bench-pro"); - let instances = raw_dir.join("instances.yaml"); - let expert_instances = raw_dir.join("instances-expert.yaml"); - let sweagent_output = raw_dir.join("sweagent-results"); - let patches = raw_dir.join("patches.json"); - let eval_dir = raw_dir.join("eval"); - let dockerhub_username = - env::var("SWE_BENCH_PRO_DOCKERHUB_USERNAME").unwrap_or_else(|_| "jefzda".to_string()); - let deployment_type = - env::var("SWE_BENCH_PRO_DEPLOYMENT_TYPE").unwrap_or_else(|_| "docker".to_string()); - let num_workers = - env_concurrency_matching_endpoint("SWE_BENCH_PRO_NUM_WORKERS", args.endpoint_concurrency)?; - let eval_workers = env::var("SWE_BENCH_PRO_EVAL_WORKERS").unwrap_or_else(|_| "100".to_string()); - let docker_platform = - env::var("SWE_BENCH_PRO_DOCKER_PLATFORM").unwrap_or_else(|_| "linux/amd64".to_string()); - let parse_function = env::var("SWE_BENCH_PRO_PARSE_FUNCTION").ok(); - let sweagent_python = env::var("SWE_BENCH_PRO_PYTHON").unwrap_or_else(|_| "3.12".to_string()); - let swerex_spec = env::var("SWE_BENCH_PRO_SWEREX_SPEC").ok(); - let swerex_pip_index_url = env::var("SWE_BENCH_PRO_SWEREX_PIP_INDEX_URL") - .unwrap_or_else(|_| "https://pypi.org/simple".to_string()); - let use_local_eval = env::var("SWE_BENCH_PRO_USE_LOCAL_DOCKER") - .map(|value| matches!(value.as_str(), "1" | "true" | "yes")) - .unwrap_or(true); - let local_eval_flag = if use_local_eval { - "--use_local_docker" - } else { - "" - }; - let model = litellm_model_name(&args.model); - let script = format!( - r#"#!/usr/bin/env zsh -set -euo pipefail - -HARNESS={harness} -RAW_DIR={raw_dir} -INSTANCES={instances} -EXPERT_INSTANCES={expert_instances} -SWEAGENT_OUTPUT={sweagent_output} -PATCHES={patches} -EVAL_DIR={eval_dir} -MODEL={model} -BASE_URL={base_url} -API_KEY={api_key} -DOCKERHUB_USERNAME={dockerhub_username} -DEPLOYMENT_TYPE={deployment_type} -NUM_WORKERS={num_workers} -EVAL_WORKERS={eval_workers} -LOCAL_EVAL_FLAG={local_eval_flag} -DOCKER_PLATFORM={docker_platform} -PARSE_FUNCTION={parse_function} -SWEAGENT_PYTHON={sweagent_python} -SWEREX_SPEC={swerex_spec} -SWEREX_PIP_INDEX_URL={swerex_pip_index_url} -HF_HOME_DIR={hf_home} -HF_DATASETS_CACHE_DIR={hf_datasets_cache} -UV_CACHE_DIR_LOCAL={uv_cache_dir} -XDG_CACHE_HOME_DIR={xdg_cache_home} - -mkdir -p \ - "$RAW_DIR" \ - "$SWEAGENT_OUTPUT" \ - "$EVAL_DIR" \ - "$HF_HOME_DIR" \ - "$HF_DATASETS_CACHE_DIR" \ - "$UV_CACHE_DIR_LOCAL" \ - "$XDG_CACHE_HOME_DIR" -export HF_HOME="$HF_HOME_DIR" -export HF_DATASETS_CACHE="$HF_DATASETS_CACHE_DIR" -export UV_CACHE_DIR="$UV_CACHE_DIR_LOCAL" -export XDG_CACHE_HOME="$XDG_CACHE_HOME_DIR" -deployment_timeout_args=() -if [[ "$DEPLOYMENT_TYPE" == "modal" ]]; then - deployment_timeout_args=( - --instances.deployment.startup_timeout 1800 - --instances.deployment.runtime_timeout 3600 - ) -fi -deployment_platform_args=() -if [[ "$DEPLOYMENT_TYPE" == "docker" && -n "$DOCKER_PLATFORM" ]]; then - deployment_platform_args=( - --instances.deployment.platform "$DOCKER_PLATFORM" - ) -fi -deployment_type_args=( - --instances.deployment.type "$DEPLOYMENT_TYPE" -) -expert_instance_args=( - --instances.type file - --instances.path "$INSTANCES" -) -parse_function_args=() -if [[ -n "$PARSE_FUNCTION" ]]; then - parse_function_args=( - --agent.tools.parse_function.type "$PARSE_FUNCTION" - ) -fi -if [[ -z "$SWEREX_SPEC" && "$DEPLOYMENT_TYPE" == "docker" ]]; then - SWEREX_SPEC="swe-rex[modal]==1.4.0" -fi - -cd "$HARNESS" - -uv run \ - --with-requirements requirements.txt \ - --with pyyaml \ - python helper_code/generate_sweagent_instances.py \ - --dockerhub_username "$DOCKERHUB_USERNAME" \ - --output_path "$INSTANCES" - -( - cd SWE-agent - uv venv --clear --python "$SWEAGENT_PYTHON" .venv - uv pip install --python .venv/bin/python -e . - if [[ -n "$SWEREX_SPEC" ]]; then - uv pip install --python .venv/bin/python --upgrade "$SWEREX_SPEC" - fi - if [[ "$DEPLOYMENT_TYPE" == "docker" && -n "$SWEREX_PIP_INDEX_URL" ]]; then - .venv/bin/python - "$SWEREX_PIP_INDEX_URL" <<'PY' -import sys -from pathlib import Path - -import swerex.deployment.docker as docker - -path = Path(docker.__file__) -text = path.read_text() -old = 'f"RUN /root/python3.11/bin/pip3 install --no-cache-dir {{PACKAGE_NAME}}\\n\\n"' -new = ( - f'f"RUN /root/python3.11/bin/pip3 install --index-url {{sys.argv[1]}} ' - '--no-cache-dir {{PACKAGE_NAME}}\\n\\n"' -) -if old in text: - path.write_text(text.replace(old, new)) -elif new not in text: - raise RuntimeError(f"could not patch SWE-ReX Docker pip index in {{path}}") -PY - fi - if [[ "$DEPLOYMENT_TYPE" == "modal" ]]; then - .venv/bin/python swerex_patches/patch.py --yes - fi -) - -if [[ "$DEPLOYMENT_TYPE" == "docker" ]]; then - ( - cd SWE-agent - .venv/bin/python - "$INSTANCES" "$EXPERT_INSTANCES" "$DOCKER_PLATFORM" <<'PY' -import sys - -import yaml -from sweagent.agent.problem_statement import TextProblemStatement -from sweagent.environment.repo import PreExistingRepoConfig -from sweagent.environment.swe_env import EnvironmentConfig -from sweagent.run.batch_instances import BatchInstance -from swerex.deployment.config import DockerDeploymentConfig - -source, target, platform = sys.argv[1:4] -with open(source) as handle: - simple_instances = yaml.safe_load(handle) - -docker_args = [ - "--entrypoint", - "", -] -instances = [] -for item in simple_instances: - deployment = DockerDeploymentConfig( - image=item["image_name"], - docker_args=docker_args, - platform=platform or None, - python_standalone_dir="/root", - startup_timeout=1800, - ) - instance = BatchInstance( - env=EnvironmentConfig( - deployment=deployment, - repo=PreExistingRepoConfig( - repo_name=item.get("repo_name") or "app", - base_commit=item.get("base_commit") or "HEAD", - ), - ), - problem_statement=TextProblemStatement( - text=item["problem_statement"], - id=item["instance_id"], - extra_fields=item.get("extra_fields") or {{}}, - ), - ) - instances.append(instance.model_dump(mode="json", exclude_none=True)) - -with open(target, "w") as handle: - yaml.safe_dump(instances, handle, sort_keys=False) -PY - ) - expert_instance_args=( - --instances.type expert_file - --instances.path "$EXPERT_INSTANCES" - ) - deployment_type_args=() - deployment_platform_args=() -fi - -( - cd SWE-agent - OPENAI_BASE_URL="$BASE_URL" \ - OPENAI_API_KEY="$API_KEY" \ - .venv/bin/sweagent run-batch \ - --config config/tool_use.yaml \ - --output_dir "$SWEAGENT_OUTPUT" \ - --num_workers "$NUM_WORKERS" \ - --random_delay_multiplier 1 \ - "${{expert_instance_args[@]}}" \ - --instances.shuffle=False \ - "${{deployment_type_args[@]}}" \ - "${{deployment_timeout_args[@]}}" \ - "${{deployment_platform_args[@]}}" \ - "${{parse_function_args[@]}}" \ - --agent.model.name "$MODEL" \ - --agent.model.api_base "$BASE_URL" \ - --agent.model.api_key "$API_KEY" \ - --agent.model.max_input_tokens 0 \ - --agent.model.per_instance_cost_limit 0 \ - --agent.model.total_cost_limit 0 -) - -uv run \ - --with-requirements requirements.txt \ - python helper_code/gather_patches.py \ - --directory "$SWEAGENT_OUTPUT" \ - --prefix skippybench \ - --output "$PATCHES" - -uv run \ - --with-requirements requirements.txt \ - --with docker \ - --with modal \ - python swe_bench_pro_eval.py \ - --raw_sample_path helper_code/sweap_eval_full_v2.jsonl \ - --patch_path "$PATCHES" \ - --output_dir "$EVAL_DIR" \ - --scripts_dir run_scripts \ - --num_workers "$EVAL_WORKERS" \ - --dockerhub_username "$DOCKERHUB_USERNAME" \ - $LOCAL_EVAL_FLAG -"#, - harness = shell_quote(&harness.display().to_string()), - raw_dir = shell_quote(&raw_dir.display().to_string()), - instances = shell_quote(&instances.display().to_string()), - expert_instances = shell_quote(&expert_instances.display().to_string()), - sweagent_output = shell_quote(&sweagent_output.display().to_string()), - patches = shell_quote(&patches.display().to_string()), - eval_dir = shell_quote(&eval_dir.display().to_string()), - model = shell_quote(&model), - base_url = shell_quote(&args.base_url), - api_key = shell_quote(&args.api_key), - dockerhub_username = shell_quote(&dockerhub_username), - deployment_type = shell_quote(&deployment_type), - num_workers = shell_quote(&num_workers), - eval_workers = shell_quote(&eval_workers), - docker_platform = shell_quote(&docker_platform), - parse_function = shell_quote(parse_function.as_deref().unwrap_or("")), - sweagent_python = shell_quote(&sweagent_python), - swerex_spec = shell_quote(swerex_spec.as_deref().unwrap_or("")), - swerex_pip_index_url = shell_quote(&swerex_pip_index_url), - local_eval_flag = shell_quote(local_eval_flag), - hf_home = shell_quote(&cache_root.join("hf").display().to_string()), - hf_datasets_cache = shell_quote(&cache_root.join("hf-datasets").display().to_string()), - uv_cache_dir = shell_quote(&cache_root.join("uv").display().to_string()), - xdg_cache_home = shell_quote(&cache_root.join("xdg").display().to_string()), - ); - fs::write(path, script).with_context(|| format!("write {}", path.display())) -} - -fn run_step(step: &CommandSpec, dry_run: bool) -> Result<()> { - println!("{}", step.display()); - if dry_run { - return Ok(()); - } - - let status = step - .command() - .status() - .with_context(|| format!("start {}", step.program))?; - if !status.success() { - bail!("command failed with status {status}: {}", step.display()); - } - Ok(()) -} - fn cache_root(override_root: Option) -> Result { if let Some(root) = override_root { return Ok(root); @@ -1858,6 +455,19 @@ fn shell_quote(raw: &str) -> String { #[cfg(test)] mod tests { use super::*; + use super::{ + adapters::{ + mcp_atlas_command, speed_bench_command, swe_bench_pro_command, terminal_bench_command, + }, + doctor::preflight_eval_run, + registry::{definition, selected_evals}, + run::{ + fill_client_rates, resolved_harness_commit, speed_bench_metrics, + speed_bench_output_path, swe_bench_pro_metrics, swe_bench_pro_output_path, + telemetry_or_unavailable, terminal_bench_metrics, terminal_bench_output_path, + }, + sync::existing_repo_sync_steps, + }; #[test] fn default_pack_selects_core_evals() { @@ -1895,8 +505,10 @@ mod tests { dry_run: true, }; let root = PathBuf::from("/tmp/skippy-cache"); - let run_dir = PathBuf::from("/tmp/skippy-run"); - let command = speed_bench_command(definition(EvalId::SpeedBench), &args, &root, &run_dir); + let run_dir = temp_run_dir("speed-command"); + fs::create_dir_all(run_dir.join("raw")).unwrap(); + let command = + speed_bench_command(definition(EvalId::SpeedBench), &args, &root, &run_dir).unwrap(); assert!(command.args.contains(&"--url".to_string())); assert!( command @@ -1904,6 +516,38 @@ mod tests { .contains(&"http://127.0.0.1:9337/v1".to_string()) ); assert!(command.args.contains(&"tiny-local".to_string())); + assert!( + command.args.contains( + &run_dir + .join("raw/speed-bench-auth.py") + .display() + .to_string() + ) + ); + assert!( + command + .secret_envs + .contains(&("SKIPPY_BENCH_API_KEY".to_string(), "test".to_string())) + ); + assert!(command.envs.contains(&( + "XDG_CACHE_HOME".to_string(), + root.join("speed-cache/xdg").display().to_string() + ))); + assert!( + command + .envs + .contains(&("SKIPPY_BENCH_BASE_URL".to_string(), args.base_url.clone())) + ); + assert!(!command.display().contains("test")); + assert!( + command + .display() + .contains("SKIPPY_BENCH_API_KEY=") + ); + let launcher = fs::read_to_string(run_dir.join("raw/speed-bench-auth.py")).unwrap(); + assert!(launcher.contains("request_origin(url) == benchmark_origin")); + assert!(launcher.contains("headers.setdefault(\"Authorization\"")); + let _ = fs::remove_dir_all(run_dir); } #[test] @@ -1912,7 +556,7 @@ mod tests { eval: EvalId::TerminalBench, base_url: "http://127.0.0.1:9337/v1".to_string(), model: "tiny-local".to_string(), - api_key: "test".to_string(), + api_key: "terminal-secret-value".to_string(), cache_root: None, output_dir: None, timeout_secs: 30, @@ -1936,6 +580,52 @@ mod tests { .envs .contains(&("OPENAI_BASE_URL".to_string(), args.base_url)) ); + assert!(command.secret_envs.contains(&( + "OPENAI_API_KEY".to_string(), + "terminal-secret-value".to_string() + ))); + assert!(!command.display().contains("terminal-secret-value")); + } + + #[test] + fn generated_eval_scripts_do_not_persist_api_keys() { + let root = temp_run_dir("script-secrets-cache"); + let run_dir = temp_run_dir("script-secrets-run"); + fs::create_dir_all(run_dir.join("raw")).unwrap(); + + for eval in [EvalId::McpAtlas, EvalId::SweBenchPro] { + let args = eval_run_args(eval, "literal-secret"); + let command = match eval { + EvalId::McpAtlas => mcp_atlas_command(&args, &root, &run_dir).unwrap(), + EvalId::SweBenchPro => swe_bench_pro_command(&args, &root, &run_dir).unwrap(), + _ => unreachable!(), + }; + let script = fs::read_to_string(&command.args[0]).unwrap(); + + assert!(command.secret_envs.contains(&( + "SKIPPY_BENCH_API_KEY".to_string(), + "literal-secret".to_string() + ))); + assert!(!command.display().contains("literal-secret")); + assert!(!script.contains("literal-secret")); + assert!(script.contains("SKIPPY_BENCH_API_KEY")); + assert!(!script.contains("--agent.model.api_key")); + } + + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(run_dir); + } + + #[test] + fn telemetry_failure_is_reported_as_unavailable() { + let telemetry = telemetry_or_unavailable( + "http://127.0.0.1:18080", + "run-id", + Err(anyhow::anyhow!("collector offline")), + ); + + assert_eq!(telemetry.status, "unavailable"); + assert_eq!(telemetry.detail.as_deref(), Some("collector offline")); } #[test] @@ -1960,6 +650,87 @@ mod tests { assert!(error.contains("skippy-bench eval doctor terminal-bench")); } + #[test] + fn existing_repo_sync_fetches_ref_and_checks_out_fetch_head() { + let steps = existing_repo_sync_steps(Path::new("/tmp/harness"), "main"); + + assert_eq!( + steps[0].args, + ["-C", "/tmp/harness", "fetch", "--prune", "origin", "main"] + ); + assert_eq!( + steps[1].args, + ["-C", "/tmp/harness", "checkout", "--detach", "FETCH_HEAD"] + ); + } + + #[test] + fn resolved_harness_commit_reads_checked_out_revision() { + let root = temp_run_dir("harness-revision"); + let definition = EvalDefinition { + id: EvalId::SpeedBench, + name: "test", + repo_url: "unused", + repo_ref: "main", + cache_name: "test-harness", + description: "test", + disk_estimate: "none", + required_tools: &[], + sync_notes: &[], + run_notes: &[], + }; + let harness = harness_dir(&root, definition); + fs::create_dir_all(&harness).unwrap(); + for args in [ + vec!["init"], + vec!["config", "user.email", "skippy-bench@example.invalid"], + vec!["config", "user.name", "Skippy Bench"], + ] { + assert!( + Command::new("git") + .args(args) + .current_dir(&harness) + .status() + .unwrap() + .success() + ); + } + fs::write(harness.join("README"), "fixture").unwrap(); + assert!( + Command::new("git") + .args(["add", "README"]) + .current_dir(&harness) + .status() + .unwrap() + .success() + ); + assert!( + Command::new("git") + .args(["commit", "-m", "fixture"]) + .current_dir(&harness) + .status() + .unwrap() + .success() + ); + let expected = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&harness) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + assert_eq!( + resolved_harness_commit(&root, definition).unwrap(), + Some(expected) + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn run_command_with_timeout_captures_successful_output() { let run_dir = temp_run_dir("command-success"); @@ -2151,4 +922,22 @@ mod tests { unix_millis().unwrap() )) } + + fn eval_run_args(eval: EvalId, api_key: &str) -> EvalRunArgs { + EvalRunArgs { + eval, + base_url: "http://127.0.0.1:9337/v1".to_string(), + model: "tiny-local".to_string(), + api_key: api_key.to_string(), + cache_root: None, + output_dir: None, + timeout_secs: 30, + harness_timeout_secs: None, + endpoint_concurrency: 1, + run_id: None, + metrics_http: "http://127.0.0.1:18080".to_string(), + metrics_run_id: None, + dry_run: true, + } + } } diff --git a/crates/skippy-bench/src/evals/adapters/mcp_atlas.rs b/crates/skippy-bench/src/evals/adapters/mcp_atlas.rs new file mode 100644 index 000000000..3bf254c43 --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/mcp_atlas.rs @@ -0,0 +1,54 @@ +use super::super::{ + registry::definition, + run::{mcp_atlas_output_path, mcp_atlas_score_dir}, + *, +}; + +pub(in crate::evals) fn mcp_atlas_command( + args: &EvalRunArgs, + root: &Path, + run_dir: &Path, +) -> Result { + let harness = harness_dir(root, definition(EvalId::McpAtlas)); + let script = run_dir.join("raw/mcp-atlas-run.sh"); + write_mcp_atlas_run_script(&script, args, root, &harness, run_dir)?; + Ok(CommandSpec::new("zsh") + .args([script.display().to_string()]) + .secret_env("SKIPPY_BENCH_API_KEY", args.api_key.clone())) +} + +fn write_mcp_atlas_run_script( + path: &Path, + args: &EvalRunArgs, + cache_root: &Path, + harness: &Path, + run_dir: &Path, +) -> Result<()> { + let raw_dir = run_dir.join("raw"); + let completion_dir = harness.join("services/mcp_eval"); + let default_output_name = format!("skippybench-{}-completion-results.csv", unix_millis()?); + let model_label = model_label(&args.model); + let score_dir = mcp_atlas_score_dir(run_dir); + let completion_concurrency = env_concurrency_matching_endpoint( + "MCP_ATLAS_COMPLETION_CONCURRENCY", + args.endpoint_concurrency, + )?; + let script = format!( + include_str!("templates/mcp_atlas_run.sh"), + harness = shell_quote(&harness.display().to_string()), + completion_dir = shell_quote(&completion_dir.display().to_string()), + raw_dir = shell_quote(&raw_dir.display().to_string()), + base_url = shell_quote(&args.base_url), + model = shell_quote(&litellm_model_name(&args.model)), + output = shell_quote(&mcp_atlas_output_path(run_dir).display().to_string()), + output_name = shell_quote(&default_output_name), + model_label = shell_quote(&model_label), + score_dir = shell_quote(&score_dir.display().to_string()), + completion_concurrency = shell_quote(&completion_concurrency), + hf_home = shell_quote(&cache_root.join("hf").display().to_string()), + hf_datasets_cache = shell_quote(&cache_root.join("hf-datasets").display().to_string()), + uv_cache_dir = shell_quote(&cache_root.join("uv").display().to_string()), + xdg_cache_home = shell_quote(&cache_root.join("xdg").display().to_string()), + ); + fs::write(path, script).with_context(|| format!("write {}", path.display())) +} diff --git a/crates/skippy-bench/src/evals/adapters/mod.rs b/crates/skippy-bench/src/evals/adapters/mod.rs new file mode 100644 index 000000000..56f6dc40e --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/mod.rs @@ -0,0 +1,25 @@ +use super::*; + +mod mcp_atlas; +mod speed_bench; +mod swe_bench_pro; +mod terminal_bench; + +pub(super) use self::{ + mcp_atlas::mcp_atlas_command, speed_bench::speed_bench_command, + swe_bench_pro::swe_bench_pro_command, terminal_bench::terminal_bench_command, +}; + +pub(super) fn run_command( + definition: EvalDefinition, + args: &EvalRunArgs, + root: &Path, + run_dir: &Path, +) -> Result { + Ok(match definition.id { + EvalId::SpeedBench => speed_bench_command(definition, args, root, run_dir)?, + EvalId::TerminalBench => terminal_bench_command(args, run_dir), + EvalId::SweBenchPro => swe_bench_pro_command(args, root, run_dir)?, + EvalId::McpAtlas => mcp_atlas_command(args, root, run_dir)?, + }) +} diff --git a/crates/skippy-bench/src/evals/adapters/speed_bench.rs b/crates/skippy-bench/src/evals/adapters/speed_bench.rs new file mode 100644 index 000000000..53cb19fdf --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/speed_bench.rs @@ -0,0 +1,85 @@ +use super::super::{run::speed_bench_output_path, *}; + +const AUTH_LAUNCHER: &str = r#"from __future__ import annotations +import os +import runpy +import sys +from urllib.parse import urlparse + +import requests + +original_request = requests.sessions.Session.request + +def request_origin(url): + if "://" not in url: + url = "http://" + url + parsed = urlparse(url) + default_port = 443 if parsed.scheme.lower() == "https" else 80 + return (parsed.scheme.lower(), parsed.hostname, parsed.port or default_port) + +benchmark_origin = request_origin(os.environ["SKIPPY_BENCH_BASE_URL"]) + +def authorized_request(self, method, url, **kwargs): + if request_origin(url) == benchmark_origin: + headers = dict(kwargs.get("headers", {}) or {}) + headers.setdefault("Authorization", f"Bearer {os.environ['SKIPPY_BENCH_API_KEY']}") + kwargs["headers"] = headers + return original_request(self, method, url, **kwargs) + +requests.sessions.Session.request = authorized_request +script = sys.argv.pop(1) +sys.argv[0] = script +runpy.run_path(script, run_name="__main__") +"#; + +pub(in crate::evals) fn speed_bench_command( + definition: EvalDefinition, + args: &EvalRunArgs, + root: &Path, + run_dir: &Path, +) -> Result { + let harness = harness_dir(root, definition); + let requirements = harness.join("tools/server/bench/speed-bench/requirements.txt"); + let script = harness.join("tools/server/bench/speed-bench/speed_bench.py"); + let launcher = run_dir.join("raw/speed-bench-auth.py"); + fs::write(&launcher, AUTH_LAUNCHER).with_context(|| format!("write {}", launcher.display()))?; + let cache_root = root.join("speed-cache"); + let command = CommandSpec::new("uv") + .args([ + "run".to_string(), + "--with-requirements".to_string(), + requirements.display().to_string(), + "python".to_string(), + launcher.display().to_string(), + script.display().to_string(), + "--url".to_string(), + args.base_url.clone(), + "--model".to_string(), + args.model.clone(), + "--bench".to_string(), + "qualitative".to_string(), + "--category".to_string(), + "all".to_string(), + "--osl".to_string(), + "1024".to_string(), + "--concurrency".to_string(), + args.endpoint_concurrency.to_string(), + "--timeout".to_string(), + args.timeout_secs.to_string(), + "--output".to_string(), + speed_bench_output_path(run_dir).display().to_string(), + ]) + .env( + "XDG_CACHE_HOME", + cache_root.join("xdg").display().to_string(), + ) + .env("HF_HOME", cache_root.join("hf").display().to_string()) + .env( + "HF_DATASETS_CACHE", + cache_root.join("hf-datasets").display().to_string(), + ) + .env("UV_CACHE_DIR", cache_root.join("uv").display().to_string()) + .env("SKIPPY_BENCH_BASE_URL", args.base_url.clone()) + .secret_env("SKIPPY_BENCH_API_KEY", args.api_key.clone()); + Ok(command) +} diff --git a/crates/skippy-bench/src/evals/adapters/swe_bench_pro.rs b/crates/skippy-bench/src/evals/adapters/swe_bench_pro.rs new file mode 100644 index 000000000..8c369d3e1 --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/swe_bench_pro.rs @@ -0,0 +1,79 @@ +use super::super::{registry::definition, *}; + +pub(in crate::evals) fn swe_bench_pro_command( + args: &EvalRunArgs, + root: &Path, + run_dir: &Path, +) -> Result { + let harness = harness_dir(root, definition(EvalId::SweBenchPro)); + let script = run_dir.join("raw/swe-bench-pro-run.sh"); + write_swe_bench_pro_run_script(&script, args, root, &harness, run_dir)?; + Ok(CommandSpec::new("zsh") + .args([script.display().to_string()]) + .secret_env("SKIPPY_BENCH_API_KEY", args.api_key.clone())) +} + +fn write_swe_bench_pro_run_script( + path: &Path, + args: &EvalRunArgs, + cache_root: &Path, + harness: &Path, + run_dir: &Path, +) -> Result<()> { + let raw_dir = run_dir.join("raw/swe-bench-pro"); + let instances = raw_dir.join("instances.yaml"); + let expert_instances = raw_dir.join("instances-expert.yaml"); + let sweagent_output = raw_dir.join("sweagent-results"); + let patches = raw_dir.join("patches.json"); + let eval_dir = raw_dir.join("eval"); + let dockerhub_username = + env::var("SWE_BENCH_PRO_DOCKERHUB_USERNAME").unwrap_or_else(|_| "jefzda".to_string()); + let deployment_type = + env::var("SWE_BENCH_PRO_DEPLOYMENT_TYPE").unwrap_or_else(|_| "docker".to_string()); + let num_workers = + env_concurrency_matching_endpoint("SWE_BENCH_PRO_NUM_WORKERS", args.endpoint_concurrency)?; + let eval_workers = env::var("SWE_BENCH_PRO_EVAL_WORKERS").unwrap_or_else(|_| "100".to_string()); + let docker_platform = + env::var("SWE_BENCH_PRO_DOCKER_PLATFORM").unwrap_or_else(|_| "linux/amd64".to_string()); + let parse_function = env::var("SWE_BENCH_PRO_PARSE_FUNCTION").ok(); + let sweagent_python = env::var("SWE_BENCH_PRO_PYTHON").unwrap_or_else(|_| "3.12".to_string()); + let swerex_spec = env::var("SWE_BENCH_PRO_SWEREX_SPEC").ok(); + let swerex_pip_index_url = env::var("SWE_BENCH_PRO_SWEREX_PIP_INDEX_URL") + .unwrap_or_else(|_| "https://pypi.org/simple".to_string()); + let use_local_eval = env::var("SWE_BENCH_PRO_USE_LOCAL_DOCKER") + .map(|value| matches!(value.as_str(), "1" | "true" | "yes")) + .unwrap_or(true); + let local_eval_flag = if use_local_eval { + "--use_local_docker" + } else { + "" + }; + let model = litellm_model_name(&args.model); + let script = format!( + include_str!("templates/swe_bench_pro_run.sh"), + harness = shell_quote(&harness.display().to_string()), + raw_dir = shell_quote(&raw_dir.display().to_string()), + instances = shell_quote(&instances.display().to_string()), + expert_instances = shell_quote(&expert_instances.display().to_string()), + sweagent_output = shell_quote(&sweagent_output.display().to_string()), + patches = shell_quote(&patches.display().to_string()), + eval_dir = shell_quote(&eval_dir.display().to_string()), + model = shell_quote(&model), + base_url = shell_quote(&args.base_url), + dockerhub_username = shell_quote(&dockerhub_username), + deployment_type = shell_quote(&deployment_type), + num_workers = shell_quote(&num_workers), + eval_workers = shell_quote(&eval_workers), + docker_platform = shell_quote(&docker_platform), + parse_function = shell_quote(parse_function.as_deref().unwrap_or("")), + sweagent_python = shell_quote(&sweagent_python), + swerex_spec = shell_quote(swerex_spec.as_deref().unwrap_or("")), + swerex_pip_index_url = shell_quote(&swerex_pip_index_url), + local_eval_flag = shell_quote(local_eval_flag), + hf_home = shell_quote(&cache_root.join("hf").display().to_string()), + hf_datasets_cache = shell_quote(&cache_root.join("hf-datasets").display().to_string()), + uv_cache_dir = shell_quote(&cache_root.join("uv").display().to_string()), + xdg_cache_home = shell_quote(&cache_root.join("xdg").display().to_string()), + ); + fs::write(path, script).with_context(|| format!("write {}", path.display())) +} diff --git a/crates/skippy-bench/src/evals/adapters/templates/mcp_atlas_run.sh b/crates/skippy-bench/src/evals/adapters/templates/mcp_atlas_run.sh new file mode 100644 index 000000000..f962a273b --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/templates/mcp_atlas_run.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env zsh +set -euo pipefail + +HARNESS={harness} +COMPLETION_DIR={completion_dir} +RAW_DIR={raw_dir} +BASE_URL={base_url} +API_KEY="${{SKIPPY_BENCH_API_KEY:?SKIPPY_BENCH_API_KEY is required}}" +MODEL={model} +EVAL_MODEL=${{EVAL_LLM_MODEL:-$MODEL}} +OUTPUT={output} +OUTPUT_NAME=${{MCP_ATLAS_COMPLETION_OUTPUT_NAME:-{output_name}}} +MODEL_LABEL={model_label} +SCORE_DIR={score_dir} +COMPLETION_CONCURRENCY={completion_concurrency} +SCORE_CONCURRENCY=${{MCP_ATLAS_SCORE_CONCURRENCY:-5}} +HF_HOME_DIR={hf_home} +HF_DATASETS_CACHE_DIR={hf_datasets_cache} +UV_CACHE_DIR_LOCAL={uv_cache_dir} +XDG_CACHE_HOME_DIR={xdg_cache_home} +agent_started=0 +completion_started=0 +completion_pid="" + +mkdir -p \ + "$HF_HOME_DIR" \ + "$HF_DATASETS_CACHE_DIR" \ + "$UV_CACHE_DIR_LOCAL" \ + "$XDG_CACHE_HOME_DIR" +export HF_HOME="$HF_HOME_DIR" +export HF_DATASETS_CACHE="$HF_DATASETS_CACHE_DIR" +export UV_CACHE_DIR="$UV_CACHE_DIR_LOCAL" +export XDG_CACHE_HOME="$XDG_CACHE_HOME_DIR" + +port_ready() {{ + python3 - "$1" <<'PY' +import socket +import sys + +port = int(sys.argv[1]) +sock = socket.socket() +sock.settimeout(0.5) +try: + sock.connect(("127.0.0.1", port)) +except OSError: + sys.exit(1) +finally: + sock.close() +PY +}} + +wait_url() {{ + local name="$1" + local url="$2" + local log="$3" + for _ in {{1..90}}; do + if curl -fsS --max-time 5 "$url" >/dev/null 2>&1; then + return 0 + fi + tail -20 "$log" 2>/dev/null || true + sleep 2 + done + echo "timed out waiting for $name at $url" >&2 + return 1 +}} + +cleanup() {{ + if [[ "$completion_started" == "1" && -n "$completion_pid" ]]; then + kill "$completion_pid" >/dev/null 2>&1 || true + wait "$completion_pid" >/dev/null 2>&1 || true + fi + if [[ "$agent_started" == "1" ]]; then + docker rm -f skippy-bench-mcp-atlas-agent-env >/dev/null 2>&1 || true + fi +}} +trap cleanup EXIT + +mkdir -p "$RAW_DIR" "$SCORE_DIR" +cd "$HARNESS" +cp -n env.template .env >/dev/null 2>&1 || true +if ! docker image inspect agent-environment:latest >/dev/null 2>&1; then + docker tag ghcr.io/scaleapi/mcp-atlas:1.2.5 agent-environment:latest +fi + +if ! port_ready 1984; then + docker rm -f skippy-bench-mcp-atlas-agent-env >/dev/null 2>&1 || true + docker run --rm \ + --name skippy-bench-mcp-atlas-agent-env \ + -p 1984:1984 \ + --env-file .env \ + agent-environment:latest \ + > "$RAW_DIR/mcp-agent-env.log" 2>&1 & + agent_started=1 +fi +wait_url "MCP-Atlas agent environment" \ + "http://localhost:1984/enabled-servers" \ + "$RAW_DIR/mcp-agent-env.log" + +if ! port_ready 3000; then + ( + cd "$COMPLETION_DIR" + LLM_BASE_URL="$BASE_URL" \ + LLM_API_KEY="$API_KEY" \ + OPENAI_BASE_URL="$BASE_URL" \ + OPENAI_API_KEY="$API_KEY" \ + uv run python -m mcp_completion.main + ) > "$RAW_DIR/mcp-completion.log" 2>&1 & + completion_pid="$!" + completion_started=1 +fi +wait_url "MCP-Atlas completion service" \ + "http://localhost:3000/docs" \ + "$RAW_DIR/mcp-completion.log" + +cd "$COMPLETION_DIR" +LLM_BASE_URL="$BASE_URL" \ + LLM_API_KEY="$API_KEY" \ + OPENAI_BASE_URL="$BASE_URL" \ + OPENAI_API_KEY="$API_KEY" \ +uv run python mcp_completion_script.py \ + --model "$MODEL" \ + --input_huggingface ScaleAI/MCP-Atlas \ + --output "$OUTPUT_NAME" \ + --no-filter \ + --concurrency "$COMPLETION_CONCURRENCY" +cp "completion_results/$OUTPUT_NAME" "$OUTPUT" +EVAL_LLM_BASE_URL="${{EVAL_LLM_BASE_URL:-$BASE_URL}}" \ + EVAL_LLM_API_KEY="${{EVAL_LLM_API_KEY:-$API_KEY}}" \ +uv run python mcp_evals_scores.py \ + --input-file "completion_results/$OUTPUT_NAME" \ + --model-label "$MODEL_LABEL" \ + --evaluator-model "$EVAL_MODEL" \ + --output-dir "$SCORE_DIR" \ + --concurrency "$SCORE_CONCURRENCY" diff --git a/crates/skippy-bench/src/evals/adapters/templates/swe_bench_pro_run.sh b/crates/skippy-bench/src/evals/adapters/templates/swe_bench_pro_run.sh new file mode 100644 index 000000000..c15ba109d --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/templates/swe_bench_pro_run.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env zsh +set -euo pipefail + +HARNESS={harness} +RAW_DIR={raw_dir} +INSTANCES={instances} +EXPERT_INSTANCES={expert_instances} +SWEAGENT_OUTPUT={sweagent_output} +PATCHES={patches} +EVAL_DIR={eval_dir} +MODEL={model} +BASE_URL={base_url} +API_KEY="${{SKIPPY_BENCH_API_KEY:?SKIPPY_BENCH_API_KEY is required}}" +DOCKERHUB_USERNAME={dockerhub_username} +DEPLOYMENT_TYPE={deployment_type} +NUM_WORKERS={num_workers} +EVAL_WORKERS={eval_workers} +LOCAL_EVAL_FLAG={local_eval_flag} +DOCKER_PLATFORM={docker_platform} +PARSE_FUNCTION={parse_function} +SWEAGENT_PYTHON={sweagent_python} +SWEREX_SPEC={swerex_spec} +SWEREX_PIP_INDEX_URL={swerex_pip_index_url} +HF_HOME_DIR={hf_home} +HF_DATASETS_CACHE_DIR={hf_datasets_cache} +UV_CACHE_DIR_LOCAL={uv_cache_dir} +XDG_CACHE_HOME_DIR={xdg_cache_home} + +mkdir -p \ + "$RAW_DIR" \ + "$SWEAGENT_OUTPUT" \ + "$EVAL_DIR" \ + "$HF_HOME_DIR" \ + "$HF_DATASETS_CACHE_DIR" \ + "$UV_CACHE_DIR_LOCAL" \ + "$XDG_CACHE_HOME_DIR" +export HF_HOME="$HF_HOME_DIR" +export HF_DATASETS_CACHE="$HF_DATASETS_CACHE_DIR" +export UV_CACHE_DIR="$UV_CACHE_DIR_LOCAL" +export XDG_CACHE_HOME="$XDG_CACHE_HOME_DIR" +deployment_timeout_args=() +if [[ "$DEPLOYMENT_TYPE" == "modal" ]]; then + deployment_timeout_args=( + --instances.deployment.startup_timeout 1800 + --instances.deployment.runtime_timeout 3600 + ) +fi +deployment_platform_args=() +if [[ "$DEPLOYMENT_TYPE" == "docker" && -n "$DOCKER_PLATFORM" ]]; then + deployment_platform_args=( + --instances.deployment.platform "$DOCKER_PLATFORM" + ) +fi +deployment_type_args=( + --instances.deployment.type "$DEPLOYMENT_TYPE" +) +expert_instance_args=( + --instances.type file + --instances.path "$INSTANCES" +) +parse_function_args=() +if [[ -n "$PARSE_FUNCTION" ]]; then + parse_function_args=( + --agent.tools.parse_function.type "$PARSE_FUNCTION" + ) +fi +if [[ -z "$SWEREX_SPEC" && "$DEPLOYMENT_TYPE" == "docker" ]]; then + SWEREX_SPEC="swe-rex[modal]==1.4.0" +fi + +cd "$HARNESS" + +uv run \ + --with-requirements requirements.txt \ + --with pyyaml \ + python helper_code/generate_sweagent_instances.py \ + --dockerhub_username "$DOCKERHUB_USERNAME" \ + --output_path "$INSTANCES" + +( + cd SWE-agent + uv venv --clear --python "$SWEAGENT_PYTHON" .venv + uv pip install --python .venv/bin/python -e . + if [[ -n "$SWEREX_SPEC" ]]; then + uv pip install --python .venv/bin/python --upgrade "$SWEREX_SPEC" + fi + if [[ "$DEPLOYMENT_TYPE" == "docker" && -n "$SWEREX_PIP_INDEX_URL" ]]; then + .venv/bin/python - "$SWEREX_PIP_INDEX_URL" <<'PY' +import sys +from pathlib import Path + +import swerex.deployment.docker as docker + +path = Path(docker.__file__) +text = path.read_text() +old = 'f"RUN /root/python3.11/bin/pip3 install --no-cache-dir {{PACKAGE_NAME}}\\n\\n"' +new = ( + f'f"RUN /root/python3.11/bin/pip3 install --index-url {{sys.argv[1]}} ' + '--no-cache-dir {{PACKAGE_NAME}}\\n\\n"' +) +if old in text: + path.write_text(text.replace(old, new)) +elif new not in text: + raise RuntimeError(f"could not patch SWE-ReX Docker pip index in {{path}}") +PY + fi + if [[ "$DEPLOYMENT_TYPE" == "modal" ]]; then + .venv/bin/python swerex_patches/patch.py --yes + fi +) + +if [[ "$DEPLOYMENT_TYPE" == "docker" ]]; then + ( + cd SWE-agent + .venv/bin/python - "$INSTANCES" "$EXPERT_INSTANCES" "$DOCKER_PLATFORM" <<'PY' +import sys + +import yaml +from sweagent.agent.problem_statement import TextProblemStatement +from sweagent.environment.repo import PreExistingRepoConfig +from sweagent.environment.swe_env import EnvironmentConfig +from sweagent.run.batch_instances import BatchInstance +from swerex.deployment.config import DockerDeploymentConfig + +source, target, platform = sys.argv[1:4] +with open(source) as handle: + simple_instances = yaml.safe_load(handle) + +docker_args = [ + "--entrypoint", + "", +] +instances = [] +for item in simple_instances: + deployment = DockerDeploymentConfig( + image=item["image_name"], + docker_args=docker_args, + platform=platform or None, + python_standalone_dir="/root", + startup_timeout=1800, + ) + instance = BatchInstance( + env=EnvironmentConfig( + deployment=deployment, + repo=PreExistingRepoConfig( + repo_name=item.get("repo_name") or "app", + base_commit=item.get("base_commit") or "HEAD", + ), + ), + problem_statement=TextProblemStatement( + text=item["problem_statement"], + id=item["instance_id"], + extra_fields=item.get("extra_fields") or {{}}, + ), + ) + instances.append(instance.model_dump(mode="json", exclude_none=True)) + +with open(target, "w") as handle: + yaml.safe_dump(instances, handle, sort_keys=False) +PY + ) + expert_instance_args=( + --instances.type expert_file + --instances.path "$EXPERT_INSTANCES" + ) + deployment_type_args=() + deployment_platform_args=() +fi + +( + cd SWE-agent + OPENAI_BASE_URL="$BASE_URL" \ + OPENAI_API_KEY="$API_KEY" \ + .venv/bin/sweagent run-batch \ + --config config/tool_use.yaml \ + --output_dir "$SWEAGENT_OUTPUT" \ + --num_workers "$NUM_WORKERS" \ + --random_delay_multiplier 1 \ + "${{expert_instance_args[@]}}" \ + --instances.shuffle=False \ + "${{deployment_type_args[@]}}" \ + "${{deployment_timeout_args[@]}}" \ + "${{deployment_platform_args[@]}}" \ + "${{parse_function_args[@]}}" \ + --agent.model.name "$MODEL" \ + --agent.model.api_base "$BASE_URL" \ + --agent.model.max_input_tokens 0 \ + --agent.model.per_instance_cost_limit 0 \ + --agent.model.total_cost_limit 0 +) + +uv run \ + --with-requirements requirements.txt \ + python helper_code/gather_patches.py \ + --directory "$SWEAGENT_OUTPUT" \ + --prefix skippybench \ + --output "$PATCHES" + +uv run \ + --with-requirements requirements.txt \ + --with docker \ + --with modal \ + python swe_bench_pro_eval.py \ + --raw_sample_path helper_code/sweap_eval_full_v2.jsonl \ + --patch_path "$PATCHES" \ + --output_dir "$EVAL_DIR" \ + --scripts_dir run_scripts \ + --num_workers "$EVAL_WORKERS" \ + --dockerhub_username "$DOCKERHUB_USERNAME" \ + $LOCAL_EVAL_FLAG diff --git a/crates/skippy-bench/src/evals/adapters/terminal_bench.rs b/crates/skippy-bench/src/evals/adapters/terminal_bench.rs new file mode 100644 index 000000000..4b88d209d --- /dev/null +++ b/crates/skippy-bench/src/evals/adapters/terminal_bench.rs @@ -0,0 +1,27 @@ +use super::super::{run::terminal_bench_output_path, *}; + +pub(in crate::evals) fn terminal_bench_command(args: &EvalRunArgs, run_dir: &Path) -> CommandSpec { + let model = litellm_model_name(&args.model); + CommandSpec::new("tb") + .args([ + "run".to_string(), + "--dataset".to_string(), + "terminal-bench-core==0.1.1".to_string(), + "--agent".to_string(), + "terminus".to_string(), + "--model".to_string(), + model, + "--n-concurrent".to_string(), + args.endpoint_concurrency.to_string(), + "--output-path".to_string(), + terminal_bench_output_path(run_dir).display().to_string(), + "--global-agent-timeout-sec".to_string(), + args.timeout_secs.to_string(), + "--global-test-timeout-sec".to_string(), + "60".to_string(), + "--no-upload-results".to_string(), + "--no-livestream".to_string(), + ]) + .env("OPENAI_BASE_URL", args.base_url.clone()) + .secret_env("OPENAI_API_KEY", args.api_key.clone()) +} diff --git a/crates/skippy-bench/src/evals/doctor.rs b/crates/skippy-bench/src/evals/doctor.rs new file mode 100644 index 000000000..5673d62d0 --- /dev/null +++ b/crates/skippy-bench/src/evals/doctor.rs @@ -0,0 +1,230 @@ +use super::{registry::selected_evals, *}; + +pub(super) fn doctor_evals(args: EvalDoctorArgs) -> Result<()> { + let root = cache_root(args.cache_root)?; + let views = selected_evals(&args.evals, args.pack) + .into_iter() + .map(|definition| doctor_view(definition, &root)) + .collect::>(); + if args.json { + println!("{}", serde_json::to_string_pretty(&views)?); + return Ok(()); + } + + println!("SkippyBench eval doctor"); + println!("cache: {}", root.display()); + for view in views { + println!(); + println!("{}:", view.eval_id); + println!(" installed: {}", view.installed); + println!(" harness: {}", view.harness_dir); + for check in view.checks { + let status = if check.ok { "ok" } else { "missing" }; + println!(" {status:<7} {} - {}", check.name, check.detail); + } + } + Ok(()) +} + +pub(super) fn preflight_eval_run(definition: EvalDefinition) -> Result<()> { + let failed = definition + .required_tools + .iter() + .map(|tool| tool_check(tool)) + .filter(|check| !check.ok) + .map(|check| format!("{} - {}", check.name, check.detail)) + .collect::>(); + if failed.is_empty() { + return Ok(()); + } + bail!( + "{} prerequisites failed; run `skippy-bench eval doctor {}` for details:\n {}", + definition.id.as_str(), + definition.id.as_str(), + failed.join("\n ") + ) +} + +fn doctor_view(definition: EvalDefinition, root: &Path) -> DoctorView { + let harness = harness_dir(root, definition); + let mut checks = definition + .required_tools + .iter() + .map(|tool| tool_check(tool)) + .collect::>(); + if definition.id == EvalId::McpAtlas { + checks.push(port_check("mcp-atlas agent environment", 1984)); + checks.push(port_check("mcp-atlas completion service", 3000)); + } + DoctorView { + eval_id: definition.id.as_str(), + installed: harness.exists(), + harness_dir: harness.display().to_string(), + checks, + } +} + +fn tool_check(tool: &str) -> DoctorCheck { + if tool == "docker" { + return docker_check(); + } + let ok = command_exists(tool); + DoctorCheck { + name: tool.to_string(), + ok, + detail: if ok { + "found on PATH".to_string() + } else { + "not found on PATH".to_string() + }, + } +} + +fn docker_check() -> DoctorCheck { + let installed = command_exists("docker"); + if !installed { + return DoctorCheck { + name: "docker".to_string(), + ok: false, + detail: "docker CLI not found on PATH".to_string(), + }; + } + + let Some(info) = run_probe_command( + CommandSpec::new("docker").args(["info"]), + "docker-info", + Duration::from_secs(10), + ) else { + return DoctorCheck { + name: "docker".to_string(), + ok: false, + detail: "docker CLI found, but daemon did not answer within 10s".to_string(), + }; + }; + if !info.outcome.success { + return DoctorCheck { + name: "docker".to_string(), + ok: false, + detail: format!( + "docker CLI found, but daemon is not reachable{}", + probe_detail_suffix(&info) + ), + }; + } + + let container_name = format!("skippybench-doctor-{}", unix_millis().unwrap_or_default()); + let start = run_probe_command( + CommandSpec::new("docker").args([ + "run", + "--rm", + "--name", + &container_name, + "--pull=missing", + "hello-world", + ]), + "docker-run", + Duration::from_secs(60), + ); + cleanup_docker_container(&container_name); + let Some(start) = start else { + return DoctorCheck { + name: "docker".to_string(), + ok: false, + detail: + "docker daemon is reachable, but a hello-world container did not finish within 60s" + .to_string(), + }; + }; + let ok = start.outcome.success; + DoctorCheck { + name: "docker".to_string(), + ok, + detail: if ok { + "docker daemon is reachable and can start containers".to_string() + } else { + format!( + "docker daemon is reachable, but hello-world container start failed{}", + probe_detail_suffix(&start) + ) + }, + } +} + +struct ProbeCommandResult { + outcome: CommandOutcome, + stdout: String, + stderr: String, +} + +fn run_probe_command( + spec: CommandSpec, + label: &str, + timeout: Duration, +) -> Option { + let run_dir = temp_probe_dir(label)?; + let stdout_path = run_dir.join("stdout.log"); + let stderr_path = run_dir.join("stderr.log"); + let outcome = + run_command_with_timeout(&spec, Some(timeout), &stdout_path, &stderr_path).ok()?; + let stdout = fs::read_to_string(&stdout_path).unwrap_or_default(); + let stderr = fs::read_to_string(&stderr_path).unwrap_or_default(); + let _ = fs::remove_dir_all(run_dir); + Some(ProbeCommandResult { + outcome, + stdout, + stderr, + }) +} + +fn temp_probe_dir(label: &str) -> Option { + let millis = unix_millis().ok()?; + let dir = env::temp_dir().join(format!( + "skippy-bench-{label}-{}-{millis}", + std::process::id() + )); + fs::create_dir_all(&dir).ok()?; + Some(dir) +} + +fn cleanup_docker_container(name: &str) { + let _ = Command::new("docker").args(["rm", "-f", name]).status(); +} + +fn probe_detail_suffix(result: &ProbeCommandResult) -> String { + if result.outcome.timed_out { + return " (timed out)".to_string(); + } + let detail = + first_nonempty_line(&result.stderr).or_else(|| first_nonempty_line(&result.stdout)); + detail.map(|line| format!(": {line}")).unwrap_or_default() +} + +fn first_nonempty_line(text: &str) -> Option { + text.lines() + .map(str::trim) + .find(|line| !line.is_empty()) + .map(str::to_string) +} + +fn port_check(name: &str, port: u16) -> DoctorCheck { + let ok = localhost_port_ready(port); + DoctorCheck { + name: name.to_string(), + ok, + detail: if ok { + format!("localhost:{port} is reachable") + } else { + format!("expected on localhost:{port} when running this eval") + }, + } +} + +fn localhost_port_ready(port: u16) -> bool { + let timeout = Duration::from_millis(500); + ("127.0.0.1", port) + .to_socket_addrs() + .ok() + .into_iter() + .flatten() + .any(|addr| TcpStream::connect_timeout(&addr, timeout).is_ok()) +} diff --git a/crates/skippy-bench/src/evals/registry.rs b/crates/skippy-bench/src/evals/registry.rs new file mode 100644 index 000000000..aae1fb199 --- /dev/null +++ b/crates/skippy-bench/src/evals/registry.rs @@ -0,0 +1,142 @@ +use super::*; + +pub(super) fn list_evals(args: EvalListArgs) -> Result<()> { + let root = cache_root(args.cache_root.clone())?; + let views = selected_evals(&[], EvalPack::Core) + .into_iter() + .map(|definition| eval_view(definition, &root)) + .collect::>(); + if args.json { + println!("{}", serde_json::to_string_pretty(&views)?); + return Ok(()); + } + + println!("SkippyBench external evals"); + println!("cache: {}", root.display()); + for view in views { + let status = if view.installed { + "installed" + } else { + "not installed" + }; + println!( + " {:<16} {:<13} {}", + view.id, + format!("[{status}]"), + view.description + ); + } + Ok(()) +} + +pub(super) fn info_eval(args: EvalInfoArgs) -> Result<()> { + let root = cache_root(args.cache_root.clone())?; + let definition = definition(args.eval); + let view = eval_view(definition, &root); + if args.json { + println!("{}", serde_json::to_string_pretty(&view)?); + return Ok(()); + } + + println!("{} ({})", view.name, view.id); + println!("description: {}", view.description); + println!("repo: {} @ {}", view.repo_url, view.repo_ref); + println!("pack: {}", view.pack); + println!("disk: {}", view.disk_estimate); + println!("installed: {}", view.installed); + println!("harness: {}", view.harness_dir); + println!("requires: {}", view.required_tools.join(", ")); + print_notes("sync", view.sync_notes); + print_notes("run", view.run_notes); + Ok(()) +} + +pub(super) fn definition(id: EvalId) -> EvalDefinition { + match id { + EvalId::SpeedBench => EvalDefinition { + id, + name: "llama.cpp SPEED-Bench", + repo_url: "https://github.com/ggml-org/llama.cpp.git", + repo_ref: "master", + cache_name: "llama.cpp", + description: "OpenAI-compatible serving latency and throughput benchmark.", + disk_estimate: "1-3GB including clone and Python dataset cache", + required_tools: &["git", "uv", "python3"], + sync_notes: &["Clones llama.cpp; SPEED-Bench data is fetched by the Python runner."], + run_notes: &[ + "Runs the upstream SPEED-Bench client with category=all and no sample limit.", + ], + }, + EvalId::TerminalBench => EvalDefinition { + id, + name: "Terminal-Bench", + repo_url: "https://github.com/harbor-framework/terminal-bench.git", + repo_ref: "main", + cache_name: "terminal-bench", + description: "Agent benchmark for real terminal tasks in Docker sandboxes.", + disk_estimate: "5-20GB depending on Docker images", + required_tools: &["git", "uv", "python3.12", "tb", "docker"], + sync_notes: &["Clones task repo and installs the terminal-bench uv tool."], + run_notes: &["Runs the Terminal-Bench CLI against the full selected dataset."], + }, + EvalId::SweBenchPro => EvalDefinition { + id, + name: "SWE-Bench Pro", + repo_url: "https://github.com/scaleapi/SWE-bench_Pro-os.git", + repo_ref: "main", + cache_name: "swe-bench-pro", + description: "Long-horizon software-engineering patch benchmark.", + disk_estimate: "10GB+ before task Docker images", + required_tools: &["git", "uv", "python3", "docker"], + sync_notes: &["Clones the official repo and initializes submodules."], + run_notes: &[ + "Generates SWE-agent instances from the full SWE-Bench Pro test split.", + "Runs the synced SWE-agent scaffold, gathers .pred patches, then invokes swe_bench_pro_eval.py.", + ], + }, + EvalId::McpAtlas => EvalDefinition { + id, + name: "MCP-Atlas", + repo_url: "https://github.com/scaleapi/mcp-atlas.git", + repo_ref: "main", + cache_name: "mcp-atlas", + description: "Tool-use benchmark over real MCP servers and tasks.", + disk_estimate: "10GB+ including Docker image", + required_tools: &["git", "uv", "python3", "docker", "make", "curl"], + sync_notes: &["Clones repo and pulls the prebuilt MCP-Atlas Docker image."], + run_notes: &[ + "Starts the MCP environment and completion service when they are not already running.", + "Runs the MCP-Atlas completion and scoring scripts with --no-filter and without --num-tasks or tool_choice overrides.", + ], + }, + } +} + +pub(super) fn selected_evals(requested: &[EvalId], pack: EvalPack) -> Vec { + let ids = if requested.is_empty() { + match pack { + EvalPack::Core => CORE_EVALS.to_vec(), + } + } else { + requested.to_vec() + }; + ids.into_iter().map(definition).collect() +} + +fn eval_view(definition: EvalDefinition, root: &Path) -> EvalView { + let harness_dir = harness_dir(root, definition); + EvalView { + id: definition.id.as_str(), + name: definition.name, + pack: "core", + repo_url: definition.repo_url, + repo_ref: definition.repo_ref, + installed: harness_dir.exists(), + harness_dir: harness_dir.display().to_string(), + description: definition.description, + disk_estimate: definition.disk_estimate, + required_tools: definition.required_tools, + sync_notes: definition.sync_notes, + run_notes: definition.run_notes, + } +} diff --git a/crates/skippy-bench/src/evals/run.rs b/crates/skippy-bench/src/evals/run.rs new file mode 100644 index 000000000..6f5ec0c05 --- /dev/null +++ b/crates/skippy-bench/src/evals/run.rs @@ -0,0 +1,454 @@ +use super::{adapters::run_command, doctor::preflight_eval_run, registry::definition, *}; + +pub(super) fn run_eval(args: EvalRunArgs) -> Result<()> { + if args.endpoint_concurrency == 0 { + bail!("--endpoint-concurrency must be greater than zero"); + } + let root = absolute_path(cache_root(args.cache_root.clone())?)?; + let definition = definition(args.eval); + let run_dir = absolute_path(output_dir(args.output_dir.clone(), args.eval)?)?; + let run_id = args + .run_id + .clone() + .unwrap_or_else(|| eval_run_id(args.eval)); + let metrics_run_id = args + .metrics_run_id + .clone() + .unwrap_or_else(|| run_id.clone()); + let metrics_http = args.metrics_http.trim_end_matches('/').to_string(); + + if !args.dry_run { + let harness = harness_dir(&root, definition); + if !harness.exists() { + bail!( + "{} is not installed at {}; run `skippy-bench eval sync {}` first", + definition.id.as_str(), + harness.display(), + definition.id.as_str() + ); + } + preflight_eval_run(definition)?; + } + + fs::create_dir_all(run_dir.join("raw")) + .with_context(|| format!("create eval run dir {}", run_dir.display()))?; + + let harness_commit = resolved_harness_commit(&root, definition)?; + let command = run_command(definition, &args, &root, &run_dir)?; + let display = command.display(); + println!("{display}"); + + let mut report = RunReport { + run_id: run_id.clone(), + eval_id: definition.id.as_str(), + model: args.model.clone(), + base_url: args.base_url.clone(), + endpoint_concurrency: args.endpoint_concurrency, + run_dir: run_dir.display().to_string(), + harness_commit, + dry_run: args.dry_run, + command: display, + exit_status: None, + success: args.dry_run, + timed_out: false, + timeout_secs: args.timeout_secs, + harness_timeout_secs: args.harness_timeout_secs, + stdout_path: None, + stderr_path: None, + metrics: EvalMetrics::default(), + telemetry: telemetry_report::pending(&metrics_http, &metrics_run_id), + artifacts: run_artifacts(definition, &run_dir), + }; + + if !args.dry_run { + create_metrics_run(&args, &run_id, &metrics_run_id)?; + let started = Instant::now(); + let stdout_path = run_dir.join("raw").join("stdout.log"); + let stderr_path = run_dir.join("raw").join("stderr.log"); + let outcome = run_command_with_timeout( + &command, + args.harness_timeout_secs.map(Duration::from_secs), + &stdout_path, + &stderr_path, + ) + .with_context(|| format!("run {}", definition.id.as_str()))?; + let duration_ms = started.elapsed().as_secs_f64() * 1000.0; + report.exit_status = outcome.exit_status; + report.success = outcome.success; + report.timed_out = outcome.timed_out; + report.stdout_path = Some(stdout_path.display().to_string()); + report.stderr_path = Some(stderr_path.display().to_string()); + report.metrics = collect_metrics(definition, &run_dir, duration_ms); + report.telemetry = telemetry_or_unavailable( + &metrics_http, + &metrics_run_id, + collect_telemetry(&metrics_http, &metrics_run_id, &run_dir), + ); + } + + let report_path = run_dir.join("run.json"); + fs::write(&report_path, serde_json::to_vec_pretty(&report)?) + .with_context(|| format!("write {}", report_path.display()))?; + if !report.success { + bail!( + "{} failed; see {}", + definition.id.as_str(), + report_path.display() + ); + } + Ok(()) +} + +pub(super) fn resolved_harness_commit( + root: &Path, + definition: EvalDefinition, +) -> Result> { + let harness = harness_dir(root, definition); + if !harness.exists() { + return Ok(None); + } + let output = Command::new("git") + .args(["-C", &harness.display().to_string(), "rev-parse", "HEAD"]) + .output() + .with_context(|| format!("read harness revision from {}", harness.display()))?; + if !output.status.success() { + bail!( + "git rev-parse failed for {}: {}", + harness.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + let commit = String::from_utf8(output.stdout) + .context("harness revision was not UTF-8")? + .trim() + .to_string(); + if commit.is_empty() { + bail!("git returned an empty revision for {}", harness.display()); + } + Ok(Some(commit)) +} + +fn run_artifacts(definition: EvalDefinition, run_dir: &Path) -> Vec { + let mut artifacts = vec![ + RunArtifact { + kind: "stdout", + path: run_dir.join("raw/stdout.log").display().to_string(), + }, + RunArtifact { + kind: "stderr", + path: run_dir.join("raw/stderr.log").display().to_string(), + }, + ]; + match definition.id { + EvalId::SpeedBench => artifacts.push(RunArtifact { + kind: "speed-bench-json", + path: speed_bench_output_path(run_dir).display().to_string(), + }), + EvalId::SweBenchPro => artifacts.extend([ + RunArtifact { + kind: "swe-bench-pro-sweagent-results", + path: swe_bench_pro_sweagent_output_path(run_dir) + .display() + .to_string(), + }, + RunArtifact { + kind: "swe-bench-pro-patches-json", + path: swe_bench_pro_patches_path(run_dir).display().to_string(), + }, + RunArtifact { + kind: "swe-bench-pro-eval-json", + path: swe_bench_pro_output_path(run_dir).display().to_string(), + }, + ]), + EvalId::McpAtlas => artifacts.extend([ + RunArtifact { + kind: "mcp-atlas-completion-results-csv", + path: mcp_atlas_output_path(run_dir).display().to_string(), + }, + RunArtifact { + kind: "mcp-atlas-score-dir", + path: mcp_atlas_score_dir(run_dir).display().to_string(), + }, + ]), + EvalId::TerminalBench => artifacts.push(RunArtifact { + kind: "terminal-bench-results", + path: terminal_bench_output_path(run_dir).display().to_string(), + }), + } + artifacts +} + +fn collect_metrics(definition: EvalDefinition, run_dir: &Path, duration_ms: f64) -> EvalMetrics { + let mut metrics = match definition.id { + EvalId::SpeedBench => speed_bench_metrics(run_dir).unwrap_or_default(), + EvalId::SweBenchPro => swe_bench_pro_metrics(run_dir).unwrap_or_default(), + EvalId::McpAtlas => mcp_atlas_metrics(run_dir).unwrap_or_default(), + EvalId::TerminalBench => terminal_bench_metrics(run_dir).unwrap_or_default(), + }; + metrics.duration_ms = Some(duration_ms); + fill_client_rates(&mut metrics, duration_ms); + metrics +} + +fn create_metrics_run(args: &EvalRunArgs, eval_run_id: &str, metrics_run_id: &str) -> Result<()> { + let config = json!({ + "eval_run_id": eval_run_id, + "mode": "skippy-bench-external-eval", + "eval_id": args.eval.as_str(), + "model": args.model, + "base_url": args.base_url, + }); + telemetry_report::create_run(&args.metrics_http, metrics_run_id, &config) +} + +fn collect_telemetry( + metrics_http: &str, + metrics_run_id: &str, + run_dir: &Path, +) -> Result { + telemetry_report::finalize_and_collect( + metrics_http, + metrics_run_id, + &metrics_report_path(run_dir), + ) +} + +pub(super) fn telemetry_or_unavailable( + metrics_http: &str, + metrics_run_id: &str, + result: Result, +) -> BenchTelemetry { + result + .unwrap_or_else(|error| telemetry_report::unavailable(metrics_http, metrics_run_id, &error)) +} + +pub(super) fn speed_bench_metrics(run_dir: &Path) -> Result { + let value = read_json(&speed_bench_output_path(run_dir))?; + let overall = value + .get("summary") + .and_then(Value::as_array) + .and_then(|rows| { + rows.iter() + .find(|row| string_field(row, "category") == Some("overall")) + }); + let results = value + .get("results") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + let mut metrics = EvalMetrics { + request_count: value.get("completed_samples").and_then(Value::as_u64), + failed_count: value.get("failed_samples").and_then(Value::as_u64), + ..EvalMetrics::default() + }; + metrics.prompt_tokens = sum_u64_field(results, "prompt_tokens"); + metrics.completion_tokens = sum_u64_field(results, "completion_tokens"); + metrics.total_tokens = sum_u64_field(results, "total_tokens"); + metrics.draft_tokens = sum_u64_field(results, "draft_n"); + metrics.draft_accepted_tokens = sum_u64_field(results, "draft_n_accepted"); + if let Some(overall) = overall { + metrics.prompt_tok_s = numeric_field(overall, "avg_prompt_t_s"); + metrics.completion_tok_s = numeric_field(overall, "avg_pred_t_s"); + metrics.avg_latency_ms = + numeric_field(overall, "avg_latency").map(|seconds| seconds * 1000.0); + metrics.draft_accept_rate = numeric_field(overall, "accept_rate"); + } + Ok(metrics) +} + +pub(super) fn swe_bench_pro_metrics(run_dir: &Path) -> Result { + let value = read_json(&swe_bench_pro_output_path(run_dir))?; + let rows = value + .get("rows") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + let mut metrics = EvalMetrics { + request_count: swe_bench_pro_request_count(&value, rows), + failed_count: swe_bench_pro_failed_count(&value), + pass_rate: swe_bench_pro_pass_rate(&value), + ..EvalMetrics::default() + }; + metrics.prompt_tokens = sum_nested_usage(rows, "prompt_tokens"); + metrics.completion_tokens = sum_nested_usage(rows, "completion_tokens"); + metrics.total_tokens = sum_nested_usage(rows, "total_tokens"); + Ok(metrics) +} + +fn swe_bench_pro_request_count(value: &Value, rows: &[Value]) -> Option { + value + .get("total_instances") + .or_else(|| value.get("total")) + .or_else(|| value.get("n_total")) + .and_then(Value::as_u64) + .or_else(|| (!rows.is_empty()).then_some(rows.len() as u64)) +} + +fn swe_bench_pro_failed_count(value: &Value) -> Option { + value + .get("failed_instances") + .or_else(|| value.get("unresolved_instances")) + .or_else(|| value.get("n_unresolved")) + .and_then(Value::as_u64) +} + +fn swe_bench_pro_pass_rate(value: &Value) -> Option { + if let Some(rate) = value + .get("pass_rate") + .or_else(|| value.get("resolved_rate")) + .or_else(|| value.get("accuracy")) + .and_then(Value::as_f64) + { + return Some(rate); + } + let resolved = value + .get("resolved_instances") + .or_else(|| value.get("n_resolved")) + .and_then(Value::as_u64)?; + let total = value + .get("total_instances") + .or_else(|| value.get("total")) + .or_else(|| value.get("n_total")) + .and_then(Value::as_u64)?; + (total > 0).then_some(resolved as f64 / total as f64) +} + +pub(super) fn terminal_bench_metrics(run_dir: &Path) -> Result { + let value = read_json(&terminal_bench_results_path(run_dir)?)?; + let results = value + .get("results") + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + let mut metrics = EvalMetrics { + request_count: Some(results.len() as u64), + failed_count: value.get("n_unresolved").and_then(Value::as_u64), + pass_rate: value.get("accuracy").and_then(Value::as_f64), + ..EvalMetrics::default() + }; + metrics.prompt_tokens = sum_u64_field(results, "total_input_tokens"); + metrics.completion_tokens = sum_u64_field(results, "total_output_tokens"); + metrics.total_tokens = match (metrics.prompt_tokens, metrics.completion_tokens) { + (Some(prompt), Some(completion)) => Some(prompt + completion), + _ => None, + }; + Ok(metrics) +} + +fn terminal_bench_results_path(run_dir: &Path) -> Result { + let root = terminal_bench_output_path(run_dir); + for entry in fs::read_dir(&root).with_context(|| format!("read {}", root.display()))? { + let entry = entry?; + let path = entry.path().join("results.json"); + if path.is_file() { + return Ok(path); + } + } + bail!("no Terminal-Bench results.json under {}", root.display()) +} + +fn mcp_atlas_metrics(run_dir: &Path) -> Result { + let mut reader = csv::Reader::from_path(mcp_atlas_output_path(run_dir))?; + let data_rows = reader.records().filter(|record| record.is_ok()).count(); + Ok(EvalMetrics { + request_count: Some(data_rows as u64), + ..EvalMetrics::default() + }) +} + +pub(super) fn fill_client_rates(metrics: &mut EvalMetrics, duration_ms: f64) { + if duration_ms <= 0.0 { + return; + } + let seconds = duration_ms / 1000.0; + if metrics.prompt_tok_s.is_none() { + metrics.prompt_tok_s = metrics.prompt_tokens.map(|tokens| tokens as f64 / seconds); + } + if metrics.completion_tok_s.is_none() { + metrics.completion_tok_s = metrics + .completion_tokens + .map(|tokens| tokens as f64 / seconds); + } + metrics.total_tok_s = metrics.total_tokens.map(|tokens| tokens as f64 / seconds); +} + +fn read_json(path: &Path) -> Result { + let bytes = fs::read(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display())) +} + +fn sum_u64_field(rows: &[Value], key: &str) -> Option { + let mut total = 0; + let mut found = false; + for row in rows { + if let Some(value) = row.get(key).and_then(Value::as_u64) { + total += value; + found = true; + } + } + found.then_some(total) +} + +fn sum_nested_usage(rows: &[Value], key: &str) -> Option { + let mut total = 0; + let mut found = false; + for row in rows { + if let Some(value) = row + .get("usage") + .and_then(|usage| usage.get(key)) + .and_then(Value::as_u64) + { + total += value; + found = true; + } + } + found.then_some(total) +} + +fn numeric_field(value: &Value, key: &str) -> Option { + value.get(key).and_then(Value::as_f64) +} + +fn string_field<'a>(value: &'a Value, key: &str) -> Option<&'a str> { + value.get(key).and_then(Value::as_str) +} + +pub(super) fn speed_bench_output_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/speed-bench.json") +} + +pub(super) fn swe_bench_pro_output_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/swe-bench-pro/eval/eval_results.json") +} + +pub(super) fn swe_bench_pro_sweagent_output_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/swe-bench-pro/sweagent-results") +} + +pub(super) fn swe_bench_pro_patches_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/swe-bench-pro/patches.json") +} + +pub(super) fn mcp_atlas_output_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/mcp-atlas-completion-results.csv") +} + +pub(super) fn mcp_atlas_score_dir(run_dir: &Path) -> PathBuf { + run_dir.join("raw/mcp-atlas-evaluation-results") +} + +pub(super) fn terminal_bench_output_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/terminal-bench") +} + +fn metrics_report_path(run_dir: &Path) -> PathBuf { + run_dir.join("raw/metrics-report.json") +} + +fn eval_run_id(eval: EvalId) -> String { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("skippy-eval-{}-{millis}", eval.as_str()) +} diff --git a/crates/skippy-bench/src/evals/sync.rs b/crates/skippy-bench/src/evals/sync.rs new file mode 100644 index 000000000..ce4d776e7 --- /dev/null +++ b/crates/skippy-bench/src/evals/sync.rs @@ -0,0 +1,90 @@ +use super::{registry::selected_evals, *}; + +pub(super) fn sync_evals(args: EvalSyncArgs) -> Result<()> { + let root = cache_root(args.cache_root)?; + fs::create_dir_all(harness_root(&root)).with_context(|| { + format!( + "create eval harness cache {}", + harness_root(&root).display() + ) + })?; + + for definition in selected_evals(&args.evals, args.pack) { + println!("sync {}", definition.id.as_str()); + sync_repo(definition, &root, args.dry_run)?; + for step in sync_steps(definition, &root) { + run_step(&step, args.dry_run)?; + } + } + Ok(()) +} + +fn sync_repo(definition: EvalDefinition, root: &Path, dry_run: bool) -> Result<()> { + let target = harness_dir(root, definition); + if target.exists() { + for step in existing_repo_sync_steps(&target, definition.repo_ref) { + run_step(&step, dry_run)?; + } + return Ok(()); + } + + run_step( + &CommandSpec::new("git").args([ + "clone", + "--recurse-submodules", + "--branch", + definition.repo_ref, + definition.repo_url, + &target.display().to_string(), + ]), + dry_run, + ) +} + +pub(super) fn existing_repo_sync_steps(target: &Path, repo_ref: &str) -> [CommandSpec; 2] { + let target = target.display().to_string(); + [ + CommandSpec::new("git").args(["-C", &target, "fetch", "--prune", "origin", repo_ref]), + CommandSpec::new("git").args(["-C", &target, "checkout", "--detach", "FETCH_HEAD"]), + ] +} + +fn sync_steps(definition: EvalDefinition, root: &Path) -> Vec { + let harness = harness_dir(root, definition); + match definition.id { + EvalId::SpeedBench => Vec::new(), + EvalId::TerminalBench => { + vec![CommandSpec::new("uv").args([ + "tool", + "install", + "--python", + "3.12", + "terminal-bench", + ])] + } + EvalId::SweBenchPro => vec![ + CommandSpec::new("git") + .args(["submodule", "update", "--init", "--recursive"]) + .cwd(harness), + ], + EvalId::McpAtlas => { + vec![CommandSpec::new("docker").args(["pull", "ghcr.io/scaleapi/mcp-atlas:1.2.5"])] + } + } +} + +fn run_step(step: &CommandSpec, dry_run: bool) -> Result<()> { + println!("{}", step.display()); + if dry_run { + return Ok(()); + } + + let status = step + .command() + .status() + .with_context(|| format!("start {}", step.program))?; + if !status.success() { + bail!("command failed with status {status}: {}", step.display()); + } + Ok(()) +}