diff --git a/packages/nemo_platform_ext/pyproject.toml b/packages/nemo_platform_ext/pyproject.toml index 2e23517fef2..dee7032ad94 100644 --- a/packages/nemo_platform_ext/pyproject.toml +++ b/packages/nemo_platform_ext/pyproject.toml @@ -88,10 +88,11 @@ package_root = "packages/nemo_platform_ext" # Globs are evaluated per top-level module (e.g. `skills/`, `cli/`, `quickstart/`). # `**/*.md` recursively includes SKILL.md plus any companion markdown files # skills ship under `resources/` (e.g. notes, sub-docs, prompts). Add further -# extension globs here as new companion file types are needed (`.json`, -# `.yaml`, `.txt`, ...); we avoid `**/*` because dev environments leave +# extension globs here as new companion file types are needed. Runtime JSON +# assets live under `references/`; skill `tests.json` manifests remain +# source-only. We avoid `**/*` because dev environments leave # `__pycache__/*.pyc` in source modules, which the vendor tool doesn't filter. -included_paths = ["**/*.py", "**/*.md"] +included_paths = ["**/*.py", "**/*.md", "**/*.yaml", "**/*.yml", "**/references/**/*.json"] replace_client_inits_from = "client/enhanced.py" vendor_tests = true tests_path = "tests" diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/SKILL.md new file mode 100644 index 00000000000..2396195aec9 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/SKILL.md @@ -0,0 +1,321 @@ +--- +name: nemo-agent-config +description: Author and validate Platform-owned NeMo Agents agent.yaml files using the nemo-agents-spec-v1 format. Use when the user wants to create, edit, validate, or adapt an agent.yaml file, choose a supported harness, add instructions, skills, MCP servers, tools, environment, or telemetry. +triggers: + - write agent.yaml + - create agent.yaml + - edit agent.yaml + - validate agent.yaml + - configure a harness + - configure agent harness + - nemo-agents-spec-v1 + - platform agent config + - adapt agent.yaml + - convert NAT workflow YAML + - migrate NAT workflow + - convert agent.yml to agent.yaml + - NeMo agent.yaml config +not-for: + - nemo-build-agent (use for full spec-to-deployed-agent build flows) + - nemo-explore (use to design what the agent should do before writing config) + - nemo-spec (use to write AGENT-SPEC.md before implementation) + - nemo-model-selection (use when the user only wants model recommendation) + - generic YAML editing unrelated to NeMo Platform agents +compatibility: nemo-platform >= 0.1.0; writes or edits agents/-spec/agent.yaml; validates through nemo agents create; supports nemo-agents-spec-v1 configs; safe under sandbox. +maturity: active +license: Apache-2.0 +user-invocable: true +allowed-tools: [Read, Write, Edit, Bash] +--- + +# NeMo Platform agent config + +Create or edit the Platform-owned `agent.yaml` for a NeMo Agent. This skill +owns the machine-readable config shape for `nemo-agents-spec-v1`; `nemo-build-agent` +owns the full build/deploy/eval workflow. + +Use product-facing Platform language. Do not ask users to write raw Fabric SDK +configuration. Fabric is an implementation dependency behind the Platform-owned +agent config. + +## Storage model + +The local config lives next to the human-readable agent spec: + +```txt +agents/-spec/ + AGENT-SPEC.md + agent.yaml +``` + +The platform stores the parsed `agent.yaml` contents in the `Agent.config` +payload with: + +```yaml +config_format: nemo-agents-spec-v1 +``` + +The canonical remote config location is derivable from workspace and agent name: +`/-spec#agent.yaml`. Do not invent a separate ref field. + +## What you do + +1. Confirm the agent name and config path. Default to + `agents/-spec/agent.yaml`. +2. Select one supported harness: + - `codex` + - `hermes` + - `deepagents` + - `claude` +3. Invoke `nemo-model-selection` to select and compatibility-test the model for + the chosen harness. Configure `models.default` only after that skill returns + a verified provider and exact model name. Add a harness-local `model` + override only when that harness intentionally uses a different verified + provider, model, credential env var, or base URL. +4. Start from `references/templates/agent.yaml` unless the user is editing an + existing file. Write only the verified model fields returned in Step 3. +5. Remove every unselected harness block and every remaining placeholder. Keep + an additional harness only when the user explicitly requests it and its + model configuration has been separately verified. +6. Add system instructions under `instructions.system.content`. +7. Add optional skills, MCP servers, blocked tools, environment directories, and + telemetry using only fields in the template. +8. Keep all local file paths relative to the directory containing `agent.yaml`. +9. Validate by running `nemo agents create` against the config. Do not treat a + successful create as validation of an unused or untested harness. + +For the standard subprocess or default-image deployment path, keep +`skills.paths` empty. Registration validates relative skill paths against the +authoring directory, but these deployment paths currently materialize only +`agent.yaml` and do not stage the referenced directories. + +When non-empty `skills.paths` are required, each path must be relative to +`agent.yaml`, remain inside its packaging context, and contain `SKILL.md`. +Require the explicit image-packaging path before deployment: + +```bash +IMAGE_TAG="${AGENT_NAME}:local" +.venv/bin/nemo agents package \ + --agent "agents/$AGENT_NAME-spec/agent.yaml" \ + --tag "$IMAGE_TAG" +``` + +Deploy that image with `--mode docker --image "$IMAGE_TAG"`; publish it first +and use the published tag for Kubernetes. Do not use a subprocess or +default-image deployment for an agent with relative `skills.paths`. + +## Migrating from legacy NAT workflow YAML + +If the user has an existing NAT workflow YAML and wants the new Platform-owned +`agent.yaml` format, treat the migration as best-effort authoring. Do not +overwrite the original NAT YAML unless the user explicitly asks. + +Map only fields with a clear Platform equivalent: + +| NAT workflow concept | Platform `agent.yaml` target | +|---|---| +| LLM/provider/model block | `models.default` or a harness-local `model` | +| System prompt or workflow prompt | `instructions.system.content` | +| Workflow/tool loop choice | `default_harness` plus `harnesses..kind` | +| Tool/function references | `skills.paths`, `mcp.servers`, `tools.blocked`, or harness settings when clearly supported | +| Tracing or telemetry settings | `telemetry` | + +For a NAT `codex_agent` workflow, apply this mapping instead of copying the +workflow fields into `harnesses.codex.settings`: + +| NAT Codex field | Platform `agent.yaml` target | +|---|---| +| `working_directory` | `environment.workspace` | +| `sandbox_mode` | `harnesses.codex.settings.sandbox` | +| `approval_policy: never` | `harnesses.codex.settings.approval_mode: deny_all` when the installed descriptor declares it; otherwise omit | +| `relay_atof_output_dir` | `environment.artifacts`, `telemetry.output_dir`, and `telemetry.atof` | +| `skip_git_repo_check` | Omit; removed CLI-only setting | +| `timeout_seconds` | Omit unless the installed Codex adapter settings schema declares it | +| `max_output_chars` | Omit; no current Codex adapter setting | +| `prefer_chatgpt_auth` | Omit; authentication is selected by the Codex adapter and model configuration | + +Treat the installed Fabric adapter descriptor's `settings_schema` as +authoritative. Only place keys declared under its `properties` in +`harnesses..settings`; do not preserve an unsupported NAT setting merely +because it existed in the source workflow. Surface omitted behavior in the +migration summary. For Codex, map `never` to `deny_all` only when +`settings_schema.properties.approval_mode.enum` includes `deny_all`. + +If behavior does not map cleanly, say so directly and choose one: + +- Keep the agent on the NAT compatibility path. +- Preserve the original NAT YAML and create a partial `agent.yaml` starter for + manual completion. +- Mark it as requiring a custom adapter or a manual harness-specific migration. + +Never claim a mechanical one-to-one conversion for arbitrary NAT workflows. + +## Config shape + +Use this structure. Keep unknown fields out of the YAML; the Platform validator +rejects unsupported fields instead of passing arbitrary execution config through. + +```yaml +config_format: nemo-agents-spec-v1 +name: +description: + +instructions: + system: + content: + +default_harness: codex + +harnesses: + codex: + kind: codex + settings: + sandbox: workspace-write + reasoning_effort: high + +models: + default: + provider: + model: + api_key_env: + +skills: + paths: [] + +mcp: + servers: {} + +tools: + blocked: [] + +environment: + workspace: ./workspace + artifacts: ./artifacts + +telemetry: + enabled: false + provider: relay + output_dir: ./artifacts/relay + project: +``` + +### Harness overrides + +Use a harness-local model only when that harness should override the default. + +```yaml +harnesses: + hermes: + kind: hermes + model: + provider: + model: + api_key_env: + base_url: + temperature: 0.0 + settings: + max_tokens: 512 + reasoning_config: + effort: none +``` + +If `base_url` is needed, put it directly in the model block, not under +`settings`. + +Use `nemo-model-selection` for every harness. It must verify the exact model +against that harness's model contract before returning it: Responses for +`codex`, the selected provider's chat path for `hermes` and `deepagents`, and +the native Anthropic provider contract for `claude`. Do not route Claude +through Platform IGW. Do not write or finalize `agent.yaml` if model selection +cannot establish a compatible model. For Platform-routed models, availability, +provider metadata, config validation, and Fabric planning do not replace a +valid inference request through the required wire API. + +## Validate and register + +Before registering, validate the YAML shape with the Platform create path. +Immediately before running `nemo agents create`, show the command to the user, +ask for explicit confirmation, and wait for approval. + +```bash +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" +``` + +If validation fails, fix the named field in `agent.yaml` and retry. Do not +silence validation errors by moving unknown fields into `settings`. + +## Deploy and invoke + +After create succeeds, show the `nemo agents deploy` command to the user, ask +for explicit confirmation, and wait for approval before running it. + +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$AGENT_NAME-deployment" +``` + +The deploy command waits for `running` by default. After it succeeds, invoke the +same explicitly named deployment without another confirmation: + +```bash +.venv/bin/nemo agents invoke \ + --agent-deployment "$AGENT_NAME-deployment" \ + --input "" +``` + +For local one-shot validation without registering an Agent entity, use this +only when the selected model already has a directly usable provider endpoint +and credentials. Platform IGW normalization is applied by the registered +deployment path, not this local path: + +```bash +.venv/bin/nemo agents invoke \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" \ + --input "" +``` + +For a local persistent server, bind to loopback by default. Use an externally +accessible host only when the user explicitly asks to expose the server: + +```bash +.venv/bin/nemo agents run \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" \ + --host 127.0.0.1 \ + --port 8080 +``` + +## If validation fails + +| Symptom | Cause | Recovery | +|---|---|---| +| `root must be a YAML mapping` | Empty file or list/scalar at the root | Replace with the template shape | +| `extra fields not permitted` | Unknown Platform config field | Remove it or map it into a supported field | +| `default_harness must reference one of harnesses` | `default_harness` does not match a key under `harnesses` | Rename one side so they match | +| `Unsupported harness kind` | Harness kind is not supported by the Platform translator | Pick `codex`, `hermes`, `deepagents`, or `claude` | +| Local file path missing in deployment | Referenced prompts, skills, or assets were not staged | Keep paths relative and package the complete agent bundle into the deployed image | +| Adapter import or binary missing | Selected harness dependency is not installed in the runtime | Install the selected adapter/runtime dependency or choose a harness already available | + +## Hard rules + +- Keep `config_format: nemo-agents-spec-v1`. +- Keep paths relative to the `agent.yaml` directory. +- Put system instructions under `instructions.system.content`. +- Do not use `prompts` for the default path; top-level prompts are not translated yet. +- Do not create profile files. Profiles are not the Platform authoring contract. +- Do not expose Fabric SDK object names as user-authored YAML fields. +- Do not emit arbitrary adapter settings unless the selected harness documents them. + +## Gotchas + +- **Default model vs harness model.** A harness-local `model` always wins over + `models.default`. +- **Registration validates and normalizes.** `nemo agents create` is the + user-facing validation command. +- **`agent.yaml` is the implementation config, not the design spec.** + `AGENT-SPEC.md` explains what the agent should do; `agent.yaml` tells the + Platform how to run it. +- **NAT workflow YAML is a compatibility path.** If the user explicitly asks + for legacy NAT, route to `nemo-build-agent` and use its NAT template. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/references/templates/agent.yaml b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/references/templates/agent.yaml new file mode 100644 index 00000000000..473c25a5408 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/references/templates/agent.yaml @@ -0,0 +1,69 @@ +config_format: nemo-agents-spec-v1 +name: replace-with-agent-name +description: Replace with a short agent description + +instructions: + system: + content: Replace with the agent's system instructions. + +default_harness: codex + +harnesses: + codex: + kind: codex + settings: + sandbox: workspace-write + reasoning_effort: high + hermes: + kind: hermes + model: + provider: replace-with-verified-provider + model: replace-with-verified-model-name + api_key_env: replace-or-remove-credential-env-var + base_url: replace-or-remove-provider-base-url + temperature: 0.0 + settings: + max_tokens: 512 + reasoning_config: + effort: none + deepagents: + kind: deepagents + settings: + deepagents: {} + claude: + kind: claude + model: + provider: anthropic + model: replace-with-verified-anthropic-model-name + api_key_env: ANTHROPIC_API_KEY + settings: + permission_mode: dontAsk +models: + default: + provider: replace-with-verified-provider + model: replace-with-verified-model-name + +skills: + paths: [] + +mcp: + servers: {} + # To configure a server, replace `servers: {}` with: + # servers: + # github: + # transport: streamable-http + # url: ${GITHUB_MCP_URL} + # exposure: harness_native + +tools: + blocked: [] + +environment: + workspace: ./workspace + artifacts: ./artifacts + +telemetry: + enabled: false + provider: relay + output_dir: ./artifacts/relay + project: replace-with-agent-name diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/tests.json new file mode 100644 index 00000000000..64f2ec74692 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-agent-config/tests.json @@ -0,0 +1,75 @@ +{ + "skill": "nemo-agent-config", + "tests": [ + { + "type": "explicit", + "prompt": "Use nemo-agent-config to write an agent.yaml for my support agent.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "explicit", + "prompt": "Run the nemo-agent-config skill and validate this nemo-agents-spec-v1 config.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "explicit", + "prompt": "Invoke nemo-agent-config for a Platform agent config using the codex harness.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "I need to create an agent.yaml with codex as the default harness and NVIDIA as the default model provider.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "Help me edit agent.yaml to configure the hermes harness block for a NeMo Platform agent.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "Can you adapt this agent config to nemo-agents-spec-v1 and make sure the paths are relative?", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "Convert this old NAT workflow YAML into the new NeMo Platform agent.yaml format.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "I have an old NAT agent.yml and want to migrate it to nemo-agents-spec-v1.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "Create agent.yaml for nemo-agents-spec-v1 with codex as the default harness.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "The agent design is done. I only need to add telemetry and MCP server config to agent.yaml.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "implicit", + "prompt": "I have an existing NeMo agent.yaml config file and want to switch default_harness from hermes to codex.", + "expected_skill": "nemo-agent-config" + }, + { + "type": "negative-control", + "prompt": "Write a generic Kubernetes YAML file for a deployment and service.", + "expected_skill_not": "nemo-agent-config" + }, + { + "type": "negative-control", + "prompt": "Build me a LangChain DeepAgents app from scratch in a new Python package.", + "expected_skill_not": "nemo-agent-config" + }, + { + "type": "negative-control", + "prompt": "Help me pick which LLM to use for a support triage agent.", + "expected_skill_not": "nemo-agent-config" + } + ] +} diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md index d00e2561435..e9711922b0f 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/SKILL.md @@ -1,23 +1,27 @@ --- name: nemo-build-agent -description: End-to-end agent build on NeMo Platform. Scaffolds a NAT workflow YAML from the agent spec, deploys it, generates eval data via Data Designer, runs evaluation, optionally adds guardrails, and signs off. Use over generic agent-building or planning skills for any NeMo Platform agent build task. +description: End-to-end NeMo Platform agent implementation from an approved agent spec. Registers and deploys the agent, generates evaluation data, runs evaluation, and signs off. Use for full spec-to-deployed-agent work, including builds from an existing legacy NAT workflow. triggers: + - nemo-build-agent - build the agent - create the agent - deploy the agent - scaffold the agent - make me an agent - build an agent on nemo - - generate the workflow yaml + - build from the agent spec + - ship the agent - nemo build + - deploy my existing NAT agent not-for: + - nemo-agent-config (use for focused agent.yaml authoring or migration) - nemo-explore (use to gather design before building) - nemo-spec (use to write the spec file before building) - - nemo-try-agent (use to query a deployed agent) + - nemo-try-agent (use to query an already deployed agent) - nemo-setup (use to install the platform first) - deploy-sandbox (use to deploy the built agent as a governed OpenShell sandbox) - - superpowers:brainstorming (use for unrelated design work) -compatibility: nemo-platform >= 0.1.0; running platform (run nemo-setup first — uses `nemo services run`, no Docker); requires agents plugin installed; writes files to agents/; runs nemo CLI commands; defers platform-health probing to `nemo-status`; LangGraph + NAT under the hood; macOS or Linux; safe under sandbox. + - generic agent framework development outside NeMo Platform +compatibility: nemo-platform >= 0.1.0; running platform; requires agents plugin; writes files under agents/; uses nemo-agents-spec-v1 by default and preserves NAT workflow YAML as a compatibility path; macOS or Linux; safe under sandbox. maturity: active license: Apache-2.0 user-invocable: true @@ -26,206 +30,385 @@ allowed-tools: [Bash, Read, Write, Edit] # NeMo Platform agent build -Concrete commands only. Conversational scaffolding lives in `nemo-explore` and `nemo-spec`. This skill is the implementation path between spec and deployed agent. +Build a deployable NeMo Platform agent from an approved `AGENT-SPEC.md`. Use +the Platform-owned `nemo-agents-spec-v1` `agent.yaml` path by default. Treat +NAT workflow YAML as a supported compatibility path, not the default output. -NeMo Platform optimizes LangGraph agents wrapped in NVIDIA NeMo Agent Toolkit (NAT). The YAML this skill writes is a NAT workflow. If the user has an agent in another framework (CrewAI, AutoGen, plain LangChain, Pydantic AI), stop and tell them they need a NAT wrapper before this skill produces value. +Use `nemo-agent-config` for the machine-readable config shape. Do not expose +Fabric SDK object names or raw runtime configuration to the user. + +## Select the config path + +Choose the config path before pre-flight. Set shared names for either path: + +```bash +AGENT_NAME= +DEPLOYMENT_NAME="${AGENT_NAME}-deployment" +``` + +If the user supplies an existing NAT workflow YAML, ask whether they want to +deploy it unchanged or migrate it best-effort to `nemo-agents-spec-v1` with +`nemo-agent-config`. For an unchanged NAT-only run, preserve the original file +and also set: + +```bash +NAT_WORKFLOW_PATH= +``` ## Pre-flight -1. Confirm the platform is up. Run `nemo-status`'s platform probe (canonical lsof + curl check) and stop if it reports `PLATFORM_DOWN` or `PLATFORM_WEDGED`; route to `nemo-setup` and return when it clears. Do not reimplement the probe here — `nemo-status` owns it so changes (new components, new ports) land in one place. +1. Run the platform probe owned by `nemo-status`. If it reports + `PLATFORM_DOWN` or `PLATFORM_WEDGED`, route to `nemo-setup` and stop. +2. Confirm the agents plugin is loaded: -2. Confirm a spec exists at `agents/$AGENT_NAME-spec/AGENT-SPEC.md`. If missing, call `nemo-explore` then `nemo-spec`, then return. -3. Confirm the agents plugin is loaded: `.venv/bin/nemo agents --help 2>&1 | grep -q "create"`. If the plugin is missing, report that explicitly; the user has not installed `plugins/nemo-agents` and the build cannot proceed. -4. Read the spec. Extract: name, categories, tools, model, constraints, success criteria. -5. Confirm the canonical spec fileset exists. By convention the spec lives at `/-spec#AGENT-SPEC.md` — there is no ref to thread through, just a one-shot presence check: + ```bash + .venv/bin/nemo agents --help 2>&1 | grep -q "create" + ``` + +3. Check for existing Agent entities and deployments. Ask whether to reuse or + replace them. Follow the lifecycle branches below before create or deploy. +4. For an unchanged NAT-only run, confirm `$NAT_WORKFLOW_PATH` exists and read + it before continuing. Do not require `AGENT-SPEC.md` or a spec fileset. +5. For the default Platform-owned path, confirm + `agents/$AGENT_NAME-spec/AGENT-SPEC.md` exists. If it does not, route through + `nemo-explore` and `nemo-spec` first. +6. Read the spec and extract the agent name, instructions, capabilities, + model requirements, tools, constraints, and success criteria. +7. Confirm the canonical spec fileset exists: ```bash - nemo files filesets get "${AGENT_NAME}-spec" --workspace "${WORKSPACE:-default}" >/dev/null 2>&1 \ + .venv/bin/nemo files filesets get "${AGENT_NAME}-spec" \ + --workspace "${WORKSPACE:-default}" >/dev/null 2>&1 \ && echo "spec_fileset_ok" \ - || { echo "spec_fileset_missing — run nemo-spec to upload before continuing"; exit 1; } + || { echo "spec_fileset_missing - run nemo-spec first"; exit 1; } ``` - If the fileset is missing, route back to `nemo-spec` and return when the upload succeeds. -6. Check for an existing deployment: `.venv/bin/nemo agents deployments list 2>/dev/null | grep -q "$AGENT_NAME"`. If the agent is already deployed, ask the user whether to skip (idempotent path) or redeploy. +Steps 5 through 7 apply only to the default Platform-owned path or an explicit +NAT migration. + +### Existing-resource lifecycle -## Step 1: Scaffold and deploy +- **Reuse:** Do not run `agents create` for an existing Agent. If a deployment + already exists, set `DEPLOYMENT_NAME` to its name, do not run `agents deploy`, + and continue to the smoke test. If only the Agent exists, skip create and run + only the deploy command in Step 1. +- **Replace:** Show each destructive command and require explicit confirmation + immediately before running it. Use `--yes` only after that confirmation. If + the resource does not exist, skip its command. -Write `agents/$AGENT_NAME.yml` from `references/templates/agent.yml`, substituting model, tools, system prompt, and the spec's constraints. The system prompt MUST contain `{tools}` and `{tool_names}` placeholders. +For a confirmed replacement, undeploy first: ```bash -AGENT_NAME= # set once; reused throughout this skill -.venv/bin/nemo agents delete "$AGENT_NAME" 2>/dev/null || true -.venv/bin/nemo agents create --name "$AGENT_NAME" \ - --agent-config "agents/$AGENT_NAME.yml" -.venv/bin/nemo agents deploy --agent "$AGENT_NAME" -.venv/bin/nemo agents deployments wait --agent "$AGENT_NAME" +.venv/bin/nemo agents undeploy "$DEPLOYMENT_NAME" --yes ``` -Show the YAML to the user. Stop. Ask: "Config and deployment look right? Adjust system prompt, model, or tools before continuing?" - -Verification: confirm the deployment reached ready state. +Wait until this command reports that the deployment is absent before +continuing: ```bash -.venv/bin/nemo agents deployments list | grep "$AGENT_NAME" | grep -qi "ready" && echo "DEPLOY_READY" || echo "DEPLOY_NOT_READY" +.venv/bin/nemo agents deployments get "$DEPLOYMENT_NAME" ``` -If `DEPLOY_NOT_READY`: jump to the recovery table at the bottom. +Then show the Agent deletion command and require explicit confirmation before +running it: -> **Governed sandbox deployment.** To deploy this agent as a policy-governed OpenShell sandbox instead of the default executor (Landlock filesystem isolation plus default-deny network egress, so its model traffic can only reach the platform), use the `deploy-sandbox` skill once the image is built. It swaps this step's deploy path for the `openshell-local` executor and an auto-generated SandboxPolicy. - -## Step 2: Try the agent +```bash +.venv/bin/nemo agents delete "$AGENT_NAME" --yes +``` -Invoke with one question from each category in the spec. +Verify this command reports that the Agent is absent before running the create +and deploy commands in Step 1: ```bash -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" +.venv/bin/nemo agents get "$AGENT_NAME" ``` -Display each verbatim response. +## Prepare the selected config -Stop. Ask if they want to proceed to evaluation or adjust the agent first. +### Default: Platform-owned `agent.yaml` -## Step 3: Identify and generate the synthetic data this agent needs +For a new build, invoke `nemo-agent-config` and create: -Data Designer (DD) is the platform's synthetic-data tool. It can produce any of: +```txt +agents/-spec/ + AGENT-SPEC.md + agent.yaml +``` -- **Knowledge base or RAG corpus.** Q&A pairs, doc snippets, or policy entries the agent retrieves from at runtime. -- **Evaluation dataset.** Input prompts plus ground-truth or judge-rubric outputs. Used by Step 4 evaluation. -- **Benchmark dataset.** A larger, diversity-weighted eval set for ongoing regression testing. -- **Persona-grounded inputs.** Adversarial or edge-case inputs simulating specific user types. -- **Training data.** When fine-tuning lands. -- **Other synthetic datasets** the user asks for. +Delegate authoring to `nemo-agent-config`. It selects the supported harness and +uses `nemo-model-selection` to verify the exact model against that harness's +model contract before writing the model block. Translate the approved spec into +system instructions, skills, MCP servers, tools, environment paths, and +telemetry. Keep every local path relative to the directory containing +`agent.yaml`. -**Do NOT hand-author any of these, even if your model is capable enough to write them inline.** Three reasons, all load-bearing: +Before registration, inspect `skills.paths`: -1. **Reproducibility.** DD configs regenerate identical datasets when seeded. Hand-authored sets are unreproducible — the moment the spec changes, you cannot regenerate matching eval data without re-doing the authoring by hand. -2. **Diversity.** DD samples across categorical axes the user (or skill) declares. Hand-authored sets cluster around whatever the author thought of, which under-tests the long tail. -3. **Capability transfer.** A less capable coding agent running this skill later cannot hand-author good eval questions. DD-generated data is independent of the coding agent's capability — the same DD config produces equivalent data whether driven by Sonnet or a 7B model. +- If it is empty, continue with the normal deployment lifecycle below. +- If it is non-empty, verify every relative directory is inside the agent + packaging context and contains `SKILL.md`. Package the complete bundle before + deployment: -### Procedure + ```bash + IMAGE_TAG="${AGENT_NAME}:local" + .venv/bin/nemo agents package \ + --agent "agents/$AGENT_NAME-spec/agent.yaml" \ + --tag "$IMAGE_TAG" + ``` -1. **Enumerate.** Read `agents/$AGENT_NAME-spec/AGENT-SPEC.md`. Surface to the user the full list of synthetic-data purposes this agent plausibly needs, based on the spec. Do not prescribe a count or shortlist; let the user pick freely from the catalog above (or add purposes you haven't anticipated). + Use the packaged-image deploy command below. Do not use subprocess deployment + or the default container image because those paths materialize only + `agent.yaml` and do not stage relative skill directories. -2. **Wait for picks.** Do not generate any DD config until the user has explicitly named which purposes they want. If the user says "you decide," default to: a knowledge base if the spec describes retrievable content, an eval dataset always, persona-grounded adversarial inputs if the spec lists safety constraints. Announce the defaults you chose. +### Compatibility: existing NAT workflow YAML -3. **Hand off per purpose.** For each chosen purpose, invoke the `data-designer` skill once. Pass it: the agent name, the purpose label (KB / eval / benchmark / persona / other), and the spec path. The DD skill is responsible for the config shape — this skill does not duplicate that logic. +If the user selected migration, preserve the original YAML. If a workflow, +tool, or custom Python component has no supported harness equivalent, keep the +NAT path or identify the need for a custom adapter. Never claim arbitrary NAT +workflows convert mechanically. -4. **Ground every config in the spec.** Each DD config MUST reference `agents/$AGENT_NAME-spec/AGENT-SPEC.md` for product context, categories, audience, and constraints. Do not redefine these inline. If the generated config inlines context, edit it to read from the spec instead — drift between agent definition and synthetic data is a reproducibility failure. +Use `references/templates/agent.yml` only when the user explicitly chooses the +legacy NAT path or needs a new NAT compatibility workflow. -5. **Run each config.** Use `.venv/bin/python agents/$AGENT_NAME..py` (or the CLI invocation once `nemo data-designer preview-local` lands in a release > 2.1.0). For larger jobs, submit via `nemo data-designer jobs create`. +## Step 1: Register and deploy -6. **Verify before Step 4.** Confirm at least one fileset in `nemo files filesets list` matching `$AGENT_NAME-eval-*` exists. Step 4 refuses to proceed without it. +For the default path: -Show 3 to 5 sample records per purpose, grouped by category. Stop. Ask: "Do these samples look realistic for each purpose? Adjust categories, prompts, or regenerate?" +For each operation retained by the selected lifecycle branch, follow the +`nemo-agent-config` confirmation requirement. Show the create command and ask +for explicit confirmation immediately before running it: -### Anti-patterns to refuse +```bash +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" +``` -- Writing eval questions inline because "they're simple" — refuse, route to DD. -- Generating a single combined dataset that conflates KB and eval — refuse, separate configs per purpose. -- Skipping DD entirely because the user said "just do it" — refuse, DD is required infrastructure, not an optional tool. -- Inlining product context in the DD config instead of referencing the spec — refuse, fix the config to read from the spec. +After create succeeds, show the deploy command and ask for explicit +confirmation immediately before running it: -## Step 3.5: Wire generated data into the agent +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$DEPLOYMENT_NAME" +``` + +If `skills.paths` is non-empty, show this command instead and ask for explicit +confirmation immediately before running it: -If Step 3 produced any synthetic data the agent is supposed to *use at runtime* (a knowledge base, a RAG corpus, a retrieval index), the agent must be wired to actually consume it. Generating the data and never connecting it is a silent product failure: the agent hallucinates against missing context while the real data sits unused next to it. +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$DEPLOYMENT_NAME" \ + --mode docker \ + --image "$IMAGE_TAG" +``` -NeMo Agent Toolkit (NAT) has first-class retrieval support: +For Kubernetes, publish the packaged image and replace `docker` and +`$IMAGE_TAG` with `k8s` and the published image tag. -- `nvidia-nat-rag` ships a `RAGRetriever` client that loads filesets or local parquet/JSONL files. -- `nvidia-nat-langchain` bridges any LangChain retriever (FAISS, Chroma, Milvus, Pinecone, OpenSearch, NeMo Retriever) into a NAT tool. -- Worked example: `examples/RAG/simple_rag/` in the NAT repo. +These commands assume the Agent and deployment are absent. If pre-flight found +existing resources, complete the selected lifecycle branch before running them. -### Retriever wiring procedure +`nemo agents deploy` waits for `running` by default. If the user passed +`--no-wait`, wait explicitly: -1. **Detect.** Scan `agents/$AGENT_NAME-spec/AGENT-SPEC.md` for tools whose names suggest retrieval: `*_search`, `*_lookup`, `query_*`, `find_*`, `rag_*`, or any tool the user described in `nemo-explore` as "the agent looks things up in X." Cross-reference against the filesets Step 3 produced. +```bash +.venv/bin/nemo agents deployments wait "$DEPLOYMENT_NAME" +``` -2. **Pair.** For each retrieval-style tool, identify which Step 3 fileset feeds it. If the spec lists `billing_kb_search` and Step 3 produced `billing-support-kb`, pair them. If a tool has no matching fileset, surface the gap to the user: "Your spec lists `billing_kb_search` but no KB fileset was generated. Generate one now (route to Step 3) or drop the tool from the agent?" +Show `agent.yaml` and the deployment result. Stop and ask whether the config, +model, harness, and instructions look right before continuing. -3. **Wire.** Update `agents/$AGENT_NAME.yml` to add a NAT retriever per pair, pointing at the fileset. The retriever appears as a tool in the agent's `tools:` list under the matching name. Use the simple_rag example as the template shape. +For an unchanged NAT workflow, registration defaults configs without +`config_format` to `nat-workflow-v1`: -4. **Redeploy.** Step 1 already deployed the agent without the retrievers wired (because the data didn't exist yet). Redeploy now so the agent's tool list includes the retrievers: +Apply the same immediate confirmation requirement. Show the create command and +wait for explicit confirmation before running it: ```bash -.venv/bin/nemo agents undeploy --agent $AGENT_NAME -.venv/bin/nemo agents create --name $AGENT_NAME \ - --agent-config agents/$AGENT_NAME.yml -.venv/bin/nemo agents deploy --agent $AGENT_NAME -.venv/bin/nemo agents deployments wait --agent $AGENT_NAME +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "$NAT_WORKFLOW_PATH" ``` -5. **Verify the wire.** Invoke the agent with a question that requires the KB and inspect the tool-call trace. The retriever tool MUST be called for any KB-grounded question. If the agent answers from system-prompt-policy text alone without calling the retriever, the tool wiring is broken — debug before declaring success. +After create succeeds, show the deploy command and wait for explicit +confirmation before running it: ```bash -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" --output-format json | python3 -c "import sys,json; d=json.loads(sys.stdin.read()); print('tools called:', d.get('tool_calls') or 'NONE — wiring broken')" +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$DEPLOYMENT_NAME" ``` -### Refuse-list +## Step 2: Try the deployed agent -- Skipping this step when the spec lists a retrieval-style tool. Generating data the agent can't reach is theater. -- Wiring the retriever but not redeploying. The fix doesn't land in a running agent until redeploy. -- Declaring success on the redeploy without confirming the tool was actually called for a KB question. A wired-but-unused retriever is indistinguishable from a missing one. +For the default path, invoke one question from each category in the spec. For +an unchanged NAT-only run without a spec, use representative questions from the +workflow and the user's stated requirements: + +```bash +.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "" +.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "" +.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "" +``` + +Display each response verbatim. Stop and ask whether to adjust the agent or +continue to evaluation. + +Before Step 3, branch explicitly: + +1. For an unchanged NAT-only run without `AGENT-SPEC.md`, stop after the smoke + test. Do not execute Steps 3–5 and do not require an evaluation fileset. +2. Continue into the spec-driven purpose selection and Data Designer flow only + when the user requests it and `agents/$AGENT_NAME-spec/AGENT-SPEC.md` exists. + If the user requests evaluation but the spec is absent, create and confirm + the spec first; do not continue to Step 3 yet. + +## Step 3: Generate synthetic data + +Use Data Designer for every synthetic dataset. Do not hand-author evaluation, +knowledge-base, benchmark, persona, or training data. + +1. Always select evaluation as a required data purpose. Read + `agents/$AGENT_NAME-spec/AGENT-SPEC.md` and list any additional plausible + purposes: knowledge/RAG corpus, benchmark, personas/adversarial inputs, + training, or another user-requested purpose. +2. Wait for the user to choose any additional purposes. Evaluation cannot be + omitted. If they delegate the decision, add a knowledge base when the spec + requires retrieval and adversarial personas when it contains safety + constraints. +3. Invoke `data-designer` once per selected purpose, passing the agent name, + purpose, and spec path. +4. Require every generated config to read product context from + `AGENT-SPEC.md`; do not duplicate that context inline. +5. Run each generated config. For evaluation, validate the generated records, + verify the resulting fileset exists, and record its exact dataset reference + as `EVAL_DATASET_REF`. +6. Show 3 to 5 sample records per purpose and ask for approval. + +A validated `$AGENT_NAME-eval-*` fileset and its exact `EVAL_DATASET_REF` must +exist before evaluation proceeds. + +## Step 3.5: Connect runtime data + +If the generated data must be available during invocation, connect it through +the selected harness's supported skills, MCP, or tool configuration. Update +`agents/$AGENT_NAME-spec/agent.yaml` through `nemo-agent-config`, then follow the +confirmed replacement branch before creating and deploying the Agent again. + +Do not invent a generic retriever field. If the selected harness cannot consume +the required data, surface that limitation and choose a supported integration, +the NAT compatibility path, or a custom adapter. + +For a legacy NAT workflow, NAT-specific retrievers may be wired into its +`functions` and `workflow` blocks using the matching NAT RAG integration. + +After the replacement deployment, invoke a question that requires the data and +verify the expected tool or retrieval path was actually used. ## Step 4: Evaluate +Select the actual Platform model reference as `EVAL_MODEL`. Create +`agents/$AGENT_NAME.eval-job.json` from `references/templates/eval-job.json` and +replace every placeholder. Its `model` must equal `EVAL_MODEL`, and its +`dataset` must equal the recorded `EVAL_DATASET_REF` from Step 3. + +Validate the rendered file before creating the benchmark job: + +```bash +.venv/bin/python -m json.tool "agents/$AGENT_NAME.eval-job.json" >/dev/null +if grep -Eq '<[^>]+>' "agents/$AGENT_NAME.eval-job.json"; then + echo "eval job still contains template placeholders" >&2 + exit 1 +fi +``` + +Also read the validated payload back and confirm its `model` and `dataset` +values match `EVAL_MODEL` and `EVAL_DATASET_REF`. Do not invoke +`benchmark-jobs create` if JSON validation, model validation, dataset +validation, or fileset validation fails. + +After all validation succeeds: + ```bash .venv/bin/nemo evaluation benchmarks list -.venv/bin/nemo evaluation benchmark-jobs create $AGENT_NAME-eval \ - --input-file agents/$AGENT_NAME.eval-job.json +.venv/bin/nemo evaluation benchmark-jobs create "$AGENT_NAME-eval" \ + --input-file "agents/$AGENT_NAME.eval-job.json" ``` -Template for the eval-job JSON in `references/templates/eval-job.json`. Poll job status: +Poll until the job reaches `completed` or `failed`, then download aggregate +scores. Show the score table and compare it with the success bar in +`AGENT-SPEC.md`. ```bash for i in $(seq 1 24); do - status=$(.venv/bin/nemo evaluation benchmark-jobs get-status $AGENT_NAME-eval 2>/dev/null) + status=$(.venv/bin/nemo evaluation benchmark-jobs get-status "$AGENT_NAME-eval" 2>/dev/null) echo "$status" echo "$status" | grep -qE "completed|failed" && break sleep 10 done -.venv/bin/nemo evaluation benchmark-jobs results aggregate-scores download $AGENT_NAME-eval -``` -Verification: confirm the job reached `completed`, not `failed`. Display the score table. Stop. Ask if scores meet the bar from the spec. +.venv/bin/nemo evaluation benchmark-jobs results aggregate-scores download \ + "$AGENT_NAME-eval" +``` ## Step 5: Guardrails (optional) -If the spec lists constraints, add a content-safety intercept to the YAML (see `references/templates/agent-with-guardrails.yml` if present, or write inline) and redeploy: +For `nemo-agents-spec-v1`, `AgentConfig` has no guardrail field and the current +skills do not define a supported composition between an Agent and an IGW +guardrailed VirtualModel. Do not add guardrail fields to `agent.yaml` or claim +that guardrails are attached. If the spec requires guardrails, report this as an +unmet requirement and stop before sign-off. `nemo-guardrails` may be used to +configure IGW VirtualModel middleware as a separate workflow, but do not treat +it as integrated with the Agent until its model routing has been explicitly +configured and validated. -```bash -.venv/bin/nemo agents undeploy --agent $AGENT_NAME -.venv/bin/nemo agents create --name $AGENT_NAME \ - --agent-config agents/$AGENT_NAME.yml -.venv/bin/nemo agents deploy --agent $AGENT_NAME -.venv/bin/nemo agents deployments wait --agent $AGENT_NAME -``` +For a legacy NAT workflow, keep the NAT compatibility behavior: add supported +guardrail `intercepts` to the NAT workflow YAML, then follow the confirmed +replacement branch before creating and deploying it again. Never add NAT +`intercepts` to a `nemo-agents-spec-v1` config. -Test with one adversarial and one legitimate prompt. Stop and report. +For the NAT path, test one adversarial prompt and one legitimate prompt. Report +both responses and do not continue to sign-off until the expected policy is +enforced without blocking the legitimate request. ## Step 6: Sign off -Run the success-criteria question from the spec through `nemo agents invoke` once more and print the verbatim output. That output is the formal sign-off. +Invoke the success-criteria prompt from the spec against +`$DEPLOYMENT_NAME`. Print the verbatim response as the formal sign-off. Do not +claim success until the deployment is `running`, evaluation has completed, and +the sign-off returns an actual model response. ## If verification fails | Symptom | Cause | Recovery | -| --- | --- | --- | -| `agents plugin unavailable` | `plugins/nemo-agents` not installed | Re-run the install loop from `nemo-setup` Step 3 for that package only | -| `DEPLOY_NOT_READY` after wait | Container startup error or YAML rejected | Run `.venv/bin/nemo agents deployments get $AGENT_NAME`; check status detail and logs | -| YAML rejected with `extra fields` | Top-level keys beyond `functions`, `llms`, `workflow`, `intercepts`, `middleware` | Strip extras from the YAML; only those five top-level keys are valid | -| Empty agent response | `{tools}` and `{tool_names}` missing from system prompt | Add both placeholders; redeploy | -| Eval job `failed` | Dataset path or model id wrong | Run `.venv/bin/nemo evaluation benchmark-jobs get $AGENT_NAME-eval` for the error string | -| Eval times out at 4 minutes | Long-running benchmark | Extend the poll loop budget; do not declare success on a timed-out job | - -If none of these apply, tail recent service logs (`.venv/bin/nemo services logs -n 100`) and surface the last error to the user. Do not claim the build succeeded until the success-criteria sign-off in Step 6 prints an actual model response. - -## Gotchas - -- **NAT workflow YAML required keys.** Top level must be `functions`, `llms`, `workflow`. Optional: `intercepts`, `middleware`. Extra top-level keys (`name`, `description`, `model`, `tools`, `system_prompt`) cause NAT to reject the config. -- **`{tools}` and `{tool_names}` are mandatory in the system prompt.** Without them the agent crashes on startup with no useful error. -- **Two model name formats coexist.** Entity-name with hyphens for NAT YAML and `nemo chat` and `nemo agents`. API-Catalog format with slashes for Data Designer. Mixing them causes silent failures or 404s. -- **`agents delete` is positional.** `nemo agents delete $AGENT_NAME`, not `--name`. Other commands take `--agent $AGENT_NAME`. -- **Data Designer uses Python config files.** Pass the `.py` file to the CLI when `preview-local` is available; otherwise run via the venv Python as shown above. -- **Guardrails are NAT intercepts, not a separate service.** They go in the same YAML under `intercepts:`. There is no `nemo guardrails create` step in the agent build. -- **Framework constraint.** Only LangGraph-in-NAT agents work end-to-end today. +|---|---|---| +| Agents plugin unavailable | `plugins/nemo-agents` is not installed | Route to `nemo-setup` | +| Config validation fails | Config does not match its declared format | Use `nemo-agent-config` for `nemo-agents-spec-v1`; use NAT schema rules only for NAT YAML | +| Deployment reaches `failed` | Runtime, adapter, image, or config startup failure | Run `.venv/bin/nemo agents deployments get "$DEPLOYMENT_NAME"` and `.venv/bin/nemo agents logs "$DEPLOYMENT_NAME"` | +| Referenced file is missing | Path is outside or absent from the staged agent directory | Keep paths relative to the config and package the complete agent bundle into the deployed image | +| Adapter or binary is missing | Selected harness dependency is not installed | Install the matching adapter/runtime package or select an available harness | +| Empty response | Runtime invocation failed or the selected configuration is incomplete | Inspect deployment logs and the returned structured error | +| Eval job fails | Dataset reference or model ID is invalid | Get the benchmark job details and correct the named input | + +## Hard rules + +- Default new builds to `agents/$AGENT_NAME-spec/agent.yaml` with + `config_format: nemo-agents-spec-v1`. +- Keep `AGENT-SPEC.md` as the human-readable design and `agent.yaml` as the + machine-readable implementation config. +- Preserve legacy NAT YAML unless the user explicitly requests migration. +- Do not mix NAT-only keys such as `functions`, `llms`, `workflow`, or + `intercepts` into `nemo-agents-spec-v1`. +- Do not put Platform `agent.yaml` fields into NAT workflow YAML. +- Use a named deployment and invoke it with `--agent-deployment`. +- Keep local artifact paths relative to the config directory. +- After changing persisted Agent config, use the confirmed replacement branch + before creating and deploying it again. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json index 51947de0acb..73a307b7e72 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-build-agent/tests.json @@ -8,7 +8,7 @@ }, { "type": "explicit", - "prompt": "Run the nemo-build-agent skill. Spec is at agents/support-triage-spec/AGENT-SPEC.md.", + "prompt": "Use nemo-build-agent to implement and deploy the approved support-triage spec.", "expected_skill": "nemo-build-agent" }, { @@ -18,47 +18,57 @@ }, { "type": "implicit", - "prompt": "Build me an IT helpdesk agent that handles password resets, VPN issues, and software access requests.", + "prompt": "The IT helpdesk agent spec is approved. Build and deploy it for password resets, VPN issues, and software access requests.", "expected_skill": "nemo-build-agent" }, { "type": "implicit", - "prompt": "Scaffold and deploy the support agent we just specced out.", + "prompt": "Scaffold and deploy the support agent we just specced out on NeMo Platform.", "expected_skill": "nemo-build-agent" }, { "type": "implicit", - "prompt": "Take the spec in agents/ and ship the agent to NeMo Platform.", + "prompt": "Take the approved spec in agents/ and ship the agent to NeMo Platform.", "expected_skill": "nemo-build-agent" }, { - "type": "contextual", - "prompt": "Platform is up, models are listed, smoke chat worked. Now I want an agent that triages support cases and routes them to the right queue. Three categories: hardware, software, account access.", + "type": "implicit", + "prompt": "Spec is signed off. Build the agent, deploy it, and run the eval pass.", "expected_skill": "nemo-build-agent" }, { - "type": "contextual", - "prompt": "Spec is signed off. Move to deploy and run the eval pass.", + "type": "implicit", + "prompt": "Deploy my existing NAT workflow YAML as a NeMo Platform agent, then smoke test it.", "expected_skill": "nemo-build-agent" }, { - "type": "contextual", - "prompt": "Already explored and specced the agent design with the team. Want the workflow YAML and a deployment by EOD.", + "type": "implicit", + "prompt": "I have an old NAT agent workflow and an approved spec. Build and deploy it while preserving the legacy config.", "expected_skill": "nemo-build-agent" }, { "type": "negative-control", - "prompt": "Let's design a system for routing customer queries across teams. I want to think through the tradeoffs first before any code.", + "prompt": "Write an agent.yaml with a Codex harness, but do not register or deploy anything.", + "expected_skill_not": "nemo-build-agent" + }, + { + "type": "negative-control", + "prompt": "Convert this old NAT workflow YAML to nemo-agents-spec-v1, but do not deploy it.", + "expected_skill_not": "nemo-build-agent" + }, + { + "type": "negative-control", + "prompt": "Let's design a system for routing customer queries across teams before any implementation.", "expected_skill_not": "nemo-build-agent" }, { "type": "negative-control", - "prompt": "Build me a new React component for the settings page.", + "prompt": "Fix the CSS spacing in the React settings page.", "expected_skill_not": "nemo-build-agent" }, { "type": "negative-control", - "prompt": "Help me write a CrewAI agent from scratch on a fresh Python env.", + "prompt": "Help me write a CrewAI agent from scratch in a fresh Python environment outside NeMo Platform.", "expected_skill_not": "nemo-build-agent" } ] diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/SKILL.md index dd70e171341..6943f3f3c74 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/SKILL.md @@ -60,8 +60,8 @@ requirements: handoff to `nemo-spec` is blocked until both are resolved. | 3 | Scope | yes | Audience, 3-6 task categories, expected in-scope work, and explicit out-of-scope work/non-goals. | | 4 | Tools | yes | Tools, APIs, and knowledge sources the agent can use, or "Prompt-only." Group related helpers by capability or source. Capture only behaviorally important purpose, credentials/scopes, side effects, freshness, and expected failures. | | 5 | Model | yes | Mode (cloud vs local NIM) + model family/size. Example: "cloud, Nemotron Super 49B." `nemo-build-agent` resolves to a specific model entity ID later. | -| 6 | Framework | **yes** | Temporary NeMo Platform compatibility field. Keep it binary: `langgraph-nat` or `needs-wrapper`; include source-framework context only when a wrapper is needed. | -| 7 | Harness | optional | Modern harness description: the extra-model layer around the agent — loop, tool dispatch, context/state, guardrails, observability, verification, and runtime. Summarize at useful granularity; omit low-level settings unless they affect behavior. Use `_(none)_` if unknown. | +| 6 | Framework | **yes** | Record execution compatibility as `supported-harness`, `nat-workflow`, or `needs-adapter`. Include the source framework when one exists. Do not assume a Python framework name alone guarantees lifecycle compatibility. | +| 7 | Harness | optional | Describe the selected or likely harness and the behavior it owns: loop, tool dispatch, context/state, guardrails, observability, verification, and runtime. Use `_(none)_` if selection should wait until config authoring. | | 8 | Behavior | yes | Behavioral rules and boundaries: constraints, refusal/escalation policy, tone, safety/compliance requirements, accepted limitations, and known non-goals. | | 9 | Success Criteria | yes | What good production behavior looks like, independent of current evals: mission-level outcomes, quality standards, escalation quality, accuracy expectations, latency/cost expectations if relevant, and examples of success. | | 10 | Evaluation Setup | yes | Current validation setup: how to run it, what datasets/checks it uses, what scorers/metrics measure, pass/fail thresholds, and known coverage gaps relative to the success criteria. If no eval suite exists, say so explicitly. | @@ -88,16 +88,20 @@ Time-box this to ~5 minutes of tool use. Read first, ask second. Greenfield projects will turn up nothing here, which is fine — move to step 2 and ask the user the full set of unfilled fields. -1. **Find agent entry points.** Look for NAT workflow YAMLs, LangGraph - builders, system prompts, tool definitions: +1. **Find agent definitions and entry points.** Look for Platform + `agent.yaml`, NAT workflow YAMLs, supported harness configuration, Python + agent builders, system prompts, skills, and tool definitions: ```bash + find . -maxdepth 5 -type f -name "agent.yaml" 2>/dev/null find . -maxdepth 4 -type f \( -name "*.workflow.yaml" -o -name "*.workflow.yml" \) 2>/dev/null find . -maxdepth 4 -type d -name "agents" 2>/dev/null ``` - Then use `Glob` / `Grep` to find `langgraph`, `StateGraph`, - `create_react_agent`, `system_prompt`, and tool definitions. + Then use `Glob` / `Grep` to find `nemo-agents-spec-v1`, + `default_harness`, `codex`, `hermes`, `deepagents`, `claude`, `langgraph`, + `StateGraph`, `create_react_agent`, `system_prompt`, skills, MCP servers, + and tool definitions. 2. **Find design context.** Look for `README.md`, `AGENTS.md`, product/design/planning docs, launch notes, and anything in `docs/`. Read @@ -132,14 +136,18 @@ the user the full set of unfilled fields. Group low-level helpers when they share credentials, side effects, freshness, and failure modes. - **Model** — model id strings in workflow YAML, env vars, config files. - - **Framework** — `langgraph` import + NAT workflow YAML → "LangGraph + - NAT." `crewai` / `autogen` / `pydantic_ai` imports → `needs-wrapper` - with source-framework context. Plain `langchain` without `langgraph` → - `needs-wrapper`. Do not turn this into a detailed implementation audit. - - **Harness** — `langgraph` imports, NAT workflow YAML, `crewai` / - `autogen` / `pydantic_ai` imports, service entrypoints, CLI commands, - Dockerfiles, notebooks, or deployment configs. Capture what exists - descriptively; platform-specific wrapper needs can go in notes. + - **Framework** — a validated `nemo-agents-spec-v1` config selecting a + supported harness → `supported-harness`; an existing NAT workflow YAML → + `nat-workflow`; an arbitrary Python entrypoint or unsupported framework + without a start/stop/invoke lifecycle contract → `needs-adapter`. Record + the source framework separately when known. Do not infer compatibility + solely from imports such as `langchain`, `langgraph`, `crewai`, + `autogen`, or `pydantic_ai`. + - **Harness** — infer from `default_harness` and `harnesses` in + `agent.yaml`, adapter configuration, NAT workflow YAML, service + entrypoints, CLI commands, Dockerfiles, notebooks, or deployment configs. + Capture behaviorally relevant capabilities, not low-level settings. If + there is no selection yet, leave it unresolved for `nemo-agent-config`. - **Behavior** — system prompt rules ("never give medical advice"), refusal/escalation policy, tone, accepted limitations, and non-goals. - **Success Criteria** — desired production outcomes, product goals, @@ -226,8 +234,8 @@ After the user's reply, apply the corrections and check the two hard preconditions: 1. **Role** is a concrete one-sentence answer (not "help with stuff"). -2. **Framework** is resolved (`langgraph-nat` or `needs-wrapper` with a - source-framework name). +2. **Framework** is resolved to `supported-harness`, `nat-workflow`, or + `needs-adapter`, with source-framework context when known. If either is still unresolved, ask for it in one final message and stop until the user provides it. Do not hand off with a hard requirement blank — @@ -273,10 +281,14 @@ to Filesets") and trigger it. into questions.** Spending the first five minutes reading earns the right to ask shorter, sharper questions. Asking something the codebase already answers loses trust immediately. -- **NeMo Platform optimizes LangGraph agents wrapped in NAT today.** Other - frameworks may still be valid AGENTSpec harnesses, but need a user-written - wrapper for the current NeMo build path. Record that as Harness notes; do - not make the standard schema a NeMo-specific capability gate. +- **Framework names do not prove execution compatibility.** A supported + harness must own the required lifecycle contract. Preserve an existing NAT + workflow as `nat-workflow`; classify an arbitrary Python entrypoint without + that contract as `needs-adapter` rather than promising direct execution. +- **Keep Platform terminology at the design boundary.** Record the desired + harness behavior and artifacts without exposing Fabric SDK types or asking + the user to design a raw runtime config. `nemo-agent-config` owns the + machine-readable Platform YAML after the spec is approved. - **Change Scope is a permissions list, not a wishlist.** It controls what the experimentalist agent will edit. Walk the defaults explicitly so the user knows what they're consenting to. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/tests.json index c5cb582cd30..06c4b600878 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-explore/tests.json @@ -32,20 +32,30 @@ "expected_skill": "nemo-explore" }, { - "type": "contextual", + "type": "implicit", + "prompt": "I have an existing NAT support agent, but before migrating anything I want to revisit its role, tools, and runtime needs.", + "expected_skill": "nemo-explore" + }, + { + "type": "implicit", "prompt": "Following up on yesterday's chat about a customer-support agent on NeMo Platform. Before I write any code, I want the design right. Platform is already running.", "expected_skill": "nemo-explore" }, { - "type": "contextual", + "type": "implicit", "prompt": "Got a vague brief from the CTO: he wants a NeMo agent that helps engineers debug pipelines. Need to pin down what it actually does.", "expected_skill": "nemo-explore" }, { - "type": "contextual", + "type": "implicit", "prompt": "Want to scope a NeMo agent for our HR team. Not sure what categories or tools it needs yet.", "expected_skill": "nemo-explore" }, + { + "type": "negative-control", + "prompt": "Convert this NAT workflow YAML to the new Platform agent.yaml format.", + "expected_skill_not": "nemo-explore" + }, { "type": "negative-control", "prompt": "What questions should I ask in my next user research interview about checkout flow?", diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/SKILL.md index eddd7eac8e4..4903251a10f 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-model-selection -description: Recommends an LLM for a NeMo Platform agent based on what the agent actually has to do, explained in plain English before any benchmark name appears. Use when the user is choosing a model for a new agent, asking which model to use, or unsure what to put in their spec or NAT workflow YAML. Invoked by nemo-explore at the model question; also runs standalone when the user starts mid-flow. +description: Recommends an LLM for a NeMo Platform agent based on what the agent actually has to do, explained in plain English before any benchmark name appears. Use when the user is choosing a model for a new agent, assessing a model they already selected, or deciding what belongs in AGENT-SPEC.md or Platform agent.yaml. Invoked by nemo-explore at the model question; also runs standalone when the user starts mid-flow. triggers: - which model should I use - what model is best for this @@ -22,7 +22,11 @@ allowed-tools: [Read, Bash] # NeMo Platform model selection -Recommends a model for a new agent (NIM or any other provider configured on the running platform). Plain-English first, benchmark numbers second, never the other way around. Output: one recommended model with a one-sentence reason, ready to drop into a spec or NAT workflow YAML. +Recommend a model for a new agent from NIM or another provider configured on +the running Platform. Explain the capability fit first and benchmark evidence +second. Return the model choice in a form suitable for `AGENT-SPEC.md` and the +Platform-owned `agent.yaml`. Preserve NAT model configuration only when the +user is explicitly maintaining a legacy NAT workflow. ## Pre-flight @@ -41,20 +45,30 @@ python scripts/refresh-benchmark-cache.py The cache (schema v6+) carries four things the rest of this skill reads: - `models[]` — editorial entries for a curated set of NIMs with `strong_at`, `watch_out_for`, `intent_hints`, `derived_from` lineage, and direct/inferred scores. - `upstream_index.bfcl_v4` and `upstream_index.arena_elo` — full BFCL and per-category Arena Elo tables for ~84 and ~360 models respectively. Use these to look up scores for ANY model name, not just the registered ones. -- `namespace_to_type[]` — namespace-prefix → NAT `_type` value mapping for the YAML emitter. +- `namespace_to_type[]` — namespace-prefix → NAT `_type` mapping used only for legacy NAT workflow output. - `name_decomposition_rules[]` — pattern→hint rules for synthesizing `intent_hints` when an unknown model name lands. -### 2. Fetch the live model list from the running platform +### 2. Fetch the live model list for a Platform-routed harness ```bash -curl -fsS http://localhost:8080/v1/models 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); names=[m.get('id') for m in d.get('data',[])]; print('\n'.join(n for n in names if n))" 2>/dev/null || echo "PLATFORM_UNREACHABLE" +nemo models list --all-pages --output-format json 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print('\n'.join(m['name'] for m in d.get('data', []) if m.get('name')))" 2>/dev/null || echo "PLATFORM_UNREACHABLE" ``` Interpretation: -- **List of model ids returned** → these are the candidates the user can actually pick from. Carry them through to Step 1+. -- `PLATFORM_UNREACHABLE` → platform isn't up. Fall back gracefully: tell the user "I can't reach the local platform, so I'll recommend from the curated NIM set in the cache instead of your actual available models. Start the platform with `nemo services run` if you want recommendations grounded in what's deployed." +- **Model names returned** → these are the candidates the user can actually pick from. Carry their exact names through to Step 1+; the JSON `id` value is a Platform entity id and must not be written to `agent.yaml` as the inference model identifier. +- `PLATFORM_UNREACHABLE` → platform isn't up. Recommendations may continue + from the curated cache, but a model for Platform `agent.yaml` cannot be + finalized until the live Platform model list and harness-specific inference + route can be checked. -The `/v1/models` response is OpenAI-shaped (`{data: [{id, ...}, ...]}`); the parse above extracts the `id` field per entry. If the platform's response shape differs, adjust the parse but keep the failure mode (graceful fallback to closed registry, never silently steer the user). +Use the `nemo` CLI rather than constructing a Platform URL or calling +`/v1/models` directly. The CLI resolves `NEMO_BASE_URL`, `NMP_BASE_URL`, the +active CLI context, authentication, and workspace consistently with subsequent +agent commands. Do not hardcode `localhost`, `127.0.0.1`, or port `8080`. + +This list is authoritative only for models routed through Platform. For a +native-provider harness such as `claude`, use the configured provider's native +model catalog and validation tooling instead. ## Step 0 — Pick the conversation direction @@ -105,19 +119,79 @@ Do not propose a model before all three answers are in. Push back on "you decide ## Step 1.5 — Build the candidate set and pick a presentation pattern +Identify the selected harness and whether it uses a Platform-routed or native +provider path before building candidates. Read it from the source config or +conversation; ask if it is still unknown. + ### Building candidates The candidate set is what the user can actually pick from. It comes from three joins: -1. **Start with the pre-flight model list from `/v1/models`** (or, if `PLATFORM_UNREACHABLE`, fall back to the cache's `models[]` editorial registry). -2. **For each available model id, look up evidence** in this order: +1. **Start with the correct live catalog.** Use the pre-flight Platform model + list for Platform-routed models. Use the configured provider's native model + catalog for native-provider harnesses. If the required catalog is + unreachable, the cache may support a conversational recommendation, but do + not finalize a Platform `agent.yaml` model block. Availability alone does + not establish compatibility with an agent harness. +2. **For each available model name, look up evidence** in this order: - Token-match against the editorial `models[]` entries → if hit, use the full editorial record (lineage, intent_hints, direct + inferred scores) - If no editorial match, token-match against `upstream_index.bfcl_v4` keys → if hit, use that BFCL score with `source: "direct_external"` - Same for `upstream_index.arena_elo` for per-category Elo - If neither editorial nor upstream matches, synthesize `intent_hints` by walking `name_decomposition_rules[]` and collecting every hint whose `pattern` token appears in the decomposed model id. Mark evidence as `source: "name_only"`. 3. **Rank candidates by the user's profile** — primary capability axis determines which score field dominates. -When `/v1/models` was unreachable, also tell the user the rest of this flow is operating on the curated NIM set, not their actual deployment. +When the live Platform model list was unreachable, also tell the user the rest +of this flow is operating on the curated NIM set, not their actual deployment. + +### Verify harness compatibility before config handoff + +When the output targets Platform `agent.yaml`, rank a short candidate list from +the appropriate live catalog and the evidence above, then verify candidates +against the selected harness's actual model contract: + +| Harness | Required model contract | Compatibility check | +|---|---|---| +| `codex` | OpenAI Responses API | Valid `v1/responses` inference request | +| `hermes` | OpenAI-compatible chat completions | Valid `v1/chat/completions` inference request | +| `deepagents` | Provider-specific; `nvidia`, `openai`, and `openai-compatible` use chat completions | Valid request for the selected provider path; use `v1/chat/completions` for an OpenAI-compatible route | +| `claude` | Native Anthropic provider | Require `provider: anthropic` and validate the configured credentials/model with native Anthropic tooling; do not route it through Platform IGW | + +Before making any inference requests, show the user the candidate names and +explain that the checks make real, potentially billable model calls. Ask for +explicit confirmation and wait. The original request to select a model or +write a config is not confirmation for these calls. For native-provider +harnesses, use the provider's native validation path instead of forcing the +request through Platform IGW. + +For a Platform-routed OpenAI-compatible candidate, use the context-aware CLI +rather than a hardcoded URL. Preserve the exact model name returned by +`nemo models list`: + +```bash +MODEL_NAME="" + +# Codex +nemo inference gateway model post v1/responses "$MODEL_NAME" \ + --body "{\"model\":\"$MODEL_NAME\",\"input\":\"Reply with exactly: compatibility check\"}" + +# Hermes or an OpenAI-compatible DeepAgents configuration +nemo inference gateway model post v1/chat/completions "$MODEL_NAME" \ + --body "{\"model\":\"$MODEL_NAME\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: compatibility check\"}]}" +``` + +A successful model list lookup, schema validation, Fabric planning, deployment +readiness, empty request, or `GET` does not establish compatibility for a +Platform-routed model. Only candidates that complete a valid request through +the required Platform model path may be returned to `nemo-agent-config`. For a +native-provider harness, require its adapter provider contract and successful +native credential/model validation instead. Exclude failed combinations and +try the next ranked candidate. If no candidate passes, stop without emitting a +model block and ask the user to configure a compatible provider or explicitly +choose a different harness. Never switch the harness silently. + +Record the selected harness, provider, exact model name, required model +contract, and successful check in the handoff. Then apply the normal benchmark +ranking only among compatible candidates. ### Picking the presentation pattern @@ -257,7 +331,11 @@ If the user asks what benchmark was used or wants the raw number, tell them. Do ## Step 4 — Output -Two ready-to-paste blocks. Show whichever fits the user's stage. **When the chosen model's primary-axis score has `source: "inferred_from_ancestor"` or the model relies on `intent_hints` only, include an explicit evidence caveat in the output** — don't let the spec or YAML carry the recommendation forward without surfacing the inference. +Show the blocks that fit the user's stage. Default machine-readable output to +Platform `agent.yaml`. **When the chosen model's primary-axis score has +`source: "inferred_from_ancestor"` or the model relies on `intent_hints` only, +include an explicit evidence caveat in the human-readable recommendation.** Do +not encode benchmark commentary as unsupported config fields. If they're authoring an agent spec for `nemo-spec`: @@ -271,7 +349,42 @@ If they're authoring an agent spec for `nemo-spec`: - **Deployment:** ``` -If they're editing a NAT workflow YAML directly (e.g. tweaking the `agent.yml` `nemo-build-agent` produced): +If they are authoring Platform `agent.yaml`, emit a default model block: + +```yaml +models: + default: + provider: + model: + api_key_env: + base_url: +``` + +Use the provider identity configured on the Platform. Omit `api_key_env` and +`base_url` when the selected provider does not require user-supplied values. +Keep `base_url` directly in the model block, not under `settings`. + +Before emitting this block, complete the harness-specific compatibility check +above. Do not infer compatibility from provider or model-list metadata alone. + +The default model applies to every harness that does not declare its own +model. Add a harness-local override only when that harness intentionally uses +a different model or provider: + +```yaml +harnesses: + : + kind: + model: + provider: + model: +``` + +Do not emit raw Fabric SDK model objects. `nemo-agent-config` owns final YAML +placement and validation. + +If they are explicitly maintaining a legacy NAT workflow YAML, emit the NAT +compatibility block: ```yaml llms: @@ -288,9 +401,10 @@ workflow: tool_names: [] ``` -### Picking the right `_type` +### Picking the right legacy NAT `_type` -Match the chosen model's namespace prefix against `namespace_to_type[]` from the cache: +Only for NAT workflow output, match the chosen model's namespace prefix against +`namespace_to_type[]` from the cache: ```txt For each rule in cache.namespace_to_type: @@ -307,9 +421,15 @@ If no rule matches: Common mappings the cache carries today: `nim/*`, `openai/*`, `anthropic/*`, `bedrock/*`, plus vendor-published NIM names (`qwen/*`, `meta/*`, `nvidia/*`, `microsoft/*`, `mistralai/*`) that route through the NIM provider when served by the platform. Ollama's local endpoint maps to `_type: openai` since it exposes an OpenAI-compatible API. -When the chosen model is non-NIM, also remind the user to set `base_url` and `api_key` (or the equivalent env vars) in the LLM block — those are mandatory for non-NIM providers and aren't auto-filled like they are for the platform's NIM defaults. +For Platform `agent.yaml`, represent credentials with `api_key_env` and put +`base_url` directly in the model block. For a legacy NAT workflow, use the +provider fields required by that NAT LLM component. + +### Pair a legacy NAT model with the right workflow type -### Pair the model with the right agent type +Use this table only when maintaining NAT workflow YAML. Harness selection for +`nemo-agents-spec-v1` belongs to `nemo-agent-config` and must not be inferred +from a NAT workflow type. | What the agent needs to do | Use | |---|---| @@ -348,19 +468,30 @@ If `nemo-explore` invoked this skill, return control to `nemo-explore` with the | User wants a model not in the table | The table is curated, not exhaustive | Tell them honestly; describe the capability gap their choice would have vs the closest recommended model | | `cache_missing` and user wants fresh data | Cache has never been refreshed in this checkout | Tell them the refresh command and note that the static table is still usable | | User picks self-hosted but the recommended model needs cloud | Hard constraint conflict | Drop the recommendation; pick the closest self-hosted-compatible model from the table | +| Available model's harness API support is unknown | Availability was mistaken for harness compatibility | Ask permission to run a valid request through the harness's required model path | +| Compatibility request fails | The exact harness/provider/model/endpoint combination is incompatible in the current environment | Exclude that combination, surface the error, and test the next ranked candidate after the user-approved probe set | +| No candidate passes | No live model satisfies the selected harness contract | Stop without emitting a model block; ask the user to configure a compatible provider or explicitly choose another harness | ## Hard rules - Never name a model before all three profile questions are answered. - Never lead with a model name, benchmark name, or score. - Never recommend a cloud-only model when the user said self-hosted. -- Never write `model_name` (YAML) or "NIM model id" (spec) without showing the plain-English reason alongside it. +- Never return an untested model route to a config-writing skill. +- Never make model inference calls without explicit user confirmation. +- Never silently change the selected harness to accommodate an available model. +- Never emit a model identifier without showing the plain-English reason alongside it. - **When the primary candidate's evidence is anything other than `direct`, the model name does not appear in your response until the user has resolved the trade-off in Pattern B.** Anchoring is the failure mode this guards against — users default to the first model named regardless of caveats. The withhold is non-negotiable. -- When emitting the spec or YAML block, always include an Evidence line/comment naming the source quality. Inferred or name-only choices that propagate downstream without that signal mislead the build skill and the user both. +- When emitting the spec recommendation, always include an Evidence line naming + the source quality. For `agent.yaml`, present the evidence next to the YAML + rather than inventing a config field. ## Gotchas - **"You decide" needs a committed default, not a silent fill-in.** Same rule as `nemo-explore`. Pick something, name it, tell the user. -- **The platform default is `nvidia/llama-3.3-nemotron-super-49b-v1`.** If `nemo-explore` already captured "cloud, no preference", you can route there without re-profiling — but still explain *why* in plain English instead of just naming it. -- **Two model name formats coexist.** Entity-name with hyphens for NAT YAML / `nemo chat` / `nemo agents`. API-Catalog format with slashes for Data Designer. Use the slashed form (`qwen/qwen3-235b-a22b`) in NAT YAML for cloud NIMs; the build skill converts when needed. +- **Do not transform model IDs by punctuation convention.** Use the identifier + returned by the selected live provider or Platform model listing and pair it + with the correct `provider`. Legacy NAT components and Data Designer may use + different provider-specific identifiers; preserve the identifier required by + that consumer instead of assuming the build skill converts it. - **Watch the deployment column.** A 235B cloud-API recommendation aimed at a self-hoster with a 24 GB GPU is the most common mismatch and the easiest to catch by re-reading Step 1. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/tests.json index 9d3c81bc9db..04f38decf39 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-model-selection/tests.json @@ -37,17 +37,17 @@ "expected_skill": "nemo-model-selection" }, { - "type": "contextual", + "type": "implicit", "prompt": "nemo-explore is finished. The agent does triage, three tools, cloud deploy. What model do I put in the spec?", "expected_skill": "nemo-model-selection" }, { - "type": "contextual", + "type": "implicit", "prompt": "I went through agent design but skipped past the model question. Let's settle that now.", "expected_skill": "nemo-model-selection" }, { - "type": "contextual", + "type": "implicit", "prompt": "Spec is done except for the model. It's a code-review agent that reads PRs and suggests fixes.", "expected_skill": "nemo-model-selection" }, @@ -67,10 +67,35 @@ "expected_skill": "nemo-model-selection" }, { - "type": "contextual", + "type": "implicit", + "prompt": "Which model should go in models.default for this Platform agent.yaml?", + "expected_skill": "nemo-model-selection" + }, + { + "type": "implicit", + "prompt": "Should this Hermes harness override the agent's default model or reuse it?", + "expected_skill": "nemo-model-selection" + }, + { + "type": "implicit", + "prompt": "Codex is my selected harness and I want to use an NVIDIA model through Platform IGW. Verify that the exact model route supports the Responses API before recommending it.", + "expected_skill": "nemo-model-selection" + }, + { + "type": "implicit", + "prompt": "The Codex candidate failed its valid Responses API compatibility request. Exclude that combination and test the next candidate before recommending it.", + "expected_skill": "nemo-model-selection" + }, + { + "type": "implicit", "prompt": "I've already picked gpt-4o-mini. Tell me what I'm trading off vs alternatives.", "expected_skill": "nemo-model-selection" }, + { + "type": "negative-control", + "prompt": "Validate the completed agent.yaml and finish its harness configuration.", + "expected_skill_not": "nemo-model-selection" + }, { "type": "negative-control", "prompt": "I want to build a NeMo agent but I haven't thought through what it needs to do yet.", diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/SKILL.md index b0597932805..dc5f3318626 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-skill-selection -description: Top-level skill selector for any task involving NeMo Platform (NVIDIA's agent platform). Picks the right downstream skill (setup, explore, spec, build, try, status, teardown, customization training) from natural-language intent. Use over generic brainstorming, planning, or onboarding skills for any NeMo Platform task. +description: Top-level skill selector for ambiguous tasks involving NeMo Platform (NVIDIA's agent platform). Picks the right downstream skill for setup, design, specification, agent configuration, build, deployment, testing, status, teardown, evaluation, optimization, security, or model customization. Use when the user needs help deciding where a NeMo Platform task should start. triggers: - build an agent - create an agent @@ -33,7 +33,15 @@ allowed-tools: [Read] You are deciding which downstream NeMo Platform skill should run. This skill never executes commands. It picks the next skill, announces the choice, and hands off. -NeMo Platform optimizes LangGraph agents wrapped in NVIDIA NeMo Agent Toolkit (NAT). State that constraint when the user describes an agent in another framework (CrewAI, AutoGen, plain LangChain, Pydantic AI). Those frameworks need a user-written NAT wrapper before the platform's optimization, evaluation, and guardrails surfaces apply. +New NeMo Platform agent builds use a Platform-owned `agent.yaml` with +`config_format: nemo-agents-spec-v1` and a supported harness. NVIDIA NeMo Agent +Toolkit (NAT) workflow YAML remains a compatibility path. Do not describe NAT +as the only supported implementation model. + +If an existing agent does not fit a supported harness contract, route based on +the user's goal: preserve an existing NAT workflow, identify a custom adapter, +or use `nemo-agent-config` for a best-effort migration. Do not promise that an +arbitrary Python entrypoint can be converted mechanically. ## Decision table @@ -44,8 +52,9 @@ Match the user's intent to one downstream skill. Pick exactly one. | "set up", "install", "get started", "try NeMo", "first time" | `setup` | Verify the platform is installed and running. If not, the skill tells the user how to run the CLI install (`make bootstrap` + `nemo setup`). Install itself is CLI-only. | | "design an agent", "I want an agent that handles X", "what should my agent do" | `nemo-explore` | Capture the agent's job, audience, categories, tools, model, constraints before any code | | "write the spec", "save the design", "capture what we agreed" | `nemo-spec` | Persist the explore answers as `agents/-spec/AGENT-SPEC.md` | -| "build the agent", "create the agent", "deploy", "scaffold from spec" | `nemo-build-agent` | Scaffold the NAT workflow YAML, deploy, eval, optional guardrails | -| "ask my agent", "try the agent", "test it" | `nemo-try-agent` | Send a query to a deployed agent or fall back to model chat | +| "write agent.yaml", "validate agent.yaml", "choose a harness", "migrate this NAT YAML", "convert to nemo-agents-spec-v1" | `nemo-agent-config` | Author or migrate the Platform-owned machine-readable config without running the full build | +| "build the agent", "create the agent", "deploy", "scaffold from spec" | `nemo-build-agent` | Build from the approved spec, default to Platform `agent.yaml`, register, deploy, evaluate, and optionally apply guardrails | +| "ask my agent", "try the agent", "test it", "invoke this agent.yaml" | `nemo-try-agent` | Invoke a named deployment or run a local agent YAML config once | | "status", "what is running", "platform health", "is the platform up", "what's deployed", "show me what's running" | `nemo-status` | Read-only dashboard: platform, agents, providers, models | | "shut down", "stop NeMo", "tear down", "clean up" | `nemo-teardown` | Stop the cluster (keep data, delete platform data, or full cleanup) | | "fine-tune", "customize the model", "train on my data", "SFT", "LoRA" | `nemo-customizer` | Model customization via installed customization contributor plugins (`nemo-customizer-plugin`). Requires plugin skills to be installed (`nemo skills install` / enabled-plugins). | @@ -55,7 +64,12 @@ Match the user's intent to one downstream skill. Pick exactly one. **Optimize vs build:** Do NOT route optimize asks to `nemo-build-agent`. Build is for creating new agents from a spec; optimize is for tuning **already deployed** agents. If the user says "make my agent faster" or "use a cheaper model," that is `agents-optimize`, not `nemo-build-agent`. -If two rows fit, pick the earliest one in the lifecycle (setup before build before try). If nothing matches, ask one disambiguating question with the relevant rows as a numbered list. +If a request includes both config authoring and deployment, choose +`nemo-build-agent`; it delegates the config portion to `nemo-agent-config`. +Choose `nemo-agent-config` when the requested output stops at a validated config +or migration. Otherwise, if two rows fit, pick the earliest one in the +lifecycle. If nothing matches, ask one disambiguating question with the +relevant rows as a numbered list. ## Pre-flight @@ -100,8 +114,9 @@ NeMo Platform skills I can route to: setup verify install or get the CLI install command nemo-explore design conversation: capture goal, audience, tools, constraints nemo-spec write the design to agents/-spec/AGENT-SPEC.md - nemo-build-agent scaffold the NAT workflow YAML and deploy - nemo-try-agent query a deployed agent or chat with a model + nemo-agent-config author, validate, or migrate Platform agent.yaml + nemo-build-agent build from the spec, register, deploy, evaluate, and sign off + nemo-try-agent invoke a named deployment or local agent YAML config nemo-status read-only platform health dashboard nemo-teardown guided shutdown @@ -143,4 +158,8 @@ Do not proactively suggest Studio as the path for anything a skill already cover - **Install must happen before any skill can do useful work.** Build, try, and status all assume the platform is up. If the user has not run the CLI install (`make bootstrap` + `nemo setup`), the skills cannot work around that; hand them to `setup` for instructions. - **NeMo Platform is the product name.** Capital N, e, M, o, P. Not "nemo" or "Nemo." NAT on first mention is "NVIDIA NeMo Agent Toolkit (NAT)." - **Model customization** goes to the `nemo-customizer` plugin skill when `nemo-customizer-plugin` (and a training backend) are installed. If that skill is not available, tell the user to enable customization plugins and install skills — do not improvise training with an external library. -- **Framework honesty.** If the user describes an agent in CrewAI, AutoGen, plain LangChain, or Pydantic AI, tell them up front that NeMo Platform's optimization and evaluation surfaces operate on NAT-wrapped LangGraph agents. They will need to wrap their agent before the build path produces value. +- **Execution compatibility.** New Platform configs must select a supported + harness. Existing NAT workflows may remain on the NAT compatibility path. + For another framework or an arbitrary Python entrypoint, inspect whether a + supported harness owns its lifecycle; otherwise identify a custom adapter or + NAT wrapper instead of claiming direct support. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/tests.json index eaa3912c5eb..07d1b65aff2 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-skill-selection/tests.json @@ -32,20 +32,30 @@ "expected_skill": "nemo-skill-selection" }, { - "type": "contextual", + "type": "implicit", "prompt": "We are prepping for a customer demo tomorrow. They want to see the NeMo agent platform end-to-end. Get it running and build something simple.", "expected_skill": "nemo-skill-selection" }, { - "type": "contextual", + "type": "implicit", "prompt": "Cloned the Platform repo this morning. Never used NeMo before. What is the first step.", "expected_skill": "nemo-skill-selection" }, { - "type": "contextual", + "type": "implicit", "prompt": "Got a LangGraph agent in production. Want to put it through NeMo Platform to harden and evaluate it.", "expected_skill": "nemo-skill-selection" }, + { + "type": "negative-control", + "prompt": "Write and validate agent.yaml with Codex as the default harness.", + "expected_skill_not": "nemo-skill-selection" + }, + { + "type": "negative-control", + "prompt": "Convert this existing NAT workflow YAML to nemo-agents-spec-v1 without deploying it.", + "expected_skill_not": "nemo-skill-selection" + }, { "type": "negative-control", "prompt": "Design a system for routing customer queries across three different teams. Help me think through the tradeoffs.", diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/SKILL.md index 42612124b50..970c5d1b2ea 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/SKILL.md @@ -23,7 +23,8 @@ allowed-tools: [Read, Write, Edit, Bash] # NeMo Platform agent spec Turn the answers from `nemo-explore` into a durable artifact. The spec is -the contract `nemo-build-agent` reads to scaffold the NAT workflow YAML and +the contract `nemo-build-agent` reads before producing the Platform-owned +`agent.yaml` or preserving an existing NAT compatibility workflow, and the `AGENT-SPEC.md` that downstream optimization agents read as their primary context. Without it, downstream skills have to re-ask everything and the optimization loop has no contract for what the agent is @@ -34,12 +35,13 @@ supposed to do or what may be changed. Two copies of the spec exist intentionally: * **Canonical**: a NeMo Filesets fileset named `-spec` in the - active workspace, holding a single file `AGENT-SPEC.md`. Downstream - optimization services read this copy server-side; the platform stores it - durably. -* **Local cache**: `agents/-spec/AGENT-SPEC.md` in the developer's - working directory. Hand-editable, version-controlled with the agent's repo, - used by this skill and by `nemo-build-agent`. + active workspace. It holds `AGENT-SPEC.md` and may also hold `agent.yaml` + plus relative artifacts used by the executable agent package. Downstream + services derive the relevant file ref from workspace and agent name. +* **Local cache**: `agents/-spec/` in the developer's working directory. + `AGENT-SPEC.md` is the human-readable contract; `agent.yaml` is the optional + machine-readable Platform config created by `nemo-agent-config` during the + build path. The Fileset wins on conflict. If a developer edits the local file, this skill re-uploads to refresh the Fileset. If the platform copy has drifted @@ -64,10 +66,10 @@ default. 1. **Role** — one concrete sentence describing the role this agent plays. Vague answers ("help with stuff", "answer questions") are rejected at write time by the `AgentSpec` validator and will fail the file write. -2. **Framework** — temporary NeMo Platform compatibility status, resolved to - one of `langgraph-nat` or `needs-wrapper` (with source-framework context - when `needs-wrapper`). The lightweight parser refuses unresolved framework - sections. +2. **Framework** — NeMo Platform execution compatibility, resolved to one of + `supported-harness`, `nat-workflow`, or `needs-adapter`. Include source + framework context when known. The lightweight parser refuses an empty or + unresolved framework section. The AGENTSpec parser (`nemo_agents_plugin.spec_parse.parse_spec`) enforces both at validation time; this skill enforces them upstream so the user sees a @@ -114,9 +116,10 @@ clear gap-question rather than a parser error. capability/source when they share credentials, side effects, freshness, and failure modes. Keep only details that change how downstream agents evaluate behavior. - - `Framework` should be binary: `langgraph-nat` or `needs-wrapper`, with - source-framework context only for `needs-wrapper`. Do not expand it into a - platform compatibility essay. + - `Framework` should use `supported-harness`, `nat-workflow`, or + `needs-adapter`. Record the source framework separately when known. Do not + infer execution compatibility from a framework import alone or expand the + section into a platform compatibility essay. - Avoid public shorthand like `AUT` or "agent under test." Use "this agent" for the agent being specified. Use "target agent" only when this agent's job is explicitly to inspect or modify another agent. @@ -169,9 +172,11 @@ clear gap-question rather than a parser error. 10. **Hand off.** Once confirmed, tell the user the next skill: - - `nemo-build-agent` will read `agents/-spec/AGENT-SPEC.md`, produce the - workflow YAML, and call `nemo agents create`. It does not need a - `--spec-file-ref` flag — the spec's location is derivable. + - `nemo-build-agent` will read `agents/-spec/AGENT-SPEC.md`, use + `nemo-agent-config` to produce `agent.yaml` by default, and call + `nemo agents create`. Existing NAT workflow YAML may remain on the + compatibility path. No `--spec-file-ref` flag is needed because the + spec's location is derivable. - The `eval-setup` skill (M2) will fill in the `Evaluation Setup` section when ready. - The insights plugin reads the same canonical fileset server-side once @@ -213,10 +218,10 @@ all print, and the user has confirmed the contents. ## What this skill is not -This skill does not produce NAT workflow YAML. The spec is the -human-readable design; the YAML is generated downstream by -`nemo-build-agent`. It also does not create the `Agent` entity on the -platform — that happens in `nemo-build-agent` via `nemo agents create`. +This skill does not produce `agent.yaml`, migrate NAT workflow YAML, or create +the `Agent` entity. The spec is the human-readable design. Machine-readable +config authoring belongs to `nemo-agent-config`, while registration and +deployment belong to `nemo-build-agent`. ## Gotchas @@ -224,9 +229,9 @@ platform — that happens in `nemo-build-agent` via `nemo agents create`. section headings intact. The parser in `nemo_agents_plugin.spec_parse` rejects missing or duplicate required sections, but section bodies remain markdown for humans and agents to read directly. -- **Spec lives next to the workflow YAML.** Local copies of both files end - up in `agents/`. Keep them adjacent so a future read of the directory - shows design and implementation together. +- **Spec lives next to the implementation config.** Keep `AGENT-SPEC.md`, + Platform `agent.yaml`, and their relative artifacts under + `agents/-spec/` so local and Filesets consumers share one package root. - **The Fileset is canonical, not the local file.** If the two disagree, the Fileset wins. Re-pull before editing if you suspect server-side drift. @@ -239,6 +244,10 @@ platform — that happens in `nemo-build-agent` via `nemo agents create`. `[a-z][a-z0-9-]*`. - **Role and Framework are hard requirements.** Do not write the spec with either missing. Route back to `nemo-explore` for the missing field only. +- **Framework is execution readiness, not a library label.** Use + `supported-harness` only when a supported harness can own the lifecycle, + `nat-workflow` for the existing NAT compatibility path, and `needs-adapter` + when no supported lifecycle contract exists. - **Purpose cannot be implementation-only by accident.** If goal context was not found in the codebase and the user did not provide outside context, make that provenance clear instead of letting implementation details masquerade as diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/references/templates/agent-spec.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/references/templates/agent-spec.md index baa405053f3..3393e82d254 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/references/templates/agent-spec.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/references/templates/agent-spec.md @@ -67,12 +67,13 @@ bad agent behavior or a normal tool/source limitation.> ## Framework -- Resolution: -- Source framework: -- Notes: +- Resolution: +- Source framework: +- Notes: ## Harness +- Selection: - Description: - Agent loop: - Tool dispatch: diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/tests.json index 94317f9e58c..5c6ba379ad4 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-spec/tests.json @@ -32,20 +32,35 @@ "expected_skill": "nemo-spec" }, { - "type": "contextual", + "type": "implicit", + "prompt": "Save the approved design and our decision to use a supported Codex harness in AGENT-SPEC.md.", + "expected_skill": "nemo-spec" + }, + { + "type": "implicit", + "prompt": "Persist our NAT compatibility decision in the agent's AGENT-SPEC.md so the design record stays accurate.", + "expected_skill": "nemo-spec" + }, + { + "type": "implicit", "prompt": "Okay we covered the categories, the tools, the model, and the constraints. Write this all down before I forget. Agent name is support-triage.", "expected_skill": "nemo-spec" }, { - "type": "contextual", + "type": "implicit", "prompt": "Explore is done. Job is clear, three categories, prompt-only, hosted Nemotron Super. Capture it.", "expected_skill": "nemo-spec" }, { - "type": "contextual", + "type": "implicit", "prompt": "Got buy-in from the team on the agent scope this morning. Want to lock it in writing before someone changes their mind.", "expected_skill": "nemo-spec" }, + { + "type": "negative-control", + "prompt": "Create agent.yaml with Hermes as the default harness.", + "expected_skill_not": "nemo-spec" + }, { "type": "negative-control", "prompt": "Write the OpenAPI spec for our new internal HR API.", diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/SKILL.md b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/SKILL.md index 228eb69cb2c..d26091eac7b 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/SKILL.md +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/SKILL.md @@ -1,19 +1,28 @@ --- name: nemo-try-agent -description: Sends a query to a deployed NeMo Platform agent (or falls back to direct model chat) and announces the routing decision before sending. Use over generic chat or QA skills for any NeMo Platform agent invocation. +description: Invokes an existing NeMo Platform agent through a named deployment or directly from a local agent YAML config. Use to try, test, or query an agent and inspect its response. triggers: + - nemo-try-agent - ask my agent + - ask my NeMo agent - try the agent - test it out + - test support agent with real question - query my agent - what does my agent say - send to the agent - try my nemo agent + - invoke deployed agent + - query running deployment + - invoke local agent config + - query local agent config + - invoke legacy NAT workflow + - test agent and show raw output not-for: - nemo-build-agent (use to deploy an agent before querying) - nemo-skill-selection (use to dispatch when intent is unclear) - nemo-status (use for read-only platform health) -compatibility: nemo-platform >= 0.1.0; running platform with at least one deployed agent (or a registered model for fallback); requires agents plugin; no destructive ops; safe under any sandbox. +compatibility: nemo-platform >= 0.1.0; requires agents plugin and either a local agent YAML config or a running platform with a deployed agent; no destructive ops; safe under any sandbox. maturity: active license: Apache-2.0 user-invocable: true @@ -22,57 +31,94 @@ allowed-tools: [Bash, Read] # NeMo Platform try-agent -Route a user query to a deployed agent or direct model. Announce the routing decision before sending. Never invoke silently. +Invoke an existing NeMo agent through a deployment or directly from a local YAML config. Announce the target before sending. Never invoke silently. ## Pre-flight -Confirm the platform is up and check what deployed agents exist before doing anything: +Choose the invocation mode from the user's target: + +- **Local one-shot:** the user provides an `agent.yaml` or legacy NAT workflow + YAML path. Read the config format and model settings before deciding whether + Platform readiness is required: + - A Platform-owned `nemo-agents-spec-v1` config invokes Fabric directly and + does not require Platform readiness. + - A legacy NAT config requires Platform readiness when any `openai` or `nim` + LLM omits `base_url`; local invocation injects the Platform IGW URL for + those entries. + - A legacy NAT config whose applicable LLMs all provide explicit `base_url` + values may invoke those endpoints directly without Platform readiness. +- **Deployed agent:** the user names a deployment or asks to use an already + deployed agent. Preserve the active CLI context and confirm the target + Platform is reachable by listing deployments: ```bash -# Ground truth: anything bound to :8080? +.venv/bin/nemo agents deployments list 2>/dev/null || { echo "PLATFORM_UNREACHABLE"; exit 1; } +``` + +When the user explicitly selects a local Platform, override any remote CLI +context for the current shell and add local process and health checks before +listing deployments: + +```bash +export NMP_BASE_URL=http://127.0.0.1:8080 lsof -iTCP:8080 -sTCP:LISTEN >/dev/null 2>&1 || { echo "PLATFORM_DOWN"; exit 1; } -# Functional check: platform readiness endpoint answers? -curl -sS --connect-timeout 2 --max-time 5 http://localhost:8080/health/ready -o /dev/null -w "%{http_code}\n" 2>/dev/null | grep -q "^200$" || { echo "PLATFORM_WEDGED"; exit 1; } +curl -sS --connect-timeout 2 --max-time 5 "$NMP_BASE_URL/health/ready" -o /dev/null -w "%{http_code}\n" 2>/dev/null | grep -q "^200$" || { echo "PLATFORM_WEDGED"; exit 1; } .venv/bin/nemo agents deployments list 2>/dev/null ``` -Do not use `nemo services status` for this check — it reports stale "running" from held locks after the process has died. +Do not use `nemo services status` for the local process check; it can report +stale "running" state from held locks after the process has died. -If `PLATFORM_DOWN`: route to `nemo-setup` and stop. If `PLATFORM_WEDGED`: route to `nemo-status` to surface the underlying error and stop. +Require these checks for a deployed invocation and for a local NAT invocation +that depends on injected Platform IGW routing. If `PLATFORM_UNREACHABLE` or +`PLATFORM_DOWN`, route to `nemo-setup` and stop. If `PLATFORM_WEDGED`, route to +`nemo-status` and stop. Do not require the checks for Platform-owned Fabric +local invocation or a NAT config with directly usable explicit endpoints. ## What you do 1. **Find the target.** - - One agent deployed: pick it as the target. - - Multiple agents deployed: ask the user which one. List names + statuses. - - No agents deployed: fall back to `nemo chat` against a model from `nemo models list`. Tell the user this is a model query, not an agent query. + - Local YAML path supplied: set `INVOCATION_MODE=local` and + `AGENT_CONFIG_PATH` to that config. + - Deployment named: confirm it is `running`, set + `INVOCATION_MODE=deployed`, and set `DEPLOYMENT_NAME` to its name. + - One running deployment and no target named: use it and set the deployed + mode variables above. + - Multiple running deployments: list their names, ask the user which one, + then set the deployed mode variables above. + - No running deployments: report that no deployed agent is available. Do not silently replace an agent invocation with `nemo chat`. 2. **Announce.** Say one of: - - "Sending to agent ``." - - "No agents deployed; sending to model `` via `nemo chat`." - - "Multiple agents deployed; which one: , ?" (then wait) + - "Invoking local agent config ``." + - "Sending to deployment ``." + - "Multiple deployments are running; which one: , ?" Then wait. 3. **Send the query.** ```bash -# Agent path -.venv/bin/nemo agents invoke --agent --input "" - -# Model fallback path -.venv/bin/nemo chat "" +if [ "$INVOCATION_MODE" = "local" ]; then + RESP=$(.venv/bin/nemo agents invoke \ + --agent-config "$AGENT_CONFIG_PATH" \ + --input "") +else + RESP=$(.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "") +fi +RC=$? ``` -4. **Show the verbatim response.** Code block, no paraphrase. If the agent used tool calls, list which tools and their outputs before the final answer. +1. **Show the verbatim response.** Print `RESP` in a code block without + paraphrasing it. If the agent used tool calls, list which tools and their + outputs before the final answer. -5. **Loop for follow-ups.** After the response, ask: "Another question, or done?" Keep the same target until the user changes it. +2. **Offer another invocation.** Keep the same target unless the user changes it. Do not claim that separate CLI invocations preserve a conversation session. ## Verification A "successful" invocation requires both: (a) the CLI returns exit code 0, and (b) the response body is non-empty. An empty body on a question the spec says the agent should handle is a quality signal, not a success. ```bash -RESP=$(.venv/bin/nemo agents invoke --agent --input "") -RC=$? if [ $RC -ne 0 ]; then echo "INVOKE_FAILED (exit $RC)" elif [ -z "$RESP" ]; then @@ -82,14 +128,19 @@ else fi ``` +Do not switch targets for verification. + If `INVOKE_FAILED` or `EMPTY_RESPONSE`: surface that to the user and stop. Do not claim the invocation succeeded. ## If verification fails | Symptom | Cause | Recovery | |---|---|---| -| 404 "agent not found" | Agent undeployed since last list | Re-run `.venv/bin/nemo agents deployments list`; ask user to pick from the new list | -| 5xx or platform error | Platform unhealthy | Route to `nemo-status` to surface the underlying error; offer to fall back to model chat | +| 404 "deployment not found" | Deployment was removed or the wrong name was used | Re-run `.venv/bin/nemo agents deployments list`; ask the user to pick from the new list | +| Deployment is not `running` | Deployment is still starting or failed | Inspect it with `.venv/bin/nemo agents deployments get `; do not invoke until it is running | +| Config validation error | Local YAML is invalid or references missing artifacts | Surface the validation details; route local YAML fixes or migrations to `nemo-agent-config`; use `nemo-build-agent` only when the user explicitly requests redeployment | +| Adapter or runtime error | Required harness package or runtime dependency is unavailable | Surface the structured invocation error and required dependency; do not substitute model chat | +| 5xx or platform error | Platform or deployed runtime is unhealthy | Route to `nemo-status` to surface the underlying error | | Empty response on a spec-handled question | Quality issue, not invocation issue | Stop and report; do not loop until the user decides next step | | "I cannot help" on every question | System prompt or tool wiring wrong in YAML | Route to `nemo-build-agent` to inspect and redeploy | | `agents plugin unavailable` | Plugin not installed | Route to `nemo-setup` Step 3 | @@ -97,6 +148,7 @@ If `INVOKE_FAILED` or `EMPTY_RESPONSE`: surface that to the user and stop. Do no ## Gotchas - **Routing must be explicit.** Silently picking a target and sending a query is the failure mode this skill exists to prevent. Announce first. -- **`nemo chat` and `nemo agents invoke` take different model id formats.** Chat uses entity-name (hyphens). Agents use whatever the YAML specifies. Pass through what the user says; do not auto-translate. -- **Use `curl` only for the pre-flight health probe.** The CLI is the documented interface for agent and model operations. Hand-rolled HTTP is not a substitute. -- **Loop in this skill, not in another.** Do not invoke `nemo-skill-selection` between turns. Stay here until the user asks to do something else. +- **Use an explicit deployment name.** `--agent-deployment` avoids ambiguity when one Agent entity has multiple deployments. +- **Keep agent and model chat distinct.** Offer `nemo chat` only when the user explicitly chooses a raw model query. +- **Use `curl` only for the pre-flight health probe.** The CLI is the documented interface for agent operations. Hand-rolled HTTP is not a substitute. +- **Do not promise session continuity.** Reusing a deployment target is not the same as resuming a specific multi-turn session. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/tests.json b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/tests.json index bb1c97b76b2..543752d5065 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/tests.json +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/skills/nemo-try-agent/tests.json @@ -18,7 +18,7 @@ }, { "type": "implicit", - "prompt": "Ask my NeMo agent how to reset a Jira password.", + "prompt": "Ask my deployed NeMo agent how to reset a Jira password.", "expected_skill": "nemo-try-agent" }, { @@ -32,18 +32,18 @@ "expected_skill": "nemo-try-agent" }, { - "type": "contextual", - "prompt": "I have the it-helpdesk agent deployed and want to test it with a real question. Also curious how it handles a question outside its categories.", + "type": "implicit", + "prompt": "Invoke the local agents/support-agent/agent.yaml once with the input hello.", "expected_skill": "nemo-try-agent" }, { - "type": "contextual", - "prompt": "Deployment finished a minute ago. Hit it with three categories of questions and show me the raw output.", + "type": "implicit", + "prompt": "Try my legacy NAT workflow YAML locally with a sample support question.", "expected_skill": "nemo-try-agent" }, { - "type": "contextual", - "prompt": "Want to see what the support-triage agent says before the team review at 2pm. Three test prompts is enough.", + "type": "implicit", + "prompt": "Invoke the support-triage deployment before the team review and show me the raw output for three test prompts.", "expected_skill": "nemo-try-agent" }, { @@ -60,6 +60,16 @@ "type": "negative-control", "prompt": "Ask ChatGPT to write a poem about coffee.", "expected_skill_not": "nemo-try-agent" + }, + { + "type": "negative-control", + "prompt": "Build and deploy a new support agent from this specification.", + "expected_skill_not": "nemo-try-agent" + }, + { + "type": "negative-control", + "prompt": "Write an agent.yaml for a new code review agent.", + "expected_skill_not": "nemo-try-agent" } ] } diff --git a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py index e7338327833..e760f7b379f 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py +++ b/packages/nemo_platform_ext/tests/cli/commands/skills/test_skill_content.py @@ -89,6 +89,19 @@ def test_each_skill_has_source_dir(self): assert skill.source_dir.is_dir() assert (skill.source_dir / "SKILL.md").exists() + def test_build_agent_templates_are_packaged(self): + skill = load_skills()["nemo-build-agent"] + assert skill.source_dir is not None + + templates_dir = skill.source_dir / "references" / "templates" + assert (templates_dir / "agent.yml").is_file() + assert (templates_dir / "eval-job.json").is_file() + + def test_model_selection_benchmark_cache_is_packaged(self): + skill = load_skills()["nemo-model-selection"] + assert skill.source_dir is not None + assert (skill.source_dir / "references" / "benchmark_cache.json").is_file() + def test_returns_new_dict_each_call(self): """Verify callers can't corrupt the cached data.""" skills1 = load_skills() diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/SKILL.md new file mode 100644 index 00000000000..2396195aec9 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/SKILL.md @@ -0,0 +1,321 @@ +--- +name: nemo-agent-config +description: Author and validate Platform-owned NeMo Agents agent.yaml files using the nemo-agents-spec-v1 format. Use when the user wants to create, edit, validate, or adapt an agent.yaml file, choose a supported harness, add instructions, skills, MCP servers, tools, environment, or telemetry. +triggers: + - write agent.yaml + - create agent.yaml + - edit agent.yaml + - validate agent.yaml + - configure a harness + - configure agent harness + - nemo-agents-spec-v1 + - platform agent config + - adapt agent.yaml + - convert NAT workflow YAML + - migrate NAT workflow + - convert agent.yml to agent.yaml + - NeMo agent.yaml config +not-for: + - nemo-build-agent (use for full spec-to-deployed-agent build flows) + - nemo-explore (use to design what the agent should do before writing config) + - nemo-spec (use to write AGENT-SPEC.md before implementation) + - nemo-model-selection (use when the user only wants model recommendation) + - generic YAML editing unrelated to NeMo Platform agents +compatibility: nemo-platform >= 0.1.0; writes or edits agents/-spec/agent.yaml; validates through nemo agents create; supports nemo-agents-spec-v1 configs; safe under sandbox. +maturity: active +license: Apache-2.0 +user-invocable: true +allowed-tools: [Read, Write, Edit, Bash] +--- + +# NeMo Platform agent config + +Create or edit the Platform-owned `agent.yaml` for a NeMo Agent. This skill +owns the machine-readable config shape for `nemo-agents-spec-v1`; `nemo-build-agent` +owns the full build/deploy/eval workflow. + +Use product-facing Platform language. Do not ask users to write raw Fabric SDK +configuration. Fabric is an implementation dependency behind the Platform-owned +agent config. + +## Storage model + +The local config lives next to the human-readable agent spec: + +```txt +agents/-spec/ + AGENT-SPEC.md + agent.yaml +``` + +The platform stores the parsed `agent.yaml` contents in the `Agent.config` +payload with: + +```yaml +config_format: nemo-agents-spec-v1 +``` + +The canonical remote config location is derivable from workspace and agent name: +`/-spec#agent.yaml`. Do not invent a separate ref field. + +## What you do + +1. Confirm the agent name and config path. Default to + `agents/-spec/agent.yaml`. +2. Select one supported harness: + - `codex` + - `hermes` + - `deepagents` + - `claude` +3. Invoke `nemo-model-selection` to select and compatibility-test the model for + the chosen harness. Configure `models.default` only after that skill returns + a verified provider and exact model name. Add a harness-local `model` + override only when that harness intentionally uses a different verified + provider, model, credential env var, or base URL. +4. Start from `references/templates/agent.yaml` unless the user is editing an + existing file. Write only the verified model fields returned in Step 3. +5. Remove every unselected harness block and every remaining placeholder. Keep + an additional harness only when the user explicitly requests it and its + model configuration has been separately verified. +6. Add system instructions under `instructions.system.content`. +7. Add optional skills, MCP servers, blocked tools, environment directories, and + telemetry using only fields in the template. +8. Keep all local file paths relative to the directory containing `agent.yaml`. +9. Validate by running `nemo agents create` against the config. Do not treat a + successful create as validation of an unused or untested harness. + +For the standard subprocess or default-image deployment path, keep +`skills.paths` empty. Registration validates relative skill paths against the +authoring directory, but these deployment paths currently materialize only +`agent.yaml` and do not stage the referenced directories. + +When non-empty `skills.paths` are required, each path must be relative to +`agent.yaml`, remain inside its packaging context, and contain `SKILL.md`. +Require the explicit image-packaging path before deployment: + +```bash +IMAGE_TAG="${AGENT_NAME}:local" +.venv/bin/nemo agents package \ + --agent "agents/$AGENT_NAME-spec/agent.yaml" \ + --tag "$IMAGE_TAG" +``` + +Deploy that image with `--mode docker --image "$IMAGE_TAG"`; publish it first +and use the published tag for Kubernetes. Do not use a subprocess or +default-image deployment for an agent with relative `skills.paths`. + +## Migrating from legacy NAT workflow YAML + +If the user has an existing NAT workflow YAML and wants the new Platform-owned +`agent.yaml` format, treat the migration as best-effort authoring. Do not +overwrite the original NAT YAML unless the user explicitly asks. + +Map only fields with a clear Platform equivalent: + +| NAT workflow concept | Platform `agent.yaml` target | +|---|---| +| LLM/provider/model block | `models.default` or a harness-local `model` | +| System prompt or workflow prompt | `instructions.system.content` | +| Workflow/tool loop choice | `default_harness` plus `harnesses..kind` | +| Tool/function references | `skills.paths`, `mcp.servers`, `tools.blocked`, or harness settings when clearly supported | +| Tracing or telemetry settings | `telemetry` | + +For a NAT `codex_agent` workflow, apply this mapping instead of copying the +workflow fields into `harnesses.codex.settings`: + +| NAT Codex field | Platform `agent.yaml` target | +|---|---| +| `working_directory` | `environment.workspace` | +| `sandbox_mode` | `harnesses.codex.settings.sandbox` | +| `approval_policy: never` | `harnesses.codex.settings.approval_mode: deny_all` when the installed descriptor declares it; otherwise omit | +| `relay_atof_output_dir` | `environment.artifacts`, `telemetry.output_dir`, and `telemetry.atof` | +| `skip_git_repo_check` | Omit; removed CLI-only setting | +| `timeout_seconds` | Omit unless the installed Codex adapter settings schema declares it | +| `max_output_chars` | Omit; no current Codex adapter setting | +| `prefer_chatgpt_auth` | Omit; authentication is selected by the Codex adapter and model configuration | + +Treat the installed Fabric adapter descriptor's `settings_schema` as +authoritative. Only place keys declared under its `properties` in +`harnesses..settings`; do not preserve an unsupported NAT setting merely +because it existed in the source workflow. Surface omitted behavior in the +migration summary. For Codex, map `never` to `deny_all` only when +`settings_schema.properties.approval_mode.enum` includes `deny_all`. + +If behavior does not map cleanly, say so directly and choose one: + +- Keep the agent on the NAT compatibility path. +- Preserve the original NAT YAML and create a partial `agent.yaml` starter for + manual completion. +- Mark it as requiring a custom adapter or a manual harness-specific migration. + +Never claim a mechanical one-to-one conversion for arbitrary NAT workflows. + +## Config shape + +Use this structure. Keep unknown fields out of the YAML; the Platform validator +rejects unsupported fields instead of passing arbitrary execution config through. + +```yaml +config_format: nemo-agents-spec-v1 +name: +description: + +instructions: + system: + content: + +default_harness: codex + +harnesses: + codex: + kind: codex + settings: + sandbox: workspace-write + reasoning_effort: high + +models: + default: + provider: + model: + api_key_env: + +skills: + paths: [] + +mcp: + servers: {} + +tools: + blocked: [] + +environment: + workspace: ./workspace + artifacts: ./artifacts + +telemetry: + enabled: false + provider: relay + output_dir: ./artifacts/relay + project: +``` + +### Harness overrides + +Use a harness-local model only when that harness should override the default. + +```yaml +harnesses: + hermes: + kind: hermes + model: + provider: + model: + api_key_env: + base_url: + temperature: 0.0 + settings: + max_tokens: 512 + reasoning_config: + effort: none +``` + +If `base_url` is needed, put it directly in the model block, not under +`settings`. + +Use `nemo-model-selection` for every harness. It must verify the exact model +against that harness's model contract before returning it: Responses for +`codex`, the selected provider's chat path for `hermes` and `deepagents`, and +the native Anthropic provider contract for `claude`. Do not route Claude +through Platform IGW. Do not write or finalize `agent.yaml` if model selection +cannot establish a compatible model. For Platform-routed models, availability, +provider metadata, config validation, and Fabric planning do not replace a +valid inference request through the required wire API. + +## Validate and register + +Before registering, validate the YAML shape with the Platform create path. +Immediately before running `nemo agents create`, show the command to the user, +ask for explicit confirmation, and wait for approval. + +```bash +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" +``` + +If validation fails, fix the named field in `agent.yaml` and retry. Do not +silence validation errors by moving unknown fields into `settings`. + +## Deploy and invoke + +After create succeeds, show the `nemo agents deploy` command to the user, ask +for explicit confirmation, and wait for approval before running it. + +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$AGENT_NAME-deployment" +``` + +The deploy command waits for `running` by default. After it succeeds, invoke the +same explicitly named deployment without another confirmation: + +```bash +.venv/bin/nemo agents invoke \ + --agent-deployment "$AGENT_NAME-deployment" \ + --input "" +``` + +For local one-shot validation without registering an Agent entity, use this +only when the selected model already has a directly usable provider endpoint +and credentials. Platform IGW normalization is applied by the registered +deployment path, not this local path: + +```bash +.venv/bin/nemo agents invoke \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" \ + --input "" +``` + +For a local persistent server, bind to loopback by default. Use an externally +accessible host only when the user explicitly asks to expose the server: + +```bash +.venv/bin/nemo agents run \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" \ + --host 127.0.0.1 \ + --port 8080 +``` + +## If validation fails + +| Symptom | Cause | Recovery | +|---|---|---| +| `root must be a YAML mapping` | Empty file or list/scalar at the root | Replace with the template shape | +| `extra fields not permitted` | Unknown Platform config field | Remove it or map it into a supported field | +| `default_harness must reference one of harnesses` | `default_harness` does not match a key under `harnesses` | Rename one side so they match | +| `Unsupported harness kind` | Harness kind is not supported by the Platform translator | Pick `codex`, `hermes`, `deepagents`, or `claude` | +| Local file path missing in deployment | Referenced prompts, skills, or assets were not staged | Keep paths relative and package the complete agent bundle into the deployed image | +| Adapter import or binary missing | Selected harness dependency is not installed in the runtime | Install the selected adapter/runtime dependency or choose a harness already available | + +## Hard rules + +- Keep `config_format: nemo-agents-spec-v1`. +- Keep paths relative to the `agent.yaml` directory. +- Put system instructions under `instructions.system.content`. +- Do not use `prompts` for the default path; top-level prompts are not translated yet. +- Do not create profile files. Profiles are not the Platform authoring contract. +- Do not expose Fabric SDK object names as user-authored YAML fields. +- Do not emit arbitrary adapter settings unless the selected harness documents them. + +## Gotchas + +- **Default model vs harness model.** A harness-local `model` always wins over + `models.default`. +- **Registration validates and normalizes.** `nemo agents create` is the + user-facing validation command. +- **`agent.yaml` is the implementation config, not the design spec.** + `AGENT-SPEC.md` explains what the agent should do; `agent.yaml` tells the + Platform how to run it. +- **NAT workflow YAML is a compatibility path.** If the user explicitly asks + for legacy NAT, route to `nemo-build-agent` and use its NAT template. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/references/templates/agent.yaml b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/references/templates/agent.yaml new file mode 100644 index 00000000000..473c25a5408 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-agent-config/references/templates/agent.yaml @@ -0,0 +1,69 @@ +config_format: nemo-agents-spec-v1 +name: replace-with-agent-name +description: Replace with a short agent description + +instructions: + system: + content: Replace with the agent's system instructions. + +default_harness: codex + +harnesses: + codex: + kind: codex + settings: + sandbox: workspace-write + reasoning_effort: high + hermes: + kind: hermes + model: + provider: replace-with-verified-provider + model: replace-with-verified-model-name + api_key_env: replace-or-remove-credential-env-var + base_url: replace-or-remove-provider-base-url + temperature: 0.0 + settings: + max_tokens: 512 + reasoning_config: + effort: none + deepagents: + kind: deepagents + settings: + deepagents: {} + claude: + kind: claude + model: + provider: anthropic + model: replace-with-verified-anthropic-model-name + api_key_env: ANTHROPIC_API_KEY + settings: + permission_mode: dontAsk +models: + default: + provider: replace-with-verified-provider + model: replace-with-verified-model-name + +skills: + paths: [] + +mcp: + servers: {} + # To configure a server, replace `servers: {}` with: + # servers: + # github: + # transport: streamable-http + # url: ${GITHUB_MCP_URL} + # exposure: harness_native + +tools: + blocked: [] + +environment: + workspace: ./workspace + artifacts: ./artifacts + +telemetry: + enabled: false + provider: relay + output_dir: ./artifacts/relay + project: replace-with-agent-name diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/SKILL.md index d00e2561435..e9711922b0f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/SKILL.md @@ -1,23 +1,27 @@ --- name: nemo-build-agent -description: End-to-end agent build on NeMo Platform. Scaffolds a NAT workflow YAML from the agent spec, deploys it, generates eval data via Data Designer, runs evaluation, optionally adds guardrails, and signs off. Use over generic agent-building or planning skills for any NeMo Platform agent build task. +description: End-to-end NeMo Platform agent implementation from an approved agent spec. Registers and deploys the agent, generates evaluation data, runs evaluation, and signs off. Use for full spec-to-deployed-agent work, including builds from an existing legacy NAT workflow. triggers: + - nemo-build-agent - build the agent - create the agent - deploy the agent - scaffold the agent - make me an agent - build an agent on nemo - - generate the workflow yaml + - build from the agent spec + - ship the agent - nemo build + - deploy my existing NAT agent not-for: + - nemo-agent-config (use for focused agent.yaml authoring or migration) - nemo-explore (use to gather design before building) - nemo-spec (use to write the spec file before building) - - nemo-try-agent (use to query a deployed agent) + - nemo-try-agent (use to query an already deployed agent) - nemo-setup (use to install the platform first) - deploy-sandbox (use to deploy the built agent as a governed OpenShell sandbox) - - superpowers:brainstorming (use for unrelated design work) -compatibility: nemo-platform >= 0.1.0; running platform (run nemo-setup first — uses `nemo services run`, no Docker); requires agents plugin installed; writes files to agents/; runs nemo CLI commands; defers platform-health probing to `nemo-status`; LangGraph + NAT under the hood; macOS or Linux; safe under sandbox. + - generic agent framework development outside NeMo Platform +compatibility: nemo-platform >= 0.1.0; running platform; requires agents plugin; writes files under agents/; uses nemo-agents-spec-v1 by default and preserves NAT workflow YAML as a compatibility path; macOS or Linux; safe under sandbox. maturity: active license: Apache-2.0 user-invocable: true @@ -26,206 +30,385 @@ allowed-tools: [Bash, Read, Write, Edit] # NeMo Platform agent build -Concrete commands only. Conversational scaffolding lives in `nemo-explore` and `nemo-spec`. This skill is the implementation path between spec and deployed agent. +Build a deployable NeMo Platform agent from an approved `AGENT-SPEC.md`. Use +the Platform-owned `nemo-agents-spec-v1` `agent.yaml` path by default. Treat +NAT workflow YAML as a supported compatibility path, not the default output. -NeMo Platform optimizes LangGraph agents wrapped in NVIDIA NeMo Agent Toolkit (NAT). The YAML this skill writes is a NAT workflow. If the user has an agent in another framework (CrewAI, AutoGen, plain LangChain, Pydantic AI), stop and tell them they need a NAT wrapper before this skill produces value. +Use `nemo-agent-config` for the machine-readable config shape. Do not expose +Fabric SDK object names or raw runtime configuration to the user. + +## Select the config path + +Choose the config path before pre-flight. Set shared names for either path: + +```bash +AGENT_NAME= +DEPLOYMENT_NAME="${AGENT_NAME}-deployment" +``` + +If the user supplies an existing NAT workflow YAML, ask whether they want to +deploy it unchanged or migrate it best-effort to `nemo-agents-spec-v1` with +`nemo-agent-config`. For an unchanged NAT-only run, preserve the original file +and also set: + +```bash +NAT_WORKFLOW_PATH= +``` ## Pre-flight -1. Confirm the platform is up. Run `nemo-status`'s platform probe (canonical lsof + curl check) and stop if it reports `PLATFORM_DOWN` or `PLATFORM_WEDGED`; route to `nemo-setup` and return when it clears. Do not reimplement the probe here — `nemo-status` owns it so changes (new components, new ports) land in one place. +1. Run the platform probe owned by `nemo-status`. If it reports + `PLATFORM_DOWN` or `PLATFORM_WEDGED`, route to `nemo-setup` and stop. +2. Confirm the agents plugin is loaded: -2. Confirm a spec exists at `agents/$AGENT_NAME-spec/AGENT-SPEC.md`. If missing, call `nemo-explore` then `nemo-spec`, then return. -3. Confirm the agents plugin is loaded: `.venv/bin/nemo agents --help 2>&1 | grep -q "create"`. If the plugin is missing, report that explicitly; the user has not installed `plugins/nemo-agents` and the build cannot proceed. -4. Read the spec. Extract: name, categories, tools, model, constraints, success criteria. -5. Confirm the canonical spec fileset exists. By convention the spec lives at `/-spec#AGENT-SPEC.md` — there is no ref to thread through, just a one-shot presence check: + ```bash + .venv/bin/nemo agents --help 2>&1 | grep -q "create" + ``` + +3. Check for existing Agent entities and deployments. Ask whether to reuse or + replace them. Follow the lifecycle branches below before create or deploy. +4. For an unchanged NAT-only run, confirm `$NAT_WORKFLOW_PATH` exists and read + it before continuing. Do not require `AGENT-SPEC.md` or a spec fileset. +5. For the default Platform-owned path, confirm + `agents/$AGENT_NAME-spec/AGENT-SPEC.md` exists. If it does not, route through + `nemo-explore` and `nemo-spec` first. +6. Read the spec and extract the agent name, instructions, capabilities, + model requirements, tools, constraints, and success criteria. +7. Confirm the canonical spec fileset exists: ```bash - nemo files filesets get "${AGENT_NAME}-spec" --workspace "${WORKSPACE:-default}" >/dev/null 2>&1 \ + .venv/bin/nemo files filesets get "${AGENT_NAME}-spec" \ + --workspace "${WORKSPACE:-default}" >/dev/null 2>&1 \ && echo "spec_fileset_ok" \ - || { echo "spec_fileset_missing — run nemo-spec to upload before continuing"; exit 1; } + || { echo "spec_fileset_missing - run nemo-spec first"; exit 1; } ``` - If the fileset is missing, route back to `nemo-spec` and return when the upload succeeds. -6. Check for an existing deployment: `.venv/bin/nemo agents deployments list 2>/dev/null | grep -q "$AGENT_NAME"`. If the agent is already deployed, ask the user whether to skip (idempotent path) or redeploy. +Steps 5 through 7 apply only to the default Platform-owned path or an explicit +NAT migration. + +### Existing-resource lifecycle -## Step 1: Scaffold and deploy +- **Reuse:** Do not run `agents create` for an existing Agent. If a deployment + already exists, set `DEPLOYMENT_NAME` to its name, do not run `agents deploy`, + and continue to the smoke test. If only the Agent exists, skip create and run + only the deploy command in Step 1. +- **Replace:** Show each destructive command and require explicit confirmation + immediately before running it. Use `--yes` only after that confirmation. If + the resource does not exist, skip its command. -Write `agents/$AGENT_NAME.yml` from `references/templates/agent.yml`, substituting model, tools, system prompt, and the spec's constraints. The system prompt MUST contain `{tools}` and `{tool_names}` placeholders. +For a confirmed replacement, undeploy first: ```bash -AGENT_NAME= # set once; reused throughout this skill -.venv/bin/nemo agents delete "$AGENT_NAME" 2>/dev/null || true -.venv/bin/nemo agents create --name "$AGENT_NAME" \ - --agent-config "agents/$AGENT_NAME.yml" -.venv/bin/nemo agents deploy --agent "$AGENT_NAME" -.venv/bin/nemo agents deployments wait --agent "$AGENT_NAME" +.venv/bin/nemo agents undeploy "$DEPLOYMENT_NAME" --yes ``` -Show the YAML to the user. Stop. Ask: "Config and deployment look right? Adjust system prompt, model, or tools before continuing?" - -Verification: confirm the deployment reached ready state. +Wait until this command reports that the deployment is absent before +continuing: ```bash -.venv/bin/nemo agents deployments list | grep "$AGENT_NAME" | grep -qi "ready" && echo "DEPLOY_READY" || echo "DEPLOY_NOT_READY" +.venv/bin/nemo agents deployments get "$DEPLOYMENT_NAME" ``` -If `DEPLOY_NOT_READY`: jump to the recovery table at the bottom. +Then show the Agent deletion command and require explicit confirmation before +running it: -> **Governed sandbox deployment.** To deploy this agent as a policy-governed OpenShell sandbox instead of the default executor (Landlock filesystem isolation plus default-deny network egress, so its model traffic can only reach the platform), use the `deploy-sandbox` skill once the image is built. It swaps this step's deploy path for the `openshell-local` executor and an auto-generated SandboxPolicy. - -## Step 2: Try the agent +```bash +.venv/bin/nemo agents delete "$AGENT_NAME" --yes +``` -Invoke with one question from each category in the spec. +Verify this command reports that the Agent is absent before running the create +and deploy commands in Step 1: ```bash -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" +.venv/bin/nemo agents get "$AGENT_NAME" ``` -Display each verbatim response. +## Prepare the selected config -Stop. Ask if they want to proceed to evaluation or adjust the agent first. +### Default: Platform-owned `agent.yaml` -## Step 3: Identify and generate the synthetic data this agent needs +For a new build, invoke `nemo-agent-config` and create: -Data Designer (DD) is the platform's synthetic-data tool. It can produce any of: +```txt +agents/-spec/ + AGENT-SPEC.md + agent.yaml +``` -- **Knowledge base or RAG corpus.** Q&A pairs, doc snippets, or policy entries the agent retrieves from at runtime. -- **Evaluation dataset.** Input prompts plus ground-truth or judge-rubric outputs. Used by Step 4 evaluation. -- **Benchmark dataset.** A larger, diversity-weighted eval set for ongoing regression testing. -- **Persona-grounded inputs.** Adversarial or edge-case inputs simulating specific user types. -- **Training data.** When fine-tuning lands. -- **Other synthetic datasets** the user asks for. +Delegate authoring to `nemo-agent-config`. It selects the supported harness and +uses `nemo-model-selection` to verify the exact model against that harness's +model contract before writing the model block. Translate the approved spec into +system instructions, skills, MCP servers, tools, environment paths, and +telemetry. Keep every local path relative to the directory containing +`agent.yaml`. -**Do NOT hand-author any of these, even if your model is capable enough to write them inline.** Three reasons, all load-bearing: +Before registration, inspect `skills.paths`: -1. **Reproducibility.** DD configs regenerate identical datasets when seeded. Hand-authored sets are unreproducible — the moment the spec changes, you cannot regenerate matching eval data without re-doing the authoring by hand. -2. **Diversity.** DD samples across categorical axes the user (or skill) declares. Hand-authored sets cluster around whatever the author thought of, which under-tests the long tail. -3. **Capability transfer.** A less capable coding agent running this skill later cannot hand-author good eval questions. DD-generated data is independent of the coding agent's capability — the same DD config produces equivalent data whether driven by Sonnet or a 7B model. +- If it is empty, continue with the normal deployment lifecycle below. +- If it is non-empty, verify every relative directory is inside the agent + packaging context and contains `SKILL.md`. Package the complete bundle before + deployment: -### Procedure + ```bash + IMAGE_TAG="${AGENT_NAME}:local" + .venv/bin/nemo agents package \ + --agent "agents/$AGENT_NAME-spec/agent.yaml" \ + --tag "$IMAGE_TAG" + ``` -1. **Enumerate.** Read `agents/$AGENT_NAME-spec/AGENT-SPEC.md`. Surface to the user the full list of synthetic-data purposes this agent plausibly needs, based on the spec. Do not prescribe a count or shortlist; let the user pick freely from the catalog above (or add purposes you haven't anticipated). + Use the packaged-image deploy command below. Do not use subprocess deployment + or the default container image because those paths materialize only + `agent.yaml` and do not stage relative skill directories. -2. **Wait for picks.** Do not generate any DD config until the user has explicitly named which purposes they want. If the user says "you decide," default to: a knowledge base if the spec describes retrievable content, an eval dataset always, persona-grounded adversarial inputs if the spec lists safety constraints. Announce the defaults you chose. +### Compatibility: existing NAT workflow YAML -3. **Hand off per purpose.** For each chosen purpose, invoke the `data-designer` skill once. Pass it: the agent name, the purpose label (KB / eval / benchmark / persona / other), and the spec path. The DD skill is responsible for the config shape — this skill does not duplicate that logic. +If the user selected migration, preserve the original YAML. If a workflow, +tool, or custom Python component has no supported harness equivalent, keep the +NAT path or identify the need for a custom adapter. Never claim arbitrary NAT +workflows convert mechanically. -4. **Ground every config in the spec.** Each DD config MUST reference `agents/$AGENT_NAME-spec/AGENT-SPEC.md` for product context, categories, audience, and constraints. Do not redefine these inline. If the generated config inlines context, edit it to read from the spec instead — drift between agent definition and synthetic data is a reproducibility failure. +Use `references/templates/agent.yml` only when the user explicitly chooses the +legacy NAT path or needs a new NAT compatibility workflow. -5. **Run each config.** Use `.venv/bin/python agents/$AGENT_NAME..py` (or the CLI invocation once `nemo data-designer preview-local` lands in a release > 2.1.0). For larger jobs, submit via `nemo data-designer jobs create`. +## Step 1: Register and deploy -6. **Verify before Step 4.** Confirm at least one fileset in `nemo files filesets list` matching `$AGENT_NAME-eval-*` exists. Step 4 refuses to proceed without it. +For the default path: -Show 3 to 5 sample records per purpose, grouped by category. Stop. Ask: "Do these samples look realistic for each purpose? Adjust categories, prompts, or regenerate?" +For each operation retained by the selected lifecycle branch, follow the +`nemo-agent-config` confirmation requirement. Show the create command and ask +for explicit confirmation immediately before running it: -### Anti-patterns to refuse +```bash +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "agents/$AGENT_NAME-spec/agent.yaml" +``` -- Writing eval questions inline because "they're simple" — refuse, route to DD. -- Generating a single combined dataset that conflates KB and eval — refuse, separate configs per purpose. -- Skipping DD entirely because the user said "just do it" — refuse, DD is required infrastructure, not an optional tool. -- Inlining product context in the DD config instead of referencing the spec — refuse, fix the config to read from the spec. +After create succeeds, show the deploy command and ask for explicit +confirmation immediately before running it: -## Step 3.5: Wire generated data into the agent +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$DEPLOYMENT_NAME" +``` + +If `skills.paths` is non-empty, show this command instead and ask for explicit +confirmation immediately before running it: -If Step 3 produced any synthetic data the agent is supposed to *use at runtime* (a knowledge base, a RAG corpus, a retrieval index), the agent must be wired to actually consume it. Generating the data and never connecting it is a silent product failure: the agent hallucinates against missing context while the real data sits unused next to it. +```bash +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$DEPLOYMENT_NAME" \ + --mode docker \ + --image "$IMAGE_TAG" +``` -NeMo Agent Toolkit (NAT) has first-class retrieval support: +For Kubernetes, publish the packaged image and replace `docker` and +`$IMAGE_TAG` with `k8s` and the published image tag. -- `nvidia-nat-rag` ships a `RAGRetriever` client that loads filesets or local parquet/JSONL files. -- `nvidia-nat-langchain` bridges any LangChain retriever (FAISS, Chroma, Milvus, Pinecone, OpenSearch, NeMo Retriever) into a NAT tool. -- Worked example: `examples/RAG/simple_rag/` in the NAT repo. +These commands assume the Agent and deployment are absent. If pre-flight found +existing resources, complete the selected lifecycle branch before running them. -### Retriever wiring procedure +`nemo agents deploy` waits for `running` by default. If the user passed +`--no-wait`, wait explicitly: -1. **Detect.** Scan `agents/$AGENT_NAME-spec/AGENT-SPEC.md` for tools whose names suggest retrieval: `*_search`, `*_lookup`, `query_*`, `find_*`, `rag_*`, or any tool the user described in `nemo-explore` as "the agent looks things up in X." Cross-reference against the filesets Step 3 produced. +```bash +.venv/bin/nemo agents deployments wait "$DEPLOYMENT_NAME" +``` -2. **Pair.** For each retrieval-style tool, identify which Step 3 fileset feeds it. If the spec lists `billing_kb_search` and Step 3 produced `billing-support-kb`, pair them. If a tool has no matching fileset, surface the gap to the user: "Your spec lists `billing_kb_search` but no KB fileset was generated. Generate one now (route to Step 3) or drop the tool from the agent?" +Show `agent.yaml` and the deployment result. Stop and ask whether the config, +model, harness, and instructions look right before continuing. -3. **Wire.** Update `agents/$AGENT_NAME.yml` to add a NAT retriever per pair, pointing at the fileset. The retriever appears as a tool in the agent's `tools:` list under the matching name. Use the simple_rag example as the template shape. +For an unchanged NAT workflow, registration defaults configs without +`config_format` to `nat-workflow-v1`: -4. **Redeploy.** Step 1 already deployed the agent without the retrievers wired (because the data didn't exist yet). Redeploy now so the agent's tool list includes the retrievers: +Apply the same immediate confirmation requirement. Show the create command and +wait for explicit confirmation before running it: ```bash -.venv/bin/nemo agents undeploy --agent $AGENT_NAME -.venv/bin/nemo agents create --name $AGENT_NAME \ - --agent-config agents/$AGENT_NAME.yml -.venv/bin/nemo agents deploy --agent $AGENT_NAME -.venv/bin/nemo agents deployments wait --agent $AGENT_NAME +.venv/bin/nemo agents create \ + --name "$AGENT_NAME" \ + --agent-config "$NAT_WORKFLOW_PATH" ``` -5. **Verify the wire.** Invoke the agent with a question that requires the KB and inspect the tool-call trace. The retriever tool MUST be called for any KB-grounded question. If the agent answers from system-prompt-policy text alone without calling the retriever, the tool wiring is broken — debug before declaring success. +After create succeeds, show the deploy command and wait for explicit +confirmation before running it: ```bash -.venv/bin/nemo agents invoke --agent $AGENT_NAME --input "" --output-format json | python3 -c "import sys,json; d=json.loads(sys.stdin.read()); print('tools called:', d.get('tool_calls') or 'NONE — wiring broken')" +.venv/bin/nemo agents deploy \ + --agent "$AGENT_NAME" \ + --name "$DEPLOYMENT_NAME" ``` -### Refuse-list +## Step 2: Try the deployed agent -- Skipping this step when the spec lists a retrieval-style tool. Generating data the agent can't reach is theater. -- Wiring the retriever but not redeploying. The fix doesn't land in a running agent until redeploy. -- Declaring success on the redeploy without confirming the tool was actually called for a KB question. A wired-but-unused retriever is indistinguishable from a missing one. +For the default path, invoke one question from each category in the spec. For +an unchanged NAT-only run without a spec, use representative questions from the +workflow and the user's stated requirements: + +```bash +.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "" +.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "" +.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "" +``` + +Display each response verbatim. Stop and ask whether to adjust the agent or +continue to evaluation. + +Before Step 3, branch explicitly: + +1. For an unchanged NAT-only run without `AGENT-SPEC.md`, stop after the smoke + test. Do not execute Steps 3–5 and do not require an evaluation fileset. +2. Continue into the spec-driven purpose selection and Data Designer flow only + when the user requests it and `agents/$AGENT_NAME-spec/AGENT-SPEC.md` exists. + If the user requests evaluation but the spec is absent, create and confirm + the spec first; do not continue to Step 3 yet. + +## Step 3: Generate synthetic data + +Use Data Designer for every synthetic dataset. Do not hand-author evaluation, +knowledge-base, benchmark, persona, or training data. + +1. Always select evaluation as a required data purpose. Read + `agents/$AGENT_NAME-spec/AGENT-SPEC.md` and list any additional plausible + purposes: knowledge/RAG corpus, benchmark, personas/adversarial inputs, + training, or another user-requested purpose. +2. Wait for the user to choose any additional purposes. Evaluation cannot be + omitted. If they delegate the decision, add a knowledge base when the spec + requires retrieval and adversarial personas when it contains safety + constraints. +3. Invoke `data-designer` once per selected purpose, passing the agent name, + purpose, and spec path. +4. Require every generated config to read product context from + `AGENT-SPEC.md`; do not duplicate that context inline. +5. Run each generated config. For evaluation, validate the generated records, + verify the resulting fileset exists, and record its exact dataset reference + as `EVAL_DATASET_REF`. +6. Show 3 to 5 sample records per purpose and ask for approval. + +A validated `$AGENT_NAME-eval-*` fileset and its exact `EVAL_DATASET_REF` must +exist before evaluation proceeds. + +## Step 3.5: Connect runtime data + +If the generated data must be available during invocation, connect it through +the selected harness's supported skills, MCP, or tool configuration. Update +`agents/$AGENT_NAME-spec/agent.yaml` through `nemo-agent-config`, then follow the +confirmed replacement branch before creating and deploying the Agent again. + +Do not invent a generic retriever field. If the selected harness cannot consume +the required data, surface that limitation and choose a supported integration, +the NAT compatibility path, or a custom adapter. + +For a legacy NAT workflow, NAT-specific retrievers may be wired into its +`functions` and `workflow` blocks using the matching NAT RAG integration. + +After the replacement deployment, invoke a question that requires the data and +verify the expected tool or retrieval path was actually used. ## Step 4: Evaluate +Select the actual Platform model reference as `EVAL_MODEL`. Create +`agents/$AGENT_NAME.eval-job.json` from `references/templates/eval-job.json` and +replace every placeholder. Its `model` must equal `EVAL_MODEL`, and its +`dataset` must equal the recorded `EVAL_DATASET_REF` from Step 3. + +Validate the rendered file before creating the benchmark job: + +```bash +.venv/bin/python -m json.tool "agents/$AGENT_NAME.eval-job.json" >/dev/null +if grep -Eq '<[^>]+>' "agents/$AGENT_NAME.eval-job.json"; then + echo "eval job still contains template placeholders" >&2 + exit 1 +fi +``` + +Also read the validated payload back and confirm its `model` and `dataset` +values match `EVAL_MODEL` and `EVAL_DATASET_REF`. Do not invoke +`benchmark-jobs create` if JSON validation, model validation, dataset +validation, or fileset validation fails. + +After all validation succeeds: + ```bash .venv/bin/nemo evaluation benchmarks list -.venv/bin/nemo evaluation benchmark-jobs create $AGENT_NAME-eval \ - --input-file agents/$AGENT_NAME.eval-job.json +.venv/bin/nemo evaluation benchmark-jobs create "$AGENT_NAME-eval" \ + --input-file "agents/$AGENT_NAME.eval-job.json" ``` -Template for the eval-job JSON in `references/templates/eval-job.json`. Poll job status: +Poll until the job reaches `completed` or `failed`, then download aggregate +scores. Show the score table and compare it with the success bar in +`AGENT-SPEC.md`. ```bash for i in $(seq 1 24); do - status=$(.venv/bin/nemo evaluation benchmark-jobs get-status $AGENT_NAME-eval 2>/dev/null) + status=$(.venv/bin/nemo evaluation benchmark-jobs get-status "$AGENT_NAME-eval" 2>/dev/null) echo "$status" echo "$status" | grep -qE "completed|failed" && break sleep 10 done -.venv/bin/nemo evaluation benchmark-jobs results aggregate-scores download $AGENT_NAME-eval -``` -Verification: confirm the job reached `completed`, not `failed`. Display the score table. Stop. Ask if scores meet the bar from the spec. +.venv/bin/nemo evaluation benchmark-jobs results aggregate-scores download \ + "$AGENT_NAME-eval" +``` ## Step 5: Guardrails (optional) -If the spec lists constraints, add a content-safety intercept to the YAML (see `references/templates/agent-with-guardrails.yml` if present, or write inline) and redeploy: +For `nemo-agents-spec-v1`, `AgentConfig` has no guardrail field and the current +skills do not define a supported composition between an Agent and an IGW +guardrailed VirtualModel. Do not add guardrail fields to `agent.yaml` or claim +that guardrails are attached. If the spec requires guardrails, report this as an +unmet requirement and stop before sign-off. `nemo-guardrails` may be used to +configure IGW VirtualModel middleware as a separate workflow, but do not treat +it as integrated with the Agent until its model routing has been explicitly +configured and validated. -```bash -.venv/bin/nemo agents undeploy --agent $AGENT_NAME -.venv/bin/nemo agents create --name $AGENT_NAME \ - --agent-config agents/$AGENT_NAME.yml -.venv/bin/nemo agents deploy --agent $AGENT_NAME -.venv/bin/nemo agents deployments wait --agent $AGENT_NAME -``` +For a legacy NAT workflow, keep the NAT compatibility behavior: add supported +guardrail `intercepts` to the NAT workflow YAML, then follow the confirmed +replacement branch before creating and deploying it again. Never add NAT +`intercepts` to a `nemo-agents-spec-v1` config. -Test with one adversarial and one legitimate prompt. Stop and report. +For the NAT path, test one adversarial prompt and one legitimate prompt. Report +both responses and do not continue to sign-off until the expected policy is +enforced without blocking the legitimate request. ## Step 6: Sign off -Run the success-criteria question from the spec through `nemo agents invoke` once more and print the verbatim output. That output is the formal sign-off. +Invoke the success-criteria prompt from the spec against +`$DEPLOYMENT_NAME`. Print the verbatim response as the formal sign-off. Do not +claim success until the deployment is `running`, evaluation has completed, and +the sign-off returns an actual model response. ## If verification fails | Symptom | Cause | Recovery | -| --- | --- | --- | -| `agents plugin unavailable` | `plugins/nemo-agents` not installed | Re-run the install loop from `nemo-setup` Step 3 for that package only | -| `DEPLOY_NOT_READY` after wait | Container startup error or YAML rejected | Run `.venv/bin/nemo agents deployments get $AGENT_NAME`; check status detail and logs | -| YAML rejected with `extra fields` | Top-level keys beyond `functions`, `llms`, `workflow`, `intercepts`, `middleware` | Strip extras from the YAML; only those five top-level keys are valid | -| Empty agent response | `{tools}` and `{tool_names}` missing from system prompt | Add both placeholders; redeploy | -| Eval job `failed` | Dataset path or model id wrong | Run `.venv/bin/nemo evaluation benchmark-jobs get $AGENT_NAME-eval` for the error string | -| Eval times out at 4 minutes | Long-running benchmark | Extend the poll loop budget; do not declare success on a timed-out job | - -If none of these apply, tail recent service logs (`.venv/bin/nemo services logs -n 100`) and surface the last error to the user. Do not claim the build succeeded until the success-criteria sign-off in Step 6 prints an actual model response. - -## Gotchas - -- **NAT workflow YAML required keys.** Top level must be `functions`, `llms`, `workflow`. Optional: `intercepts`, `middleware`. Extra top-level keys (`name`, `description`, `model`, `tools`, `system_prompt`) cause NAT to reject the config. -- **`{tools}` and `{tool_names}` are mandatory in the system prompt.** Without them the agent crashes on startup with no useful error. -- **Two model name formats coexist.** Entity-name with hyphens for NAT YAML and `nemo chat` and `nemo agents`. API-Catalog format with slashes for Data Designer. Mixing them causes silent failures or 404s. -- **`agents delete` is positional.** `nemo agents delete $AGENT_NAME`, not `--name`. Other commands take `--agent $AGENT_NAME`. -- **Data Designer uses Python config files.** Pass the `.py` file to the CLI when `preview-local` is available; otherwise run via the venv Python as shown above. -- **Guardrails are NAT intercepts, not a separate service.** They go in the same YAML under `intercepts:`. There is no `nemo guardrails create` step in the agent build. -- **Framework constraint.** Only LangGraph-in-NAT agents work end-to-end today. +|---|---|---| +| Agents plugin unavailable | `plugins/nemo-agents` is not installed | Route to `nemo-setup` | +| Config validation fails | Config does not match its declared format | Use `nemo-agent-config` for `nemo-agents-spec-v1`; use NAT schema rules only for NAT YAML | +| Deployment reaches `failed` | Runtime, adapter, image, or config startup failure | Run `.venv/bin/nemo agents deployments get "$DEPLOYMENT_NAME"` and `.venv/bin/nemo agents logs "$DEPLOYMENT_NAME"` | +| Referenced file is missing | Path is outside or absent from the staged agent directory | Keep paths relative to the config and package the complete agent bundle into the deployed image | +| Adapter or binary is missing | Selected harness dependency is not installed | Install the matching adapter/runtime package or select an available harness | +| Empty response | Runtime invocation failed or the selected configuration is incomplete | Inspect deployment logs and the returned structured error | +| Eval job fails | Dataset reference or model ID is invalid | Get the benchmark job details and correct the named input | + +## Hard rules + +- Default new builds to `agents/$AGENT_NAME-spec/agent.yaml` with + `config_format: nemo-agents-spec-v1`. +- Keep `AGENT-SPEC.md` as the human-readable design and `agent.yaml` as the + machine-readable implementation config. +- Preserve legacy NAT YAML unless the user explicitly requests migration. +- Do not mix NAT-only keys such as `functions`, `llms`, `workflow`, or + `intercepts` into `nemo-agents-spec-v1`. +- Do not put Platform `agent.yaml` fields into NAT workflow YAML. +- Use a named deployment and invoke it with `--agent-deployment`. +- Keep local artifact paths relative to the config directory. +- After changing persisted Agent config, use the confirmed replacement branch + before creating and deploying it again. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/agent.yml b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/agent.yml new file mode 100644 index 00000000000..3e1a41c98ac --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/agent.yml @@ -0,0 +1,48 @@ +# NAT workflow YAML for a NeMo Platform agent. +# Required top-level keys: functions, llms, workflow. +# Optional: intercepts, middleware. +# Anything else at top level causes NAT to reject the config. + +functions: + clock: + _type: current_datetime + +llms: + llm: + _type: openai + api_key: not-used + model_name: nvidia-llama-3-3-nemotron-super-49b-v1 + temperature: 0.0 + max_tokens: 1024 + +general: + telemetry: + tracing: + nemo_files_trace: + _type: nemo_files + # workspace and agent_name are injected at deploy time + +workflow: + _type: react_agent + tool_names: [clock] + llm_name: llm + verbose: false + parse_agent_response_max_retries: 3 + system_prompt: | + + + Your job: . + + You handle these categories: + - + - + - + + Constraints: + - + - + + You have access to the following tools: + {tools} + + Tool names: {tool_names} diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/eval-job.json b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/eval-job.json new file mode 100644 index 00000000000..7c79eda6082 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-build-agent/references/templates/eval-job.json @@ -0,0 +1,7 @@ +{ + "_comment": "Substitute , , and the dataset reference. The dataset is the artifact path produced by `nemo data-designer preview-local --save-results`. Workspace prefix `default/` is the platform default.", + "name": "-eval", + "benchmark": "default/agent-quality", + "model": "default/nim-llm/", + "dataset": "default/agents/.eval" +} diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-explore/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-explore/SKILL.md index dd70e171341..6943f3f3c74 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-explore/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-explore/SKILL.md @@ -60,8 +60,8 @@ requirements: handoff to `nemo-spec` is blocked until both are resolved. | 3 | Scope | yes | Audience, 3-6 task categories, expected in-scope work, and explicit out-of-scope work/non-goals. | | 4 | Tools | yes | Tools, APIs, and knowledge sources the agent can use, or "Prompt-only." Group related helpers by capability or source. Capture only behaviorally important purpose, credentials/scopes, side effects, freshness, and expected failures. | | 5 | Model | yes | Mode (cloud vs local NIM) + model family/size. Example: "cloud, Nemotron Super 49B." `nemo-build-agent` resolves to a specific model entity ID later. | -| 6 | Framework | **yes** | Temporary NeMo Platform compatibility field. Keep it binary: `langgraph-nat` or `needs-wrapper`; include source-framework context only when a wrapper is needed. | -| 7 | Harness | optional | Modern harness description: the extra-model layer around the agent — loop, tool dispatch, context/state, guardrails, observability, verification, and runtime. Summarize at useful granularity; omit low-level settings unless they affect behavior. Use `_(none)_` if unknown. | +| 6 | Framework | **yes** | Record execution compatibility as `supported-harness`, `nat-workflow`, or `needs-adapter`. Include the source framework when one exists. Do not assume a Python framework name alone guarantees lifecycle compatibility. | +| 7 | Harness | optional | Describe the selected or likely harness and the behavior it owns: loop, tool dispatch, context/state, guardrails, observability, verification, and runtime. Use `_(none)_` if selection should wait until config authoring. | | 8 | Behavior | yes | Behavioral rules and boundaries: constraints, refusal/escalation policy, tone, safety/compliance requirements, accepted limitations, and known non-goals. | | 9 | Success Criteria | yes | What good production behavior looks like, independent of current evals: mission-level outcomes, quality standards, escalation quality, accuracy expectations, latency/cost expectations if relevant, and examples of success. | | 10 | Evaluation Setup | yes | Current validation setup: how to run it, what datasets/checks it uses, what scorers/metrics measure, pass/fail thresholds, and known coverage gaps relative to the success criteria. If no eval suite exists, say so explicitly. | @@ -88,16 +88,20 @@ Time-box this to ~5 minutes of tool use. Read first, ask second. Greenfield projects will turn up nothing here, which is fine — move to step 2 and ask the user the full set of unfilled fields. -1. **Find agent entry points.** Look for NAT workflow YAMLs, LangGraph - builders, system prompts, tool definitions: +1. **Find agent definitions and entry points.** Look for Platform + `agent.yaml`, NAT workflow YAMLs, supported harness configuration, Python + agent builders, system prompts, skills, and tool definitions: ```bash + find . -maxdepth 5 -type f -name "agent.yaml" 2>/dev/null find . -maxdepth 4 -type f \( -name "*.workflow.yaml" -o -name "*.workflow.yml" \) 2>/dev/null find . -maxdepth 4 -type d -name "agents" 2>/dev/null ``` - Then use `Glob` / `Grep` to find `langgraph`, `StateGraph`, - `create_react_agent`, `system_prompt`, and tool definitions. + Then use `Glob` / `Grep` to find `nemo-agents-spec-v1`, + `default_harness`, `codex`, `hermes`, `deepagents`, `claude`, `langgraph`, + `StateGraph`, `create_react_agent`, `system_prompt`, skills, MCP servers, + and tool definitions. 2. **Find design context.** Look for `README.md`, `AGENTS.md`, product/design/planning docs, launch notes, and anything in `docs/`. Read @@ -132,14 +136,18 @@ the user the full set of unfilled fields. Group low-level helpers when they share credentials, side effects, freshness, and failure modes. - **Model** — model id strings in workflow YAML, env vars, config files. - - **Framework** — `langgraph` import + NAT workflow YAML → "LangGraph + - NAT." `crewai` / `autogen` / `pydantic_ai` imports → `needs-wrapper` - with source-framework context. Plain `langchain` without `langgraph` → - `needs-wrapper`. Do not turn this into a detailed implementation audit. - - **Harness** — `langgraph` imports, NAT workflow YAML, `crewai` / - `autogen` / `pydantic_ai` imports, service entrypoints, CLI commands, - Dockerfiles, notebooks, or deployment configs. Capture what exists - descriptively; platform-specific wrapper needs can go in notes. + - **Framework** — a validated `nemo-agents-spec-v1` config selecting a + supported harness → `supported-harness`; an existing NAT workflow YAML → + `nat-workflow`; an arbitrary Python entrypoint or unsupported framework + without a start/stop/invoke lifecycle contract → `needs-adapter`. Record + the source framework separately when known. Do not infer compatibility + solely from imports such as `langchain`, `langgraph`, `crewai`, + `autogen`, or `pydantic_ai`. + - **Harness** — infer from `default_harness` and `harnesses` in + `agent.yaml`, adapter configuration, NAT workflow YAML, service + entrypoints, CLI commands, Dockerfiles, notebooks, or deployment configs. + Capture behaviorally relevant capabilities, not low-level settings. If + there is no selection yet, leave it unresolved for `nemo-agent-config`. - **Behavior** — system prompt rules ("never give medical advice"), refusal/escalation policy, tone, accepted limitations, and non-goals. - **Success Criteria** — desired production outcomes, product goals, @@ -226,8 +234,8 @@ After the user's reply, apply the corrections and check the two hard preconditions: 1. **Role** is a concrete one-sentence answer (not "help with stuff"). -2. **Framework** is resolved (`langgraph-nat` or `needs-wrapper` with a - source-framework name). +2. **Framework** is resolved to `supported-harness`, `nat-workflow`, or + `needs-adapter`, with source-framework context when known. If either is still unresolved, ask for it in one final message and stop until the user provides it. Do not hand off with a hard requirement blank — @@ -273,10 +281,14 @@ to Filesets") and trigger it. into questions.** Spending the first five minutes reading earns the right to ask shorter, sharper questions. Asking something the codebase already answers loses trust immediately. -- **NeMo Platform optimizes LangGraph agents wrapped in NAT today.** Other - frameworks may still be valid AGENTSpec harnesses, but need a user-written - wrapper for the current NeMo build path. Record that as Harness notes; do - not make the standard schema a NeMo-specific capability gate. +- **Framework names do not prove execution compatibility.** A supported + harness must own the required lifecycle contract. Preserve an existing NAT + workflow as `nat-workflow`; classify an arbitrary Python entrypoint without + that contract as `needs-adapter` rather than promising direct execution. +- **Keep Platform terminology at the design boundary.** Record the desired + harness behavior and artifacts without exposing Fabric SDK types or asking + the user to design a raw runtime config. `nemo-agent-config` owns the + machine-readable Platform YAML after the spec is approved. - **Change Scope is a permissions list, not a wishlist.** It controls what the experimentalist agent will edit. Walk the defaults explicitly so the user knows what they're consenting to. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/SKILL.md index eddd7eac8e4..4903251a10f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-model-selection -description: Recommends an LLM for a NeMo Platform agent based on what the agent actually has to do, explained in plain English before any benchmark name appears. Use when the user is choosing a model for a new agent, asking which model to use, or unsure what to put in their spec or NAT workflow YAML. Invoked by nemo-explore at the model question; also runs standalone when the user starts mid-flow. +description: Recommends an LLM for a NeMo Platform agent based on what the agent actually has to do, explained in plain English before any benchmark name appears. Use when the user is choosing a model for a new agent, assessing a model they already selected, or deciding what belongs in AGENT-SPEC.md or Platform agent.yaml. Invoked by nemo-explore at the model question; also runs standalone when the user starts mid-flow. triggers: - which model should I use - what model is best for this @@ -22,7 +22,11 @@ allowed-tools: [Read, Bash] # NeMo Platform model selection -Recommends a model for a new agent (NIM or any other provider configured on the running platform). Plain-English first, benchmark numbers second, never the other way around. Output: one recommended model with a one-sentence reason, ready to drop into a spec or NAT workflow YAML. +Recommend a model for a new agent from NIM or another provider configured on +the running Platform. Explain the capability fit first and benchmark evidence +second. Return the model choice in a form suitable for `AGENT-SPEC.md` and the +Platform-owned `agent.yaml`. Preserve NAT model configuration only when the +user is explicitly maintaining a legacy NAT workflow. ## Pre-flight @@ -41,20 +45,30 @@ python scripts/refresh-benchmark-cache.py The cache (schema v6+) carries four things the rest of this skill reads: - `models[]` — editorial entries for a curated set of NIMs with `strong_at`, `watch_out_for`, `intent_hints`, `derived_from` lineage, and direct/inferred scores. - `upstream_index.bfcl_v4` and `upstream_index.arena_elo` — full BFCL and per-category Arena Elo tables for ~84 and ~360 models respectively. Use these to look up scores for ANY model name, not just the registered ones. -- `namespace_to_type[]` — namespace-prefix → NAT `_type` value mapping for the YAML emitter. +- `namespace_to_type[]` — namespace-prefix → NAT `_type` mapping used only for legacy NAT workflow output. - `name_decomposition_rules[]` — pattern→hint rules for synthesizing `intent_hints` when an unknown model name lands. -### 2. Fetch the live model list from the running platform +### 2. Fetch the live model list for a Platform-routed harness ```bash -curl -fsS http://localhost:8080/v1/models 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); names=[m.get('id') for m in d.get('data',[])]; print('\n'.join(n for n in names if n))" 2>/dev/null || echo "PLATFORM_UNREACHABLE" +nemo models list --all-pages --output-format json 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print('\n'.join(m['name'] for m in d.get('data', []) if m.get('name')))" 2>/dev/null || echo "PLATFORM_UNREACHABLE" ``` Interpretation: -- **List of model ids returned** → these are the candidates the user can actually pick from. Carry them through to Step 1+. -- `PLATFORM_UNREACHABLE` → platform isn't up. Fall back gracefully: tell the user "I can't reach the local platform, so I'll recommend from the curated NIM set in the cache instead of your actual available models. Start the platform with `nemo services run` if you want recommendations grounded in what's deployed." +- **Model names returned** → these are the candidates the user can actually pick from. Carry their exact names through to Step 1+; the JSON `id` value is a Platform entity id and must not be written to `agent.yaml` as the inference model identifier. +- `PLATFORM_UNREACHABLE` → platform isn't up. Recommendations may continue + from the curated cache, but a model for Platform `agent.yaml` cannot be + finalized until the live Platform model list and harness-specific inference + route can be checked. -The `/v1/models` response is OpenAI-shaped (`{data: [{id, ...}, ...]}`); the parse above extracts the `id` field per entry. If the platform's response shape differs, adjust the parse but keep the failure mode (graceful fallback to closed registry, never silently steer the user). +Use the `nemo` CLI rather than constructing a Platform URL or calling +`/v1/models` directly. The CLI resolves `NEMO_BASE_URL`, `NMP_BASE_URL`, the +active CLI context, authentication, and workspace consistently with subsequent +agent commands. Do not hardcode `localhost`, `127.0.0.1`, or port `8080`. + +This list is authoritative only for models routed through Platform. For a +native-provider harness such as `claude`, use the configured provider's native +model catalog and validation tooling instead. ## Step 0 — Pick the conversation direction @@ -105,19 +119,79 @@ Do not propose a model before all three answers are in. Push back on "you decide ## Step 1.5 — Build the candidate set and pick a presentation pattern +Identify the selected harness and whether it uses a Platform-routed or native +provider path before building candidates. Read it from the source config or +conversation; ask if it is still unknown. + ### Building candidates The candidate set is what the user can actually pick from. It comes from three joins: -1. **Start with the pre-flight model list from `/v1/models`** (or, if `PLATFORM_UNREACHABLE`, fall back to the cache's `models[]` editorial registry). -2. **For each available model id, look up evidence** in this order: +1. **Start with the correct live catalog.** Use the pre-flight Platform model + list for Platform-routed models. Use the configured provider's native model + catalog for native-provider harnesses. If the required catalog is + unreachable, the cache may support a conversational recommendation, but do + not finalize a Platform `agent.yaml` model block. Availability alone does + not establish compatibility with an agent harness. +2. **For each available model name, look up evidence** in this order: - Token-match against the editorial `models[]` entries → if hit, use the full editorial record (lineage, intent_hints, direct + inferred scores) - If no editorial match, token-match against `upstream_index.bfcl_v4` keys → if hit, use that BFCL score with `source: "direct_external"` - Same for `upstream_index.arena_elo` for per-category Elo - If neither editorial nor upstream matches, synthesize `intent_hints` by walking `name_decomposition_rules[]` and collecting every hint whose `pattern` token appears in the decomposed model id. Mark evidence as `source: "name_only"`. 3. **Rank candidates by the user's profile** — primary capability axis determines which score field dominates. -When `/v1/models` was unreachable, also tell the user the rest of this flow is operating on the curated NIM set, not their actual deployment. +When the live Platform model list was unreachable, also tell the user the rest +of this flow is operating on the curated NIM set, not their actual deployment. + +### Verify harness compatibility before config handoff + +When the output targets Platform `agent.yaml`, rank a short candidate list from +the appropriate live catalog and the evidence above, then verify candidates +against the selected harness's actual model contract: + +| Harness | Required model contract | Compatibility check | +|---|---|---| +| `codex` | OpenAI Responses API | Valid `v1/responses` inference request | +| `hermes` | OpenAI-compatible chat completions | Valid `v1/chat/completions` inference request | +| `deepagents` | Provider-specific; `nvidia`, `openai`, and `openai-compatible` use chat completions | Valid request for the selected provider path; use `v1/chat/completions` for an OpenAI-compatible route | +| `claude` | Native Anthropic provider | Require `provider: anthropic` and validate the configured credentials/model with native Anthropic tooling; do not route it through Platform IGW | + +Before making any inference requests, show the user the candidate names and +explain that the checks make real, potentially billable model calls. Ask for +explicit confirmation and wait. The original request to select a model or +write a config is not confirmation for these calls. For native-provider +harnesses, use the provider's native validation path instead of forcing the +request through Platform IGW. + +For a Platform-routed OpenAI-compatible candidate, use the context-aware CLI +rather than a hardcoded URL. Preserve the exact model name returned by +`nemo models list`: + +```bash +MODEL_NAME="" + +# Codex +nemo inference gateway model post v1/responses "$MODEL_NAME" \ + --body "{\"model\":\"$MODEL_NAME\",\"input\":\"Reply with exactly: compatibility check\"}" + +# Hermes or an OpenAI-compatible DeepAgents configuration +nemo inference gateway model post v1/chat/completions "$MODEL_NAME" \ + --body "{\"model\":\"$MODEL_NAME\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: compatibility check\"}]}" +``` + +A successful model list lookup, schema validation, Fabric planning, deployment +readiness, empty request, or `GET` does not establish compatibility for a +Platform-routed model. Only candidates that complete a valid request through +the required Platform model path may be returned to `nemo-agent-config`. For a +native-provider harness, require its adapter provider contract and successful +native credential/model validation instead. Exclude failed combinations and +try the next ranked candidate. If no candidate passes, stop without emitting a +model block and ask the user to configure a compatible provider or explicitly +choose a different harness. Never switch the harness silently. + +Record the selected harness, provider, exact model name, required model +contract, and successful check in the handoff. Then apply the normal benchmark +ranking only among compatible candidates. ### Picking the presentation pattern @@ -257,7 +331,11 @@ If the user asks what benchmark was used or wants the raw number, tell them. Do ## Step 4 — Output -Two ready-to-paste blocks. Show whichever fits the user's stage. **When the chosen model's primary-axis score has `source: "inferred_from_ancestor"` or the model relies on `intent_hints` only, include an explicit evidence caveat in the output** — don't let the spec or YAML carry the recommendation forward without surfacing the inference. +Show the blocks that fit the user's stage. Default machine-readable output to +Platform `agent.yaml`. **When the chosen model's primary-axis score has +`source: "inferred_from_ancestor"` or the model relies on `intent_hints` only, +include an explicit evidence caveat in the human-readable recommendation.** Do +not encode benchmark commentary as unsupported config fields. If they're authoring an agent spec for `nemo-spec`: @@ -271,7 +349,42 @@ If they're authoring an agent spec for `nemo-spec`: - **Deployment:** ``` -If they're editing a NAT workflow YAML directly (e.g. tweaking the `agent.yml` `nemo-build-agent` produced): +If they are authoring Platform `agent.yaml`, emit a default model block: + +```yaml +models: + default: + provider: + model: + api_key_env: + base_url: +``` + +Use the provider identity configured on the Platform. Omit `api_key_env` and +`base_url` when the selected provider does not require user-supplied values. +Keep `base_url` directly in the model block, not under `settings`. + +Before emitting this block, complete the harness-specific compatibility check +above. Do not infer compatibility from provider or model-list metadata alone. + +The default model applies to every harness that does not declare its own +model. Add a harness-local override only when that harness intentionally uses +a different model or provider: + +```yaml +harnesses: + : + kind: + model: + provider: + model: +``` + +Do not emit raw Fabric SDK model objects. `nemo-agent-config` owns final YAML +placement and validation. + +If they are explicitly maintaining a legacy NAT workflow YAML, emit the NAT +compatibility block: ```yaml llms: @@ -288,9 +401,10 @@ workflow: tool_names: [] ``` -### Picking the right `_type` +### Picking the right legacy NAT `_type` -Match the chosen model's namespace prefix against `namespace_to_type[]` from the cache: +Only for NAT workflow output, match the chosen model's namespace prefix against +`namespace_to_type[]` from the cache: ```txt For each rule in cache.namespace_to_type: @@ -307,9 +421,15 @@ If no rule matches: Common mappings the cache carries today: `nim/*`, `openai/*`, `anthropic/*`, `bedrock/*`, plus vendor-published NIM names (`qwen/*`, `meta/*`, `nvidia/*`, `microsoft/*`, `mistralai/*`) that route through the NIM provider when served by the platform. Ollama's local endpoint maps to `_type: openai` since it exposes an OpenAI-compatible API. -When the chosen model is non-NIM, also remind the user to set `base_url` and `api_key` (or the equivalent env vars) in the LLM block — those are mandatory for non-NIM providers and aren't auto-filled like they are for the platform's NIM defaults. +For Platform `agent.yaml`, represent credentials with `api_key_env` and put +`base_url` directly in the model block. For a legacy NAT workflow, use the +provider fields required by that NAT LLM component. + +### Pair a legacy NAT model with the right workflow type -### Pair the model with the right agent type +Use this table only when maintaining NAT workflow YAML. Harness selection for +`nemo-agents-spec-v1` belongs to `nemo-agent-config` and must not be inferred +from a NAT workflow type. | What the agent needs to do | Use | |---|---| @@ -348,19 +468,30 @@ If `nemo-explore` invoked this skill, return control to `nemo-explore` with the | User wants a model not in the table | The table is curated, not exhaustive | Tell them honestly; describe the capability gap their choice would have vs the closest recommended model | | `cache_missing` and user wants fresh data | Cache has never been refreshed in this checkout | Tell them the refresh command and note that the static table is still usable | | User picks self-hosted but the recommended model needs cloud | Hard constraint conflict | Drop the recommendation; pick the closest self-hosted-compatible model from the table | +| Available model's harness API support is unknown | Availability was mistaken for harness compatibility | Ask permission to run a valid request through the harness's required model path | +| Compatibility request fails | The exact harness/provider/model/endpoint combination is incompatible in the current environment | Exclude that combination, surface the error, and test the next ranked candidate after the user-approved probe set | +| No candidate passes | No live model satisfies the selected harness contract | Stop without emitting a model block; ask the user to configure a compatible provider or explicitly choose another harness | ## Hard rules - Never name a model before all three profile questions are answered. - Never lead with a model name, benchmark name, or score. - Never recommend a cloud-only model when the user said self-hosted. -- Never write `model_name` (YAML) or "NIM model id" (spec) without showing the plain-English reason alongside it. +- Never return an untested model route to a config-writing skill. +- Never make model inference calls without explicit user confirmation. +- Never silently change the selected harness to accommodate an available model. +- Never emit a model identifier without showing the plain-English reason alongside it. - **When the primary candidate's evidence is anything other than `direct`, the model name does not appear in your response until the user has resolved the trade-off in Pattern B.** Anchoring is the failure mode this guards against — users default to the first model named regardless of caveats. The withhold is non-negotiable. -- When emitting the spec or YAML block, always include an Evidence line/comment naming the source quality. Inferred or name-only choices that propagate downstream without that signal mislead the build skill and the user both. +- When emitting the spec recommendation, always include an Evidence line naming + the source quality. For `agent.yaml`, present the evidence next to the YAML + rather than inventing a config field. ## Gotchas - **"You decide" needs a committed default, not a silent fill-in.** Same rule as `nemo-explore`. Pick something, name it, tell the user. -- **The platform default is `nvidia/llama-3.3-nemotron-super-49b-v1`.** If `nemo-explore` already captured "cloud, no preference", you can route there without re-profiling — but still explain *why* in plain English instead of just naming it. -- **Two model name formats coexist.** Entity-name with hyphens for NAT YAML / `nemo chat` / `nemo agents`. API-Catalog format with slashes for Data Designer. Use the slashed form (`qwen/qwen3-235b-a22b`) in NAT YAML for cloud NIMs; the build skill converts when needed. +- **Do not transform model IDs by punctuation convention.** Use the identifier + returned by the selected live provider or Platform model listing and pair it + with the correct `provider`. Legacy NAT components and Data Designer may use + different provider-specific identifiers; preserve the identifier required by + that consumer instead of assuming the build skill converts it. - **Watch the deployment column.** A 235B cloud-API recommendation aimed at a self-hoster with a 24 GB GPU is the most common mismatch and the easiest to catch by re-reading Step 1. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/references/benchmark_cache.json b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/references/benchmark_cache.json new file mode 100644 index 00000000000..1700ebab647 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-model-selection/references/benchmark_cache.json @@ -0,0 +1,11206 @@ +{ + "generated_at": "2026-06-01T20:20:26.761568+00:00", + "schema_version": "6", + "sources": { + "bfcl": "https://gorilla.cs.berkeley.edu/data_overall.csv", + "arena_elo": "https://datasets-server.huggingface.co/rows?dataset=lmarena-ai%2Fleaderboard-dataset&config=text&split=latest" + }, + "benchmarks": { + "bfcl_v4": { + "full_name": "Berkeley Function Calling Leaderboard v4", + "url": "https://gorilla.cs.berkeley.edu/leaderboard.html", + "what_it_measures": "Whether a model can correctly decide which tool to call, with what arguments, and when not to call any tool at all \u2014 tested across single calls, parallel calls, multi-step sequences, and situations where calling a tool would be the wrong answer.", + "what_it_predicts": "How reliably your agent will invoke tools without hallucinating function names, fabricating arguments, or calling the wrong tool when the user's intent is ambiguous. A high score here means fewer silent failures in production where the agent confidently calls something that doesn't exist.", + "does_not_predict": "General reasoning quality or how well the model writes prose. A model can score poorly here and still be excellent at open-ended tasks that don't involve structured tool calls.", + "scale": "Reported as a percentage 0\u2013100%, higher is better. Current leaderboard top is ~77% (Claude Opus 4.5). This script buckets scores as top \u226570%, strong \u226555%, mid \u226535%, weak below that \u2014 editorial bands calibrated against the live distribution, not published by Berkeley.", + "primary_signal_for": [ + "tool_calling", + "mcp_tools", + "api_agents" + ] + }, + "swe_bench_verified": { + "full_name": "SWE-bench Verified", + "url": "https://www.swebench.com", + "what_it_measures": "Whether a model can read a real GitHub issue, understand an existing codebase it has never seen before, write a patch that fixes the issue, and pass the repo's existing test suite \u2014 all without human guidance.", + "what_it_predicts": "How well your agent will handle multi-step software tasks: reading unfamiliar code, making targeted edits, and not breaking things that were already working. Good signal for agents that interact with codebases rather than just generating isolated snippets.", + "does_not_predict": "Performance on short code-generation prompts, or on tasks outside software engineering. Also does not predict tool-calling accuracy.", + "scale": "% of issues resolved, higher is better. Top OSS models reach 40\u201355%.", + "primary_signal_for": [ + "code_agents", + "software_tasks", + "repo_interaction" + ] + }, + "arena_elo": { + "full_name": "LMSYS Chatbot Arena Elo (overall)", + "url": "https://lmarena.ai", + "what_it_measures": "How often real users, shown two anonymous model responses side by side, prefer one model over the other \u2014 aggregated across millions of head-to-head votes covering every topic imaginable.", + "what_it_predicts": "Whether the model will feel good to interact with in practice: clear writing, appropriate length, not over-hedging, following instructions without being obtuse. Good proxy for general-purpose agents whose output is prose the user will read directly.", + "does_not_predict": "Structured output quality, tool-calling accuracy, or correctness on any specific domain. Human preference can favor confident-sounding wrong answers over correct but uncertain ones.", + "scale": "Elo points, similar to chess ratings. Current leaderboard top is ~1500. This script buckets ratings as top \u22651450, strong \u22651350, mid \u22651250, weak below that \u2014 editorial bands calibrated against the live distribution, not published by LMSYS.", + "primary_signal_for": [ + "general_purpose", + "conversational", + "instruction_following" + ] + }, + "gpqa_diamond": { + "full_name": "Graduate-Level Google-Proof Q&A (Diamond set)", + "url": "https://arxiv.org/abs/2311.12022", + "what_it_measures": "Whether a model can answer questions that were written by PhD researchers to be hard enough that other experts in adjacent fields get them wrong \u2014 covering biology, chemistry, and physics. The 'Diamond' subset is the hardest tier.", + "what_it_predicts": "Deep multi-step reasoning and the ability to synthesize complex information without shortcuts. Relevant for agents that need to draw real conclusions from dense technical or scientific content, not just retrieve and reformat it.", + "does_not_predict": "Performance on everyday tasks, instruction following, or anything involving tool use. Overkill as a signal for most business agents.", + "scale": "% correct, higher is better. Human expert baseline is ~70%; top models reach 75\u201380%.", + "primary_signal_for": [ + "reasoning", + "scientific_analysis", + "technical_depth" + ] + }, + "ruler": { + "full_name": "RULER (Realistic and Unbiased Long-context Evaluation)", + "url": "https://arxiv.org/abs/2404.06654", + "what_it_measures": "Whether a model can actually find and use information buried deep in a long document \u2014 not just claim it supports a large context window. Tests retrieval, tracking multiple entities, and aggregating information across a long input.", + "what_it_predicts": "How much of the model's advertised context window you can actually rely on. A model claiming 128K context might only reliably use 50\u201370K. Critical for agents that process long documents, codebases, or conversation histories.", + "does_not_predict": "Quality on short inputs. A model can be excellent at short tasks and lose the thread badly at 60K tokens.", + "scale": "Score 0\u2013100; also reported as effective context length vs advertised length.", + "primary_signal_for": [ + "long_context", + "document_analysis", + "rag" + ] + } + }, + "models": [ + { + "nim_model": "qwen/qwen3-235b-a22b", + "architecture_note": "235B sparse MoE, ~22B active parameters per forward pass", + "strong_at": [ + "Calling multiple tools in a single turn without mixing up their arguments", + "Parallel tool invocation where the order of calls matters", + "Recovering gracefully when a tool returns an error instead of hallucinating a result" + ], + "watch_out_for": [ + "Requires significant VRAM for self-hosted deployment despite low active params", + "Overkill for agents that only call one or two simple tools" + ], + "best_deployment": "cloud_api", + "primary_benchmarks": [ + "bfcl_v4" + ], + "intent_hints": [ + "Qwen3 family \u2014 Alibaba's instruction-tuned series with a strong public tool-calling track record", + "235B total parameters, 22B active per forward pass \u2014 sparse mixture-of-experts", + "A22B suffix marks the 'active-22B' MoE configuration variant" + ], + "scores": { + "bfcl_v4": { + "raw": 0.5215, + "percent": "52%", + "tier": "mid", + "plain": "Gets tool calls right about 52% of the time. Workable for single-tool or low-complexity agents; expect reliability issues with parallel calls or chained sequences.", + "source": "direct" + }, + "arena_elo": { + "chinese": { + "raw": 1466, + "tier": "top", + "plain": "Rated 1466 Elo on Chinese prompts in head-to-head matches with real users \u2014 at the frontier for this capability.", + "source": "direct" + }, + "coding": { + "raw": 1444, + "tier": "strong", + "plain": "Rated 1444 Elo on coding tasks \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "creative_writing": { + "raw": 1376, + "tier": "strong", + "plain": "Rated 1376 Elo on creative writing \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "english": { + "raw": 1424, + "tier": "strong", + "plain": "Rated 1424 Elo on English prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "exclude_ties": { + "raw": 1407, + "tier": "strong", + "plain": "Rated 1407 Elo on head-to-head preference (excluding ties) \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "expert": { + "raw": 1444, + "tier": "strong", + "plain": "Rated 1444 Elo on expert-level prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "french": { + "raw": 1449, + "tier": "strong", + "plain": "Rated 1449 Elo on French prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "german": { + "raw": 1436, + "tier": "strong", + "plain": "Rated 1436 Elo on German prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "hard_prompts": { + "raw": 1434, + "tier": "strong", + "plain": "Rated 1434 Elo on hard prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "hard_prompts_english": { + "raw": 1436, + "tier": "strong", + "plain": "Rated 1436 Elo on hard prompts (English) \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_business_and_management_and_financial_operations": { + "raw": 1418, + "tier": "strong", + "plain": "Rated 1418 Elo on industry business and management and financial operations \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_entertainment_and_sports_and_media": { + "raw": 1375, + "tier": "strong", + "plain": "Rated 1375 Elo on industry entertainment and sports and media \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_legal_and_government": { + "raw": 1429, + "tier": "strong", + "plain": "Rated 1429 Elo on industry legal and government \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_life_and_physical_and_social_science": { + "raw": 1438, + "tier": "strong", + "plain": "Rated 1438 Elo on industry life and physical and social science \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_mathematical": { + "raw": 1441, + "tier": "strong", + "plain": "Rated 1441 Elo on industry mathematical \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_medicine_and_healthcare": { + "raw": 1445, + "tier": "strong", + "plain": "Rated 1445 Elo on industry medicine and healthcare \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_software_and_it_services": { + "raw": 1446, + "tier": "strong", + "plain": "Rated 1446 Elo on industry software and it services \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_writing_and_literature_and_language": { + "raw": 1390, + "tier": "strong", + "plain": "Rated 1390 Elo on industry writing and literature and language \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "instruction_following": { + "raw": 1408, + "tier": "strong", + "plain": "Rated 1408 Elo on instruction following \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "japanese": { + "raw": 1388, + "tier": "strong", + "plain": "Rated 1388 Elo on Japanese prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "korean": { + "raw": 1393, + "tier": "strong", + "plain": "Rated 1393 Elo on Korean prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "longer_query": { + "raw": 1427, + "tier": "strong", + "plain": "Rated 1427 Elo on longer query \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "math": { + "raw": 1432, + "tier": "strong", + "plain": "Rated 1432 Elo on math \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "multi_turn": { + "raw": 1433, + "tier": "strong", + "plain": "Rated 1433 Elo on multi turn \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "overall": { + "raw": 1419, + "tier": "strong", + "plain": "Rated 1419 Elo on overall head-to-head preference \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "russian": { + "raw": 1412, + "tier": "strong", + "plain": "Rated 1412 Elo on russian \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "spanish": { + "raw": 1426, + "tier": "strong", + "plain": "Rated 1426 Elo on spanish \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + } + } + } + }, + { + "nim_model": "qwen/qwen3-30b-a3b", + "architecture_note": "30B sparse MoE, ~3B active parameters \u2014 very fast inference", + "strong_at": [ + "Tool-calling tasks where speed and cost matter more than perfection", + "High-throughput agents that run many short tool-calling loops per minute", + "Good baseline for tool-heavy agents before you know if you need the larger model" + ], + "watch_out_for": [ + "Lower ceiling than the 235B for complex nested or parallel tool chains", + "May struggle with ambiguous tool selection when multiple tools fit the query" + ], + "best_deployment": "cloud_api_cost_sensitive", + "primary_benchmarks": [ + "bfcl_v4" + ], + "intent_hints": [ + "Qwen3 family", + "30B total parameters, 3B active per forward pass \u2014 fast-inference sparse MoE", + "Designed for high throughput at moderate quality" + ], + "scores": { + "bfcl_v4": { + "raw": 0.4139, + "percent": "41%", + "tier": "mid", + "plain": "Gets tool calls right about 41% of the time. Workable for single-tool or low-complexity agents; expect reliability issues with parallel calls or chained sequences.", + "source": "direct" + }, + "arena_elo": { + "chinese": { + "raw": 1434, + "tier": "strong", + "plain": "Rated 1434 Elo on Chinese prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "coding": { + "raw": 1418, + "tier": "strong", + "plain": "Rated 1418 Elo on coding tasks \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "creative_writing": { + "raw": 1319, + "tier": "mid", + "plain": "Rated 1319 Elo on creative writing \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "english": { + "raw": 1391, + "tier": "strong", + "plain": "Rated 1391 Elo on English prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "exclude_ties": { + "raw": 1358, + "tier": "strong", + "plain": "Rated 1358 Elo on head-to-head preference (excluding ties) \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "expert": { + "raw": 1391, + "tier": "strong", + "plain": "Rated 1391 Elo on expert-level prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "french": { + "raw": 1422, + "tier": "strong", + "plain": "Rated 1422 Elo on French prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "german": { + "raw": 1370, + "tier": "strong", + "plain": "Rated 1370 Elo on German prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "hard_prompts": { + "raw": 1398, + "tier": "strong", + "plain": "Rated 1398 Elo on hard prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "hard_prompts_english": { + "raw": 1406, + "tier": "strong", + "plain": "Rated 1406 Elo on hard prompts (English) \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_business_and_management_and_financial_operations": { + "raw": 1389, + "tier": "strong", + "plain": "Rated 1389 Elo on industry business and management and financial operations \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_entertainment_and_sports_and_media": { + "raw": 1336, + "tier": "mid", + "plain": "Rated 1336 Elo on industry entertainment and sports and media \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_legal_and_government": { + "raw": 1385, + "tier": "strong", + "plain": "Rated 1385 Elo on industry legal and government \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_life_and_physical_and_social_science": { + "raw": 1391, + "tier": "strong", + "plain": "Rated 1391 Elo on industry life and physical and social science \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_mathematical": { + "raw": 1409, + "tier": "strong", + "plain": "Rated 1409 Elo on industry mathematical \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_medicine_and_healthcare": { + "raw": 1391, + "tier": "strong", + "plain": "Rated 1391 Elo on industry medicine and healthcare \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_software_and_it_services": { + "raw": 1418, + "tier": "strong", + "plain": "Rated 1418 Elo on industry software and it services \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_writing_and_literature_and_language": { + "raw": 1337, + "tier": "mid", + "plain": "Rated 1337 Elo on industry writing and literature and language \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "instruction_following": { + "raw": 1362, + "tier": "strong", + "plain": "Rated 1362 Elo on instruction following \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "japanese": { + "raw": 1337, + "tier": "mid", + "plain": "Rated 1337 Elo on Japanese prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "korean": { + "raw": 1328, + "tier": "mid", + "plain": "Rated 1328 Elo on Korean prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "longer_query": { + "raw": 1378, + "tier": "strong", + "plain": "Rated 1378 Elo on longer query \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "math": { + "raw": 1394, + "tier": "strong", + "plain": "Rated 1394 Elo on math \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "multi_turn": { + "raw": 1381, + "tier": "strong", + "plain": "Rated 1381 Elo on multi turn \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "overall": { + "raw": 1384, + "tier": "strong", + "plain": "Rated 1384 Elo on overall head-to-head preference \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "russian": { + "raw": 1364, + "tier": "strong", + "plain": "Rated 1364 Elo on russian \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "spanish": { + "raw": 1392, + "tier": "strong", + "plain": "Rated 1392 Elo on spanish \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + } + } + } + }, + { + "nim_model": "qwen/qwen3-coder-30b-a3b-instruct", + "architecture_note": "30B MoE fine-tuned specifically on software engineering tasks", + "strong_at": [ + "Navigating unfamiliar codebases and making targeted edits", + "Writing patches that don't break existing tests", + "Agents that interact with git, CI, or code review workflows" + ], + "watch_out_for": [ + "Not the right choice for non-code tasks \u2014 general reasoning suffers from the specialization", + "Tool-calling accuracy is good but not best-in-class; use qwen3-235b if tools matter more than code" + ], + "best_deployment": "cloud_api", + "primary_benchmarks": [ + "swe_bench_verified" + ], + "intent_hints": [ + "Qwen3 family, code-specialized fine-tune", + "Same 30B-A3B backbone as qwen3-30b-a3b (30B total, 3B active)", + "Instruction-tuned variant, not base-model completion" + ], + "scores": { + "bfcl_v4": { + "raw": 0.4139, + "percent": "41%", + "tier": "mid", + "plain": "Berkeley hasn't published a BFCL score for this exact variant. Best available signal: its ancestor qwen3-30b-a3b scored 41%. The coder variant is fine-tuned on software engineering data, which usually tightens structured-output discipline (helpful for tool calls) but narrows the variety of tool patterns the model has seen during training (potentially hurts on unfamiliar APIs). Treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "arena_elo": { + "chinese": { + "raw": 1434, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on Chinese prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1434 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "coding": { + "raw": 1418, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on coding tasks. Best available signal: its ancestor qwen3-30b-a3b sits at 1418 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "creative_writing": { + "raw": 1319, + "tier": "mid", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on creative writing. Best available signal: its ancestor qwen3-30b-a3b sits at 1319 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "english": { + "raw": 1391, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on English prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1391 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "exclude_ties": { + "raw": 1358, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on head-to-head preference (excluding ties). Best available signal: its ancestor qwen3-30b-a3b sits at 1358 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "expert": { + "raw": 1391, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on expert-level prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1391 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "french": { + "raw": 1422, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on French prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1422 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "german": { + "raw": 1370, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on German prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1370 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "hard_prompts": { + "raw": 1398, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on hard prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1398 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "hard_prompts_english": { + "raw": 1406, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on hard prompts (English). Best available signal: its ancestor qwen3-30b-a3b sits at 1406 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_business_and_management_and_financial_operations": { + "raw": 1389, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry business and management and financial operations. Best available signal: its ancestor qwen3-30b-a3b sits at 1389 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_entertainment_and_sports_and_media": { + "raw": 1336, + "tier": "mid", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry entertainment and sports and media. Best available signal: its ancestor qwen3-30b-a3b sits at 1336 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_legal_and_government": { + "raw": 1385, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry legal and government. Best available signal: its ancestor qwen3-30b-a3b sits at 1385 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_life_and_physical_and_social_science": { + "raw": 1391, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry life and physical and social science. Best available signal: its ancestor qwen3-30b-a3b sits at 1391 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_mathematical": { + "raw": 1409, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry mathematical. Best available signal: its ancestor qwen3-30b-a3b sits at 1409 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_medicine_and_healthcare": { + "raw": 1391, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry medicine and healthcare. Best available signal: its ancestor qwen3-30b-a3b sits at 1391 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_software_and_it_services": { + "raw": 1418, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry software and it services. Best available signal: its ancestor qwen3-30b-a3b sits at 1418 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "industry_writing_and_literature_and_language": { + "raw": 1337, + "tier": "mid", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on industry writing and literature and language. Best available signal: its ancestor qwen3-30b-a3b sits at 1337 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "instruction_following": { + "raw": 1362, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on instruction following. Best available signal: its ancestor qwen3-30b-a3b sits at 1362 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "japanese": { + "raw": 1337, + "tier": "mid", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on Japanese prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1337 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "korean": { + "raw": 1328, + "tier": "mid", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on Korean prompts. Best available signal: its ancestor qwen3-30b-a3b sits at 1328 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "longer_query": { + "raw": 1378, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on longer query. Best available signal: its ancestor qwen3-30b-a3b sits at 1378 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "math": { + "raw": 1394, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on math. Best available signal: its ancestor qwen3-30b-a3b sits at 1394 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "multi_turn": { + "raw": 1381, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on multi turn. Best available signal: its ancestor qwen3-30b-a3b sits at 1381 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "overall": { + "raw": 1384, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on overall head-to-head preference. Best available signal: its ancestor qwen3-30b-a3b sits at 1384 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "russian": { + "raw": 1364, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on russian. Best available signal: its ancestor qwen3-30b-a3b sits at 1364 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + }, + "spanish": { + "raw": 1392, + "tier": "strong", + "plain": "LMSYS hasn't published an Arena rating for this exact variant on spanish. Best available signal: its ancestor qwen3-30b-a3b sits at 1392 Elo. Post-training can shift human-preference scores in either direction \u2014 treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "qwen/qwen3-30b-a3b" + } + } + } + }, + { + "nim_model": "nvidia/llama-3.1-nemotron-ultra-253b", + "architecture_note": "253B dense model, NVIDIA post-trained on Llama 3.1 for reasoning", + "strong_at": [ + "Multi-step reasoning over long documents without losing the thread", + "Technical and scientific analysis where intermediate reasoning steps matter", + "Agents that need to synthesize information from many retrieved chunks" + ], + "watch_out_for": [ + "Very large model \u2014 cloud API is the practical deployment path for most teams", + "Slower inference than MoE alternatives; not ideal for latency-sensitive loops" + ], + "best_deployment": "cloud_api", + "primary_benchmarks": [ + "gpqa_diamond", + "ruler" + ], + "intent_hints": [ + "Llama 3.1 base, post-trained by NVIDIA", + "253B dense parameters \u2014 large, slower inference", + "'Ultra' tier in Nemotron lineup, optimized for multi-step reasoning over function calling" + ], + "scores": { + "bfcl_v4": { + "raw": 0.1, + "percent": "10%", + "tier": "weak", + "plain": "Tool-calling accuracy around 10% \u2014 well behind current competitive models. Consider a stronger model if precise tool invocation is the main job.", + "source": "direct" + }, + "arena_elo": { + "coding": { + "raw": 1312, + "tier": "mid", + "plain": "Rated 1312 Elo on coding tasks \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "creative_writing": { + "raw": 1315, + "tier": "mid", + "plain": "Rated 1315 Elo on creative writing \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "english": { + "raw": 1344, + "tier": "mid", + "plain": "Rated 1344 Elo on English prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "exclude_ties": { + "raw": 1264, + "tier": "mid", + "plain": "Rated 1264 Elo on head-to-head preference (excluding ties) \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "hard_prompts": { + "raw": 1316, + "tier": "mid", + "plain": "Rated 1316 Elo on hard prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "hard_prompts_english": { + "raw": 1336, + "tier": "mid", + "plain": "Rated 1336 Elo on hard prompts (English) \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_business_and_management_and_financial_operations": { + "raw": 1301, + "tier": "mid", + "plain": "Rated 1301 Elo on industry business and management and financial operations \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_entertainment_and_sports_and_media": { + "raw": 1277, + "tier": "mid", + "plain": "Rated 1277 Elo on industry entertainment and sports and media \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_life_and_physical_and_social_science": { + "raw": 1324, + "tier": "mid", + "plain": "Rated 1324 Elo on industry life and physical and social science \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_software_and_it_services": { + "raw": 1327, + "tier": "mid", + "plain": "Rated 1327 Elo on industry software and it services \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_writing_and_literature_and_language": { + "raw": 1300, + "tier": "mid", + "plain": "Rated 1300 Elo on industry writing and literature and language \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "instruction_following": { + "raw": 1308, + "tier": "mid", + "plain": "Rated 1308 Elo on instruction following \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "longer_query": { + "raw": 1299, + "tier": "mid", + "plain": "Rated 1299 Elo on longer query \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "math": { + "raw": 1359, + "tier": "strong", + "plain": "Rated 1359 Elo on math \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "multi_turn": { + "raw": 1317, + "tier": "mid", + "plain": "Rated 1317 Elo on multi turn \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "overall": { + "raw": 1319, + "tier": "mid", + "plain": "Rated 1319 Elo on overall head-to-head preference \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "russian": { + "raw": 1283, + "tier": "mid", + "plain": "Rated 1283 Elo on russian \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + } + } + } + }, + { + "nim_model": "nvidia/llama-3.3-nemotron-super-49b-v1", + "architecture_note": "49B dense model, NVIDIA post-trained on Llama 3.3; platform default for cloud agents", + "strong_at": [ + "General-purpose agent work where you don't yet know the bottleneck", + "Reasonable tool-calling accuracy plus solid prose \u2014 a balanced starting point", + "The platform's curated default; well-tested across the agent build path" + ], + "watch_out_for": [ + "Not specialized \u2014 a tool-calling specialist will beat it on heavy tool chains", + "Not the fastest \u2014 a smaller MoE wins on cost and latency for simple loops" + ], + "best_deployment": "cloud_api", + "primary_benchmarks": [ + "arena_elo" + ], + "intent_hints": [ + "Llama 3.3 70B base, distilled and post-trained by NVIDIA", + "49B dense \u2014 distilled from a larger model", + "'Super' tier, v1 \u2014 NVIDIA's current cloud agent default" + ], + "scores": { + "bfcl_v4": { + "raw": 0.319, + "percent": "32%", + "tier": "weak", + "plain": "Berkeley hasn't published a BFCL score for this exact variant. Best available signal: its ancestor llama-3.3-70b-instruct scored 32%. NVIDIA's Nemotron post-training emphasizes reasoning and instruction following, not function-call discipline, so a meaningful uplift over the base model isn't likely. Treat as rough signal, not measurement.", + "source": "inferred_from_ancestor", + "inferred_from": "meta/llama-3.3-70b-instruct" + }, + "arena_elo": { + "coding": { + "raw": 1357, + "tier": "strong", + "plain": "Rated 1357 Elo on coding tasks \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "creative_writing": { + "raw": 1310, + "tier": "mid", + "plain": "Rated 1310 Elo on creative writing \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "english": { + "raw": 1356, + "tier": "strong", + "plain": "Rated 1356 Elo on English prompts \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "exclude_ties": { + "raw": 1292, + "tier": "mid", + "plain": "Rated 1292 Elo on head-to-head preference (excluding ties) \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "expert": { + "raw": 1320, + "tier": "mid", + "plain": "Rated 1320 Elo on expert-level prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "hard_prompts": { + "raw": 1333, + "tier": "mid", + "plain": "Rated 1333 Elo on hard prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "hard_prompts_english": { + "raw": 1336, + "tier": "mid", + "plain": "Rated 1336 Elo on hard prompts (English) \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_business_and_management_and_financial_operations": { + "raw": 1310, + "tier": "mid", + "plain": "Rated 1310 Elo on industry business and management and financial operations \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_entertainment_and_sports_and_media": { + "raw": 1280, + "tier": "mid", + "plain": "Rated 1280 Elo on industry entertainment and sports and media \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_legal_and_government": { + "raw": 1345, + "tier": "mid", + "plain": "Rated 1345 Elo on industry legal and government \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_life_and_physical_and_social_science": { + "raw": 1364, + "tier": "strong", + "plain": "Rated 1364 Elo on industry life and physical and social science \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_mathematical": { + "raw": 1401, + "tier": "strong", + "plain": "Rated 1401 Elo on industry mathematical \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_medicine_and_healthcare": { + "raw": 1365, + "tier": "strong", + "plain": "Rated 1365 Elo on industry medicine and healthcare \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_software_and_it_services": { + "raw": 1359, + "tier": "strong", + "plain": "Rated 1359 Elo on industry software and it services \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "industry_writing_and_literature_and_language": { + "raw": 1293, + "tier": "mid", + "plain": "Rated 1293 Elo on industry writing and literature and language \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "instruction_following": { + "raw": 1299, + "tier": "mid", + "plain": "Rated 1299 Elo on instruction following \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "japanese": { + "raw": 1296, + "tier": "mid", + "plain": "Rated 1296 Elo on Japanese prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "longer_query": { + "raw": 1315, + "tier": "mid", + "plain": "Rated 1315 Elo on longer query \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "math": { + "raw": 1394, + "tier": "strong", + "plain": "Rated 1394 Elo on math \u2014 competitive, reliable across the kinds of prompts real users send in this category.", + "source": "direct" + }, + "multi_turn": { + "raw": 1334, + "tier": "mid", + "plain": "Rated 1334 Elo on multi turn \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "overall": { + "raw": 1338, + "tier": "mid", + "plain": "Rated 1338 Elo on overall head-to-head preference \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + } + } + } + }, + { + "nim_model": "meta/llama-3.3-70b-instruct", + "architecture_note": "70B dense model, widely deployed and well-understood", + "strong_at": [ + "General-purpose instruction following across a wide range of tasks", + "Stable, predictable outputs \u2014 well-characterized by the community", + "Good starting point for any agent before you know its bottlenecks" + ], + "watch_out_for": [ + "Not a specialist in any one area \u2014 if tool-calling or code quality is critical, use a specialist", + "Human preference scores well but that doesn't translate directly to agentic reliability" + ], + "best_deployment": "cloud_api_or_self_hosted", + "primary_benchmarks": [ + "arena_elo" + ], + "intent_hints": [ + "Meta's Llama 3.3 70B, instruction-tuned", + "Dense 70B \u2014 well-characterized, widely deployed", + "Most reference implementations target this size class" + ], + "scores": { + "bfcl_v4": { + "raw": 0.319, + "percent": "32%", + "tier": "weak", + "plain": "Tool-calling accuracy around 32% \u2014 well behind current competitive models. Consider a stronger model if precise tool invocation is the main job.", + "source": "direct" + }, + "arena_elo": { + "chinese": { + "raw": 1215, + "tier": "weak", + "plain": "Rated 1215 Elo on Chinese prompts \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "coding": { + "raw": 1269, + "tier": "mid", + "plain": "Rated 1269 Elo on coding tasks \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "creative_writing": { + "raw": 1252, + "tier": "mid", + "plain": "Rated 1252 Elo on creative writing \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "english": { + "raw": 1304, + "tier": "mid", + "plain": "Rated 1304 Elo on English prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "exclude_ties": { + "raw": 1201, + "tier": "weak", + "plain": "Rated 1201 Elo on head-to-head preference (excluding ties) \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "expert": { + "raw": 1224, + "tier": "weak", + "plain": "Rated 1224 Elo on expert-level prompts \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "french": { + "raw": 1282, + "tier": "mid", + "plain": "Rated 1282 Elo on French prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "german": { + "raw": 1251, + "tier": "mid", + "plain": "Rated 1251 Elo on German prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "hard_prompts": { + "raw": 1258, + "tier": "mid", + "plain": "Rated 1258 Elo on hard prompts \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "hard_prompts_english": { + "raw": 1279, + "tier": "mid", + "plain": "Rated 1279 Elo on hard prompts (English) \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_business_and_management_and_financial_operations": { + "raw": 1243, + "tier": "weak", + "plain": "Rated 1243 Elo on industry business and management and financial operations \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "industry_entertainment_and_sports_and_media": { + "raw": 1242, + "tier": "weak", + "plain": "Rated 1242 Elo on industry entertainment and sports and media \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "industry_legal_and_government": { + "raw": 1284, + "tier": "mid", + "plain": "Rated 1284 Elo on industry legal and government \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_life_and_physical_and_social_science": { + "raw": 1284, + "tier": "mid", + "plain": "Rated 1284 Elo on industry life and physical and social science \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_mathematical": { + "raw": 1267, + "tier": "mid", + "plain": "Rated 1267 Elo on industry mathematical \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_medicine_and_healthcare": { + "raw": 1274, + "tier": "mid", + "plain": "Rated 1274 Elo on industry medicine and healthcare \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_software_and_it_services": { + "raw": 1275, + "tier": "mid", + "plain": "Rated 1275 Elo on industry software and it services \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "industry_writing_and_literature_and_language": { + "raw": 1256, + "tier": "mid", + "plain": "Rated 1256 Elo on industry writing and literature and language \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "instruction_following": { + "raw": 1243, + "tier": "weak", + "plain": "Rated 1243 Elo on instruction following \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "japanese": { + "raw": 1149, + "tier": "weak", + "plain": "Rated 1149 Elo on Japanese prompts \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "korean": { + "raw": 1139, + "tier": "weak", + "plain": "Rated 1139 Elo on Korean prompts \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "longer_query": { + "raw": 1256, + "tier": "mid", + "plain": "Rated 1256 Elo on longer query \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "math": { + "raw": 1267, + "tier": "mid", + "plain": "Rated 1267 Elo on math \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "multi_turn": { + "raw": 1281, + "tier": "mid", + "plain": "Rated 1281 Elo on multi turn \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "overall": { + "raw": 1275, + "tier": "mid", + "plain": "Rated 1275 Elo on overall head-to-head preference \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + }, + "russian": { + "raw": 1250, + "tier": "weak", + "plain": "Rated 1250 Elo on russian \u2014 well below current competitive models. Consider a stronger model if this capability is the main job.", + "source": "direct" + }, + "spanish": { + "raw": 1266, + "tier": "mid", + "plain": "Rated 1266 Elo on spanish \u2014 workable, but expect to lose preference comparisons against top models in this category.", + "source": "direct" + } + } + } + }, + { + "nim_model": "microsoft/phi-4-mini-instruct", + "architecture_note": "Small dense model (~4B), optimized for quality-per-parameter", + "strong_at": [ + "Fast, cheap inference for agents where latency is the bottleneck", + "Single-tool or low-complexity tool-calling loops", + "Edge or resource-constrained deployments" + ], + "watch_out_for": [ + "Ceiling is lower than larger models for complex multi-tool orchestration", + "May miss subtle nuance in tool argument generation under ambiguous prompts" + ], + "best_deployment": "self_hosted_low_vram", + "primary_benchmarks": [ + "arena_elo" + ], + "intent_hints": [ + "Microsoft's Phi-4 small variant, instruction-tuned", + "'Mini' \u2014 roughly 3.8B dense parameters, distinct from the 14B Phi-4", + "Designed for high quality-per-parameter at edge / low-VRAM deployments" + ], + "scores": {} + }, + { + "nim_model": "qwen/qwen3-8b", + "architecture_note": "8B dense model with strong BFCL performance for its size class", + "strong_at": [ + "Tool-calling on a single consumer GPU (fits in 12\u201316 GB VRAM)", + "Local development and prototyping before scaling to a larger model", + "Agents where self-hosting is non-negotiable and tools are the primary task" + ], + "watch_out_for": [ + "Not competitive with larger models on complex reasoning chains", + "Context window reliability drops faster than in larger models" + ], + "best_deployment": "self_hosted_local_gpu", + "primary_benchmarks": [ + "bfcl_v4" + ], + "intent_hints": [ + "Qwen3 family, dense", + "8B parameters \u2014 fits on a single consumer GPU (12\u201316 GB VRAM)", + "Strong BFCL performance for its size class" + ], + "scores": { + "bfcl_v4": { + "raw": 0.4257, + "percent": "43%", + "tier": "mid", + "plain": "Gets tool calls right about 43% of the time. Workable for single-tool or low-complexity agents; expect reliability issues with parallel calls or chained sequences.", + "source": "direct" + } + } + } + ], + "upstream_index": { + "bfcl_v4": { + "Claude-Opus-4-5-20251101": 0.7746999999999999, + "Claude-Sonnet-4-5-20250929": 0.7323999999999999, + "Gemini-3-Pro-Preview": 0.7251000000000001, + "GLM-4.6 (FC thinking)": 0.7238, + "Grok-4-1-fast-reasoning": 0.6957, + "Claude-Haiku-4-5-20251001": 0.687, + "o3-2025-04-16": 0.6305, + "Grok-4-0709": 0.6297, + "Moonshotai-Kimi-K2-Instruct": 0.5906, + "Grok-4-1-fast-non-reasoning": 0.5829, + "Command A Reasoning": 0.5706, + "DeepSeek-V3.2-Exp (Prompt + Thinking)": 0.5672999999999999, + "Gemini-2.5-Flash": 0.5624, + "GPT-5.2-2025-12-11": 0.5587, + "GPT-5-mini-2025-08-07": 0.5546, + "xLAM-2-32b-fc-r": 0.5466, + "DeepSeek-V3.2-Exp": 0.5412, + "GPT-4.1-2025-04-14": 0.5396, + "o4-mini-2025-04-16": 0.5324, + "xLAM-2-70b-fc-r": 0.5307, + "Qwen3-235B-A22B-Instruct-2507": 0.5215, + "GPT-5-nano-2025-08-07": 0.5145000000000001, + "Nanbeige4-3B-Thinking-2511": 0.514, + "GPT-4.1-mini-2025-04-14": 0.5045000000000001, + "Qwen3-32B": 0.48710000000000003, + "Nanbeige3.5-Pro-Thinking": 0.4768, + "xLAM-2-8b-fc-r": 0.4668, + "Command A": 0.46490000000000004, + "BitAgent-Bounty-8B": 0.4623, + "Arch-Agent-32B": 0.4537, + "Qwen3-8B": 0.4257, + "ToolACE-2-8B": 0.4244, + "Qwen3-30B-A3B-Instruct-2507": 0.4139, + "xLAM-2-3b-fc-r": 0.4122, + "Qwen3-14B": 0.4103, + "mistral-large-2411": 0.3837, + "Mistral-Medium-2505": 0.37689999999999996, + "Llama-4-Maverick-17B-128E-Instruct-FP8": 0.3729, + "Mistral-small-2506": 0.3715, + "Gemini-2.5-Flash-Lite": 0.36869999999999997, + "Qwen3-4B-Instruct-2507": 0.3568, + "Arch-Agent-3B": 0.35359999999999997, + "GPT-4.1-nano-2025-04-14": 0.33049999999999996, + "Mistral-Small-2506": 0.32380000000000003, + "Arch-Agent-1.5B": 0.3214, + "Command R7B": 0.3207, + "Llama-3.3-70B-Instruct": 0.319, + "Hammer2.1-7b": 0.31670000000000004, + "xLAM-2-1b-fc-r": 0.3044, + "Gemma-3-12b-it": 0.3043, + "Hammer2.1-3b": 0.29710000000000003, + "Gemma-3-27b-it": 0.29469999999999996, + "Phi-4": 0.2879, + "Qwen3-1.7B": 0.2841, + "Llama-4-Scout-17B-16E-Instruct": 0.2813, + "CoALM-70B": 0.2799, + "Hammer2.1-1.5b": 0.2788, + "palmyra-x-004": 0.2787, + "Open-Mistral-Nemo-2407": 0.2763, + "Amazon-Nova-2-Lite-v1:0": 0.271, + "Granite-3.1-8B-Instruct": 0.271, + "Falcon3-10B-Instruct": 0.2701, + "Granite-3.2-8B-Instruct": 0.2687, + "CoALM-8B": 0.2681, + "Llama-3.1-8B-Instruct": 0.2583, + "MiniCPM3-4B-FC": 0.2555, + "Amazon-Nova-Pro-v1:0": 0.24969999999999998, + "Falcon3-7B-Instruct": 0.2403, + "Qwen3-0.6B": 0.23929999999999998, + "Granite-20b-FunctionCalling": 0.2323, + "Amazon-Nova-Micro-v1:0": 0.2229, + "RZN-T": 0.2225, + "MiniCPM3-4B": 0.2208, + "Llama-3.2-3B-Instruct": 0.2195, + "Bielik-11B-v2.3-Instruct": 0.21899999999999997, + "Hammer2.1-0.5b": 0.2122, + "Gemma-3-4b-it": 0.1962, + "Granite-4.0-350m": 0.1898, + "Falcon3-3B-Instruct": 0.1625, + "Ministral-8B-Instruct-2410": 0.111, + "Falcon3-1B-Instruct": 0.1108, + "Llama-3.2-1B-Instruct": 0.1082, + "Llama-3.1-Nemotron-Ultra-253B-v1": 0.1, + "Gemma-3-1b-it": 0.0717 + }, + "arena_elo": { + "claude-opus-4-6-thinking": { + "overall": 1499.3173401234678, + "chinese": 1547.3985756205745, + "coding": 1535.6421998688645, + "creative_writing": 1501.0230754802017, + "english": 1511.9255726002698, + "exclude_ties": 1516.3630582216379, + "expert": 1542.2594844436146, + "french": 1507.541807648552, + "german": 1517.1746329020166, + "hard_prompts": 1526.8979571178922, + "hard_prompts_english": 1535.4080965176854, + "industry_business_and_management_and_financial_operations": 1498.2677323848413, + "industry_entertainment_and_sports_and_media": 1491.3542784973772, + "industry_legal_and_government": 1513.714040326311, + "industry_life_and_physical_and_social_science": 1511.335256277081, + "industry_mathematical": 1522.4311418239886, + "industry_medicine_and_healthcare": 1499.0627521179358, + "industry_software_and_it_services": 1527.9383542497476, + "industry_writing_and_literature_and_language": 1497.5134416221183, + "instruction_following": 1523.6857361076848, + "japanese": 1480.5897592978017, + "korean": 1433.8424879204972, + "longer_query": 1521.1503719668663, + "math": 1512.7890416425882, + "multi_turn": 1505.2539596675826, + "russian": 1494.4264098509595, + "spanish": 1510.1398207499712 + }, + "claude-opus-4-6": { + "overall": 1497.521432838257, + "chinese": 1545.3694283722323, + "coding": 1535.0084409173076, + "creative_writing": 1482.2842542214598, + "english": 1505.8018795598296, + "exclude_ties": 1515.018014179085, + "expert": 1543.7340692352877, + "french": 1503.2496038473496, + "german": 1495.1507129835925, + "hard_prompts": 1525.2016573950923, + "hard_prompts_english": 1529.419504357384, + "industry_business_and_management_and_financial_operations": 1504.0398005007144, + "industry_entertainment_and_sports_and_media": 1478.7179227044548, + "industry_legal_and_government": 1510.350193014646, + "industry_life_and_physical_and_social_science": 1507.9513341614454, + "industry_mathematical": 1522.9765471206056, + "industry_medicine_and_healthcare": 1504.4125223877204, + "industry_software_and_it_services": 1526.5545835369348, + "industry_writing_and_literature_and_language": 1486.9215255007089, + "instruction_following": 1512.1374015412814, + "japanese": 1478.5906482496011, + "korean": 1451.7934175760163, + "longer_query": 1516.0219671839654, + "math": 1509.498700120151, + "multi_turn": 1507.5749859976097, + "russian": 1495.53669025658, + "spanish": 1507.054242332413 + }, + "claude-opus-4-7-thinking": { + "overall": 1486.1495408462329, + "chinese": 1534.8875517086112, + "coding": 1521.1282341074025, + "creative_writing": 1478.2300483312274, + "english": 1493.8434626960493, + "exclude_ties": 1497.8838629277275, + "expert": 1504.759556093219, + "french": 1499.147440025911, + "german": 1484.6502487370742, + "hard_prompts": 1503.2519122187712, + "hard_prompts_english": 1507.2777370321712, + "industry_business_and_management_and_financial_operations": 1481.954133209535, + "industry_entertainment_and_sports_and_media": 1469.9915008556186, + "industry_legal_and_government": 1497.5072701300571, + "industry_life_and_physical_and_social_science": 1504.7244445768463, + "industry_mathematical": 1496.560502906988, + "industry_medicine_and_healthcare": 1498.7980559310583, + "industry_software_and_it_services": 1515.2872313075634, + "industry_writing_and_literature_and_language": 1485.2574690334059, + "instruction_following": 1497.028380323529, + "japanese": 1407.2306587620144, + "korean": 1450.7816055103528, + "longer_query": 1502.9046115775325, + "math": 1493.729397838298, + "multi_turn": 1507.2908996656465, + "russian": 1489.0037467470115, + "spanish": 1477.4450256687085 + }, + "gemini-3.5-flash": { + "overall": 1482.49371928515, + "chinese": 1522.6396047160486, + "coding": 1491.2020807508566, + "creative_writing": 1469.9657686954133, + "english": 1476.346929308338, + "exclude_ties": 1493.0917381143113, + "expert": 1510.2016111997732, + "french": 1456.7386138814907, + "hard_prompts": 1486.3227168483518, + "hard_prompts_english": 1475.5605225796849, + "industry_business_and_management_and_financial_operations": 1453.9712425538264, + "industry_entertainment_and_sports_and_media": 1455.9281014112603, + "industry_legal_and_government": 1482.7051750271694, + "industry_life_and_physical_and_social_science": 1498.8261031729285, + "industry_mathematical": 1509.8196947876972, + "industry_medicine_and_healthcare": 1487.5451250930025, + "industry_software_and_it_services": 1492.9739842433623, + "industry_writing_and_literature_and_language": 1470.6283094124583, + "instruction_following": 1463.537047169242, + "korean": 1477.3907873800722, + "longer_query": 1481.0802057917315, + "math": 1527.1920125392726, + "multi_turn": 1488.5159420584516, + "russian": 1510.5325358360215, + "spanish": 1470.5944099717292 + }, + "gemini-3.1-pro-preview": { + "overall": 1481.4180878042043, + "chinese": 1533.1733652624648, + "coding": 1487.659115824263, + "creative_writing": 1485.5262542705987, + "english": 1478.5870610495683, + "exclude_ties": 1493.2685477370405, + "expert": 1494.450934920278, + "french": 1490.9072858740844, + "german": 1499.562538328094, + "hard_prompts": 1486.8514100134582, + "hard_prompts_english": 1482.092198508572, + "industry_business_and_management_and_financial_operations": 1460.4946664006065, + "industry_entertainment_and_sports_and_media": 1466.7173036917104, + "industry_legal_and_government": 1489.0554247440862, + "industry_life_and_physical_and_social_science": 1497.9064514046509, + "industry_mathematical": 1486.1247019218515, + "industry_medicine_and_healthcare": 1487.8656731923031, + "industry_software_and_it_services": 1488.6729862946463, + "industry_writing_and_literature_and_language": 1479.7163789706335, + "instruction_following": 1470.310802766444, + "japanese": 1507.3448193463173, + "korean": 1476.164447664522, + "longer_query": 1483.256289737149, + "math": 1496.9775698941046, + "multi_turn": 1487.926316061956, + "russian": 1495.2994303192077, + "spanish": 1487.188202579015 + }, + "claude-opus-4-7": { + "overall": 1480.7868818055385, + "chinese": 1533.0867128011437, + "coding": 1514.1299868316885, + "creative_writing": 1477.7283343677113, + "english": 1486.4882829014944, + "exclude_ties": 1489.8562340094586, + "expert": 1514.3014345269178, + "french": 1508.4035176732532, + "german": 1470.5646457226635, + "hard_prompts": 1496.189291404401, + "hard_prompts_english": 1499.2164555466566, + "industry_business_and_management_and_financial_operations": 1477.7790132104374, + "industry_entertainment_and_sports_and_media": 1467.961415604359, + "industry_legal_and_government": 1476.9450394441417, + "industry_life_and_physical_and_social_science": 1491.9469459021755, + "industry_mathematical": 1495.4247899617549, + "industry_medicine_and_healthcare": 1490.8653919933422, + "industry_software_and_it_services": 1503.303267730759, + "industry_writing_and_literature_and_language": 1480.0723948321693, + "instruction_following": 1486.9120785518298, + "japanese": 1447.1862236385105, + "korean": 1450.4454166191501, + "longer_query": 1497.1371429697103, + "math": 1489.3348298411036, + "multi_turn": 1501.837330112253, + "russian": 1484.7758538230366, + "spanish": 1464.997395604464 + }, + "gemini-3-pro": { + "overall": 1479.5712224145673, + "chinese": 1528.2527678323852, + "coding": 1482.6789653179812, + "creative_writing": 1482.6248956722595, + "english": 1479.5640044517909, + "exclude_ties": 1491.2536393741677, + "expert": 1479.7221954953106, + "french": 1496.988913993734, + "german": 1513.7759606576246, + "hard_prompts": 1482.114352264052, + "hard_prompts_english": 1477.9055304677097, + "industry_business_and_management_and_financial_operations": 1460.0286729991058, + "industry_entertainment_and_sports_and_media": 1463.8277600181061, + "industry_legal_and_government": 1492.9607220434184, + "industry_life_and_physical_and_social_science": 1486.3780187344437, + "industry_mathematical": 1475.3480989483285, + "industry_medicine_and_healthcare": 1492.7009926919452, + "industry_software_and_it_services": 1486.7807903655726, + "industry_writing_and_literature_and_language": 1477.650337032488, + "instruction_following": 1459.2416988034615, + "japanese": 1510.497526506423, + "korean": 1453.0147145931207, + "longer_query": 1473.226891650758, + "math": 1476.2397083181238, + "multi_turn": 1483.0039006356003, + "russian": 1494.8469789608882, + "spanish": 1468.6747273612168 + }, + "qwen3.7-max-preview": { + "overall": 1474.5873703060963, + "chinese": 1530.9033993380122, + "coding": 1497.8485703328934, + "creative_writing": 1448.5016648408173, + "english": 1472.216199424985, + "exclude_ties": 1478.9878325369639, + "expert": 1489.3234085106808, + "hard_prompts": 1481.8083220340318, + "hard_prompts_english": 1474.190518213697, + "industry_business_and_management_and_financial_operations": 1446.9529692076828, + "industry_entertainment_and_sports_and_media": 1445.4309560630334, + "industry_legal_and_government": 1474.3302318743454, + "industry_life_and_physical_and_social_science": 1478.9056211754294, + "industry_mathematical": 1470.7368890861655, + "industry_medicine_and_healthcare": 1474.5607890440804, + "industry_software_and_it_services": 1497.0635699962997, + "industry_writing_and_literature_and_language": 1457.0472268918825, + "instruction_following": 1460.928448298582, + "longer_query": 1485.1479617067243, + "math": 1497.8314143411233, + "multi_turn": 1479.6882149373228, + "russian": 1480.3211769341185 + }, + "muse-spark": { + "overall": 1474.0831938110816, + "chinese": 1507.6827990413276, + "coding": 1478.3896792776031, + "creative_writing": 1456.2657628245693, + "english": 1479.7864868860938, + "exclude_ties": 1482.0731237073269, + "expert": 1452.6055646309653, + "french": 1504.7377630886344, + "german": 1489.7045028485988, + "hard_prompts": 1474.995595475922, + "hard_prompts_english": 1475.1629549237684, + "industry_business_and_management_and_financial_operations": 1461.4555312759626, + "industry_entertainment_and_sports_and_media": 1446.0603964356396, + "industry_legal_and_government": 1496.5505110068473, + "industry_life_and_physical_and_social_science": 1481.8263098204036, + "industry_mathematical": 1455.9944212087569, + "industry_medicine_and_healthcare": 1493.2484262310577, + "industry_software_and_it_services": 1483.7833089753026, + "industry_writing_and_literature_and_language": 1451.2670265236725, + "instruction_following": 1439.5680636849986, + "korean": 1456.5464903782017, + "longer_query": 1446.1709117069736, + "math": 1451.3098254829704, + "multi_turn": 1479.398518959663, + "russian": 1470.6342137314402, + "spanish": 1466.4914639514866 + }, + "gpt-5.4-high": { + "overall": 1472.3236292855293, + "chinese": 1519.4520776005133, + "coding": 1496.2167460197766, + "creative_writing": 1441.2416116690715, + "english": 1471.1187958425326, + "exclude_ties": 1478.9170368572202, + "expert": 1516.6864284188885, + "french": 1501.9420194261952, + "german": 1481.6084355077278, + "hard_prompts": 1491.1044947370497, + "hard_prompts_english": 1484.9588229347833, + "industry_business_and_management_and_financial_operations": 1475.4967848727242, + "industry_entertainment_and_sports_and_media": 1444.7507799137747, + "industry_legal_and_government": 1481.7469409242385, + "industry_life_and_physical_and_social_science": 1480.8958692176357, + "industry_mathematical": 1507.9065674869016, + "industry_medicine_and_healthcare": 1460.9059915656285, + "industry_software_and_it_services": 1491.1526923133645, + "industry_writing_and_literature_and_language": 1463.9776107447913, + "instruction_following": 1474.6911141478804, + "japanese": 1503.303847200336, + "korean": 1442.074293583107, + "longer_query": 1478.57537432309, + "math": 1509.1266554125618, + "multi_turn": 1483.93813591659, + "russian": 1486.9998626579309, + "spanish": 1459.2137762122215 + }, + "qwen3.5-max-preview": { + "overall": 1470.8029479971258, + "chinese": 1540.316830027614, + "coding": 1486.8240264185765, + "creative_writing": 1464.3378350764801, + "english": 1476.468881961519, + "exclude_ties": 1480.605897897866, + "expert": 1489.9132058890275, + "french": 1483.124994753049, + "german": 1478.261578024181, + "hard_prompts": 1483.9910617059245, + "hard_prompts_english": 1482.4370035057393, + "industry_business_and_management_and_financial_operations": 1459.9978653436142, + "industry_entertainment_and_sports_and_media": 1455.8012880627211, + "industry_legal_and_government": 1480.016643532494, + "industry_life_and_physical_and_social_science": 1495.8036765833506, + "industry_mathematical": 1493.565599796071, + "industry_medicine_and_healthcare": 1498.144766039269, + "industry_software_and_it_services": 1486.1205832647277, + "industry_writing_and_literature_and_language": 1465.700991723397, + "instruction_following": 1469.0044879345642, + "japanese": 1513.626226646541, + "korean": 1433.4260036280843, + "longer_query": 1475.7707973455722, + "math": 1476.4669849742227, + "multi_turn": 1478.6656337201184, + "russian": 1472.8232725622793, + "spanish": 1466.043355524607 + }, + "ernie-5.1": { + "overall": 1469.5375976414755, + "chinese": 1510.3195715761235, + "coding": 1489.3872068671717, + "creative_writing": 1452.4978374340799, + "english": 1475.7345699840764, + "exclude_ties": 1475.1715395600193, + "expert": 1468.9676367542972, + "french": 1505.6554731680571, + "german": 1472.751079697047, + "hard_prompts": 1480.9452401437034, + "hard_prompts_english": 1480.7351343243438, + "industry_business_and_management_and_financial_operations": 1469.809137229938, + "industry_entertainment_and_sports_and_media": 1440.446194861115, + "industry_legal_and_government": 1474.005277903905, + "industry_life_and_physical_and_social_science": 1487.8945211263551, + "industry_mathematical": 1492.8767138647772, + "industry_medicine_and_healthcare": 1485.693360418243, + "industry_software_and_it_services": 1491.6154273899126, + "industry_writing_and_literature_and_language": 1450.0352268879196, + "instruction_following": 1463.994407696477, + "korean": 1428.038432816792, + "longer_query": 1465.1477162337474, + "math": 1481.2341976946807, + "multi_turn": 1473.2331165416826, + "russian": 1464.5248462360914, + "spanish": 1466.9075133549634 + }, + "glm-5.1": { + "overall": 1469.2713478519543, + "chinese": 1516.0767283615564, + "coding": 1499.4700821582335, + "creative_writing": 1459.6275063051557, + "english": 1480.5955028566825, + "exclude_ties": 1475.6883393278351, + "expert": 1486.4903194612864, + "french": 1477.6652958247737, + "german": 1464.3518883196964, + "hard_prompts": 1481.2856008963101, + "hard_prompts_english": 1487.1746936316426, + "industry_business_and_management_and_financial_operations": 1442.7322845366282, + "industry_entertainment_and_sports_and_media": 1450.6854288007798, + "industry_legal_and_government": 1469.5386606286143, + "industry_life_and_physical_and_social_science": 1487.4882999233967, + "industry_mathematical": 1474.5956717432393, + "industry_medicine_and_healthcare": 1468.9484684125543, + "industry_software_and_it_services": 1492.9932696157678, + "industry_writing_and_literature_and_language": 1459.6885398873771, + "instruction_following": 1455.8740730323395, + "korean": 1420.3262421570653, + "longer_query": 1476.0389374149213, + "math": 1477.212539873537, + "multi_turn": 1475.8924761936905, + "russian": 1459.2021342148632, + "spanish": 1464.528464461083 + }, + "gpt-5.5-high": { + "overall": 1468.8136034817203, + "chinese": 1536.3529327670133, + "coding": 1498.1364338849576, + "creative_writing": 1450.6167198876196, + "english": 1469.5995505691737, + "exclude_ties": 1473.8640501385873, + "expert": 1518.4236679829828, + "french": 1501.0361851183131, + "german": 1437.067682771133, + "hard_prompts": 1486.4990121246146, + "hard_prompts_english": 1483.9879502878143, + "industry_business_and_management_and_financial_operations": 1479.5174805958193, + "industry_entertainment_and_sports_and_media": 1446.2691082328333, + "industry_legal_and_government": 1477.1122561631053, + "industry_life_and_physical_and_social_science": 1479.3437855564046, + "industry_mathematical": 1493.3050599394385, + "industry_medicine_and_healthcare": 1451.2425881890426, + "industry_software_and_it_services": 1492.5201061469525, + "industry_writing_and_literature_and_language": 1468.4041355431796, + "instruction_following": 1478.4916352493933, + "korean": 1469.918557416972, + "longer_query": 1487.5067607159604, + "math": 1480.2018038121532, + "multi_turn": 1468.3943753906492, + "russian": 1475.7636755808896, + "spanish": 1471.8634322700088 + }, + "gemini-3-flash": { + "overall": 1466.1453996796067, + "chinese": 1513.2320283676677, + "coding": 1462.9252452192059, + "creative_writing": 1458.8605172806244, + "english": 1465.153719913004, + "exclude_ties": 1474.3414511190126, + "expert": 1463.2306229588044, + "french": 1493.0895156615861, + "german": 1493.1458366351037, + "hard_prompts": 1464.7076101032305, + "hard_prompts_english": 1458.0278547089777, + "industry_business_and_management_and_financial_operations": 1449.4421385268627, + "industry_entertainment_and_sports_and_media": 1448.7655911616994, + "industry_legal_and_government": 1476.9412561573495, + "industry_life_and_physical_and_social_science": 1472.0164954511295, + "industry_mathematical": 1465.5773500646126, + "industry_medicine_and_healthcare": 1465.1164486481052, + "industry_software_and_it_services": 1471.3438860891138, + "industry_writing_and_literature_and_language": 1458.731282649358, + "instruction_following": 1437.0203643647046, + "japanese": 1495.2689697933154, + "korean": 1447.2385254166595, + "longer_query": 1451.1146348195562, + "math": 1473.9407883538295, + "multi_turn": 1472.6104033388292, + "russian": 1478.494872021623, + "spanish": 1459.9884613333918 + }, + "gpt-5.5": { + "overall": 1463.895626253054, + "chinese": 1553.3625336508994, + "coding": 1481.1885443632282, + "creative_writing": 1449.7714737210617, + "english": 1462.9705939162056, + "exclude_ties": 1466.9808238918583, + "expert": 1501.6175124655047, + "french": 1512.4340891244767, + "german": 1448.8189975937955, + "hard_prompts": 1479.9619249438138, + "hard_prompts_english": 1473.9291284445717, + "industry_business_and_management_and_financial_operations": 1478.0807977365048, + "industry_entertainment_and_sports_and_media": 1441.5261406977193, + "industry_legal_and_government": 1468.4710771788825, + "industry_life_and_physical_and_social_science": 1482.3313860869248, + "industry_mathematical": 1493.993887693794, + "industry_medicine_and_healthcare": 1450.4952283054963, + "industry_software_and_it_services": 1481.0335768338966, + "industry_writing_and_literature_and_language": 1465.1050776847849, + "instruction_following": 1471.4784891852105, + "korean": 1421.1830120251052, + "longer_query": 1474.334632741447, + "math": 1481.0907165678443, + "multi_turn": 1467.4563504648315, + "russian": 1471.1128483017856, + "spanish": 1459.5366971988676 + }, + "mimo-v2.5-pro": { + "overall": 1460.695520222813, + "chinese": 1498.9265133613742, + "coding": 1499.3168681947839, + "creative_writing": 1441.8417730722995, + "english": 1474.6020887319667, + "exclude_ties": 1462.4565008624759, + "expert": 1511.1498539309723, + "french": 1488.9527272316088, + "german": 1422.6827379075387, + "hard_prompts": 1485.3356355529907, + "hard_prompts_english": 1496.1922175486004, + "industry_business_and_management_and_financial_operations": 1455.7264717906307, + "industry_entertainment_and_sports_and_media": 1442.5195966984918, + "industry_legal_and_government": 1473.7464553049845, + "industry_life_and_physical_and_social_science": 1479.3294748684416, + "industry_mathematical": 1509.3341090929523, + "industry_medicine_and_healthcare": 1469.1333830545693, + "industry_software_and_it_services": 1488.9608658111242, + "industry_writing_and_literature_and_language": 1450.31896951535, + "instruction_following": 1480.413583691658, + "korean": 1467.313723885786, + "longer_query": 1481.7757757462628, + "math": 1485.970677291682, + "multi_turn": 1474.8956094129965, + "russian": 1451.0750013914626, + "spanish": 1472.8763186797698 + }, + "gemini-2.5-pro": { + "overall": 1457.3758028182863, + "chinese": 1507.6485625818962, + "coding": 1451.8298920801453, + "creative_writing": 1454.628490587313, + "english": 1457.2960783722283, + "exclude_ties": 1460.491737907124, + "expert": 1455.2021925823076, + "french": 1476.0220197759734, + "german": 1486.080696728571, + "hard_prompts": 1454.9355285980191, + "hard_prompts_english": 1451.1512383325166, + "industry_business_and_management_and_financial_operations": 1439.8808816450592, + "industry_entertainment_and_sports_and_media": 1440.9993786477985, + "industry_legal_and_government": 1479.6672408947554, + "industry_life_and_physical_and_social_science": 1480.0814807214438, + "industry_mathematical": 1453.1391057573564, + "industry_medicine_and_healthcare": 1476.3042712875665, + "industry_software_and_it_services": 1459.6566257891882, + "industry_writing_and_literature_and_language": 1454.0064315756733, + "instruction_following": 1436.8416027437422, + "japanese": 1465.6161472448966, + "korean": 1436.0756615120267, + "longer_query": 1449.3870710519716, + "math": 1450.406396903678, + "multi_turn": 1452.6454029677614, + "russian": 1460.7711277419774, + "spanish": 1469.0406436960495 + }, + "gpt-5.4": { + "overall": 1456.2504911639153, + "chinese": 1521.7940267998365, + "coding": 1481.2977380051072, + "creative_writing": 1432.2484750953256, + "english": 1458.5447044661084, + "exclude_ties": 1456.3782509174346, + "expert": 1488.559968771741, + "french": 1458.8819589010313, + "german": 1444.429494565977, + "hard_prompts": 1473.5188428176032, + "hard_prompts_english": 1471.256203943208, + "industry_business_and_management_and_financial_operations": 1463.3387104166845, + "industry_entertainment_and_sports_and_media": 1428.4371432521255, + "industry_legal_and_government": 1468.6492727883042, + "industry_life_and_physical_and_social_science": 1459.3568661812415, + "industry_mathematical": 1458.1553699348378, + "industry_medicine_and_healthcare": 1445.326294810528, + "industry_software_and_it_services": 1477.4436582828769, + "industry_writing_and_literature_and_language": 1452.2864402463854, + "instruction_following": 1458.1267002604295, + "japanese": 1480.1446445290271, + "korean": 1436.708964219911, + "longer_query": 1470.0832934689292, + "math": 1451.1733556842971, + "multi_turn": 1465.9828389062548, + "russian": 1461.71688222166, + "spanish": 1459.6957964449732 + }, + "kimi-k2.6": { + "overall": 1456.144290476874, + "chinese": 1529.46838716885, + "coding": 1485.367951775284, + "creative_writing": 1430.8957515270858, + "english": 1459.6099154564033, + "exclude_ties": 1456.93908809863, + "expert": 1488.0947668437527, + "french": 1473.0959936487486, + "german": 1442.6785852694263, + "hard_prompts": 1472.679277996638, + "hard_prompts_english": 1475.2093042335432, + "industry_business_and_management_and_financial_operations": 1448.8685427602713, + "industry_entertainment_and_sports_and_media": 1427.2863050497738, + "industry_legal_and_government": 1453.140697209966, + "industry_life_and_physical_and_social_science": 1465.7991462116622, + "industry_mathematical": 1486.2541136928992, + "industry_medicine_and_healthcare": 1464.3853976257665, + "industry_software_and_it_services": 1481.9657485801354, + "industry_writing_and_literature_and_language": 1435.1472694418203, + "instruction_following": 1450.3395570510474, + "japanese": 1458.1147793898017, + "korean": 1435.436309192858, + "longer_query": 1465.0977443444438, + "math": 1472.2892124987025, + "multi_turn": 1448.2022101965763, + "russian": 1444.4051012910563, + "spanish": 1453.1914783016593 + }, + "claude-sonnet-4-6": { + "overall": 1454.411873211459, + "chinese": 1481.8998427525705, + "coding": 1497.910790273034, + "creative_writing": 1436.1639621455124, + "english": 1468.7090016630298, + "exclude_ties": 1454.3978363544434, + "expert": 1496.9958192315817, + "french": 1460.5888029995315, + "german": 1434.3082132517861, + "hard_prompts": 1482.3705151974646, + "hard_prompts_english": 1490.5925863289283, + "industry_business_and_management_and_financial_operations": 1459.8339948258204, + "industry_entertainment_and_sports_and_media": 1432.6033989523191, + "industry_legal_and_government": 1467.8892114363086, + "industry_life_and_physical_and_social_science": 1469.2287180502067, + "industry_mathematical": 1480.364741073149, + "industry_medicine_and_healthcare": 1442.1174446417067, + "industry_software_and_it_services": 1487.3499835731582, + "industry_writing_and_literature_and_language": 1442.7073314192457, + "instruction_following": 1477.2072770396737, + "japanese": 1372.8457951332553, + "korean": 1422.2119498088557, + "longer_query": 1482.0611783221473, + "math": 1455.1231953053637, + "multi_turn": 1461.5505634317285, + "russian": 1434.4919799231739, + "spanish": 1465.3264442714744 + }, + "grok-4.20-beta-0309-reasoning": { + "overall": 1453.7932199178938, + "chinese": 1484.9848955329937, + "coding": 1459.3142866698972, + "creative_writing": 1432.1536820706367, + "english": 1456.6680714705103, + "exclude_ties": 1454.2308043709693, + "expert": 1441.4870938361298, + "french": 1484.1291077466317, + "german": 1426.4692351038645, + "hard_prompts": 1453.2758679081924, + "hard_prompts_english": 1452.9573275492594, + "industry_business_and_management_and_financial_operations": 1433.1360282354383, + "industry_entertainment_and_sports_and_media": 1427.7058989587333, + "industry_legal_and_government": 1445.0818906184893, + "industry_life_and_physical_and_social_science": 1457.7779508546944, + "industry_mathematical": 1455.071853718877, + "industry_medicine_and_healthcare": 1456.4836897633988, + "industry_software_and_it_services": 1463.6596950024243, + "industry_writing_and_literature_and_language": 1439.331162636297, + "instruction_following": 1422.390992642355, + "japanese": 1421.27145531954, + "korean": 1424.636542273824, + "longer_query": 1434.3131367974206, + "math": 1461.4536685308733, + "multi_turn": 1458.7743906695866, + "russian": 1466.6378157618115, + "spanish": 1441.4360166370661 + }, + "grok-4.20-multi-agent-beta-0309": { + "overall": 1450.5170000589649, + "chinese": 1480.2851232222863, + "coding": 1460.6970174882954, + "creative_writing": 1431.188138754255, + "english": 1452.1322986113987, + "exclude_ties": 1449.1248197362606, + "expert": 1451.189817329838, + "french": 1470.5600587605147, + "german": 1443.782304852502, + "hard_prompts": 1448.4500219397967, + "hard_prompts_english": 1445.779748370298, + "industry_business_and_management_and_financial_operations": 1423.6187011460975, + "industry_entertainment_and_sports_and_media": 1422.5197078701406, + "industry_legal_and_government": 1451.853009380501, + "industry_life_and_physical_and_social_science": 1458.515105963525, + "industry_mathematical": 1450.051992594455, + "industry_medicine_and_healthcare": 1460.6658329680995, + "industry_software_and_it_services": 1463.7321132348839, + "industry_writing_and_literature_and_language": 1431.5186707788228, + "instruction_following": 1419.2479016140333, + "japanese": 1381.479531392776, + "korean": 1413.2538209420456, + "longer_query": 1430.2822119796579, + "math": 1443.9126239463635, + "multi_turn": 1456.66930850805, + "russian": 1458.7555543682636, + "spanish": 1432.3868977127054 + }, + "claude-opus-4-5-20251101": { + "overall": 1449.2958392887874, + "chinese": 1474.1022338272753, + "coding": 1497.3831328549584, + "creative_writing": 1443.4006434021162, + "english": 1459.8956471522886, + "exclude_ties": 1447.4870616311061, + "expert": 1485.207802752956, + "french": 1483.4682123544794, + "german": 1439.2788803477924, + "hard_prompts": 1475.5868245112547, + "hard_prompts_english": 1480.6428063874896, + "industry_business_and_management_and_financial_operations": 1446.8972423471512, + "industry_entertainment_and_sports_and_media": 1432.0981139863745, + "industry_legal_and_government": 1454.5411131478531, + "industry_life_and_physical_and_social_science": 1451.7001650995353, + "industry_mathematical": 1469.706015973763, + "industry_medicine_and_healthcare": 1442.7585337553287, + "industry_software_and_it_services": 1480.8034857635782, + "industry_writing_and_literature_and_language": 1449.5214063319595, + "instruction_following": 1473.1279983509837, + "japanese": 1408.8055551350903, + "korean": 1425.562956700509, + "longer_query": 1478.5248619072374, + "math": 1460.934592607259, + "multi_turn": 1464.7948140612698, + "russian": 1446.6302007075326, + "spanish": 1450.188850471379 + }, + "dola-seed-2.0-pro": { + "overall": 1449.2268434247617, + "chinese": 1507.6575489058287, + "coding": 1472.557238342167, + "creative_writing": 1409.4014637899013, + "english": 1448.3152998274873, + "exclude_ties": 1449.2470316121046, + "expert": 1442.232461608551, + "french": 1473.3364439828936, + "german": 1432.9050510978282, + "hard_prompts": 1456.2581900397204, + "hard_prompts_english": 1448.2036986766461, + "industry_business_and_management_and_financial_operations": 1438.6915457075525, + "industry_entertainment_and_sports_and_media": 1415.1041803973176, + "industry_legal_and_government": 1455.5387935868262, + "industry_life_and_physical_and_social_science": 1454.0745100070203, + "industry_mathematical": 1431.700932320539, + "industry_medicine_and_healthcare": 1477.0413881905895, + "industry_software_and_it_services": 1475.3226096636772, + "industry_writing_and_literature_and_language": 1411.9540997046859, + "instruction_following": 1413.2540026784477, + "japanese": 1384.6058651700353, + "korean": 1429.5386923993406, + "longer_query": 1429.3709367180395, + "math": 1437.693074184045, + "multi_turn": 1441.662964372787, + "russian": 1449.9474086952237, + "spanish": 1465.7463951578331 + }, + "amazon-nova-experimental-chat-26-02-10": { + "overall": 1448.2126909502824, + "chinese": 1457.2377239491564, + "coding": 1484.8792488706888, + "creative_writing": 1365.3581833775306, + "english": 1456.1312015132805, + "exclude_ties": 1448.0717061351588, + "expert": 1502.943740878393, + "hard_prompts": 1457.5501438515687, + "hard_prompts_english": 1458.8077150721442, + "industry_business_and_management_and_financial_operations": 1450.7015495891906, + "industry_entertainment_and_sports_and_media": 1399.9503336902856, + "industry_legal_and_government": 1466.9400235912408, + "industry_life_and_physical_and_social_science": 1448.965813893865, + "industry_medicine_and_healthcare": 1427.835846140004, + "industry_software_and_it_services": 1481.026881045893, + "industry_writing_and_literature_and_language": 1389.062363869137, + "instruction_following": 1427.4950215838762, + "longer_query": 1435.981142460604, + "math": 1440.3454350422041, + "multi_turn": 1444.625926025285, + "russian": 1422.661869952047 + }, + "deepseek-v4-pro-thinking": { + "overall": 1446.4377494669338, + "chinese": 1497.3187077807997, + "coding": 1456.4617331676436, + "creative_writing": 1435.2093338485506, + "english": 1453.95523001753, + "exclude_ties": 1442.8394902212406, + "expert": 1449.5492386029384, + "french": 1406.2528579112052, + "german": 1412.849290510566, + "hard_prompts": 1455.119451822371, + "hard_prompts_english": 1460.8371690765164, + "industry_business_and_management_and_financial_operations": 1432.153980614767, + "industry_entertainment_and_sports_and_media": 1417.0388154681593, + "industry_legal_and_government": 1467.5409310672762, + "industry_life_and_physical_and_social_science": 1456.7722175378237, + "industry_mathematical": 1456.3236211147662, + "industry_medicine_and_healthcare": 1473.5088758415754, + "industry_software_and_it_services": 1459.5554493232246, + "industry_writing_and_literature_and_language": 1431.6692938916613, + "instruction_following": 1434.1784266703107, + "korean": 1417.3232480797715, + "longer_query": 1452.4708002864036, + "math": 1466.7899010432373, + "multi_turn": 1449.0216037119014, + "russian": 1444.907644657396, + "spanish": 1432.2371074993403 + }, + "claude-opus-4-5-20251101-thinking-32k": { + "overall": 1446.3539728654698, + "chinese": 1462.7245430748528, + "coding": 1502.6045737518893, + "creative_writing": 1443.6042186008458, + "english": 1458.187115944798, + "exclude_ties": 1443.9522043765378, + "expert": 1479.9417219105696, + "french": 1459.7704824298241, + "german": 1441.5345505527732, + "hard_prompts": 1470.932555887865, + "hard_prompts_english": 1479.9710405316796, + "industry_business_and_management_and_financial_operations": 1432.2321176700516, + "industry_entertainment_and_sports_and_media": 1425.2818236687697, + "industry_legal_and_government": 1450.9288242315038, + "industry_life_and_physical_and_social_science": 1443.510222735619, + "industry_mathematical": 1455.394520109276, + "industry_medicine_and_healthcare": 1427.9732623714804, + "industry_software_and_it_services": 1480.2082784333847, + "industry_writing_and_literature_and_language": 1443.8590236894095, + "instruction_following": 1477.032440809388, + "japanese": 1426.8148269221203, + "korean": 1374.0803301551282, + "longer_query": 1477.3697264055572, + "math": 1458.9152045675903, + "multi_turn": 1460.9925067403594, + "russian": 1436.1599163012747, + "spanish": 1451.3693717357485 + }, + "gemini-3-flash (thinking-minimal)": { + "overall": 1446.1744186190592, + "chinese": 1479.3233167640046, + "coding": 1440.5303603060906, + "creative_writing": 1438.295699583489, + "english": 1445.1610789639783, + "exclude_ties": 1444.4947792839316, + "expert": 1425.1295755996468, + "french": 1452.5581991343404, + "german": 1463.584568212416, + "hard_prompts": 1441.4854063966384, + "hard_prompts_english": 1436.0253569791098, + "industry_business_and_management_and_financial_operations": 1427.2917060446985, + "industry_entertainment_and_sports_and_media": 1426.0740598419013, + "industry_legal_and_government": 1452.595258984554, + "industry_life_and_physical_and_social_science": 1459.3110632104626, + "industry_mathematical": 1438.2048437969404, + "industry_medicine_and_healthcare": 1453.2697572625787, + "industry_software_and_it_services": 1450.1266267882922, + "industry_writing_and_literature_and_language": 1437.9311088747563, + "instruction_following": 1416.950294428959, + "japanese": 1429.633242558962, + "korean": 1434.120630287317, + "longer_query": 1431.5270889971357, + "math": 1449.252666528253, + "multi_turn": 1460.4273497475976, + "russian": 1458.7295025993851, + "spanish": 1424.6381837624292 + }, + "ernie-5.0-0110": { + "overall": 1446.0984972264002, + "chinese": 1501.4137273516035, + "coding": 1457.2914601755876, + "creative_writing": 1428.3094853271696, + "english": 1448.1901583317751, + "exclude_ties": 1445.963481821208, + "expert": 1426.822149219055, + "french": 1465.3589447868408, + "german": 1435.8952699644308, + "hard_prompts": 1447.7105761794808, + "hard_prompts_english": 1448.142256665142, + "industry_business_and_management_and_financial_operations": 1429.2767337277326, + "industry_entertainment_and_sports_and_media": 1414.621653379929, + "industry_legal_and_government": 1438.9972006900095, + "industry_life_and_physical_and_social_science": 1450.8870387225513, + "industry_mathematical": 1429.366285817159, + "industry_medicine_and_healthcare": 1461.9216315338072, + "industry_software_and_it_services": 1464.5699797893637, + "industry_writing_and_literature_and_language": 1423.136691751539, + "instruction_following": 1415.4590330896674, + "japanese": 1379.6939808887535, + "korean": 1405.5117335819814, + "longer_query": 1423.195173419592, + "math": 1438.9296881095302, + "multi_turn": 1432.7394743788882, + "russian": 1446.0745450966442, + "spanish": 1445.9396201840807 + }, + "deepseek-v4-pro": { + "overall": 1446.0037784590718, + "chinese": 1503.6440306844859, + "coding": 1466.3830105934956, + "creative_writing": 1438.1606772371167, + "english": 1451.57883513097, + "exclude_ties": 1442.9194299934584, + "expert": 1452.8684566388463, + "french": 1448.2254927986787, + "german": 1454.7770716872049, + "hard_prompts": 1454.3826195127776, + "hard_prompts_english": 1453.7258317136118, + "industry_business_and_management_and_financial_operations": 1436.9622305952487, + "industry_entertainment_and_sports_and_media": 1428.2057803960968, + "industry_legal_and_government": 1454.1628408202496, + "industry_life_and_physical_and_social_science": 1462.7796000481212, + "industry_mathematical": 1454.434529570864, + "industry_medicine_and_healthcare": 1461.4876404273873, + "industry_software_and_it_services": 1463.5728572775172, + "industry_writing_and_literature_and_language": 1444.2347198167195, + "instruction_following": 1436.8243759450313, + "japanese": 1432.9742307949284, + "korean": 1402.5140741275397, + "longer_query": 1452.022280383459, + "math": 1431.7698591869953, + "multi_turn": 1462.6088448161447, + "russian": 1442.9824127111788, + "spanish": 1441.6145425601107 + }, + "grok-4.20-beta1": { + "overall": 1445.726396193007, + "chinese": 1479.1219971550438, + "coding": 1446.3459502684616, + "creative_writing": 1436.6704224147368, + "english": 1452.20958702568, + "exclude_ties": 1442.7772122428833, + "expert": 1425.554061815197, + "french": 1444.985981219795, + "german": 1459.6250036511208, + "hard_prompts": 1442.0321742582248, + "hard_prompts_english": 1442.850001752323, + "industry_business_and_management_and_financial_operations": 1425.9828281077248, + "industry_entertainment_and_sports_and_media": 1417.5993211506595, + "industry_legal_and_government": 1438.7922623115812, + "industry_life_and_physical_and_social_science": 1443.3820835752201, + "industry_mathematical": 1434.4210499794913, + "industry_medicine_and_healthcare": 1455.202446010425, + "industry_software_and_it_services": 1452.5061918356805, + "industry_writing_and_literature_and_language": 1427.4543432687065, + "instruction_following": 1415.9053609950727, + "japanese": 1421.274524300643, + "korean": 1417.2781535100903, + "longer_query": 1426.3962485819316, + "math": 1436.5178805109344, + "multi_turn": 1453.0991544812957, + "russian": 1447.0742494794024, + "spanish": 1441.713574150887 + }, + "glm-5": { + "overall": 1445.444984601615, + "chinese": 1517.724706586262, + "coding": 1456.9655187022531, + "creative_writing": 1439.0752797829175, + "english": 1456.2707412565144, + "exclude_ties": 1443.0032430000688, + "expert": 1460.7864938427563, + "french": 1467.1417184595257, + "german": 1438.1365264873305, + "hard_prompts": 1451.5701290662412, + "hard_prompts_english": 1457.6399945722458, + "industry_business_and_management_and_financial_operations": 1428.7161874491371, + "industry_entertainment_and_sports_and_media": 1423.1248866688745, + "industry_legal_and_government": 1448.1235148037717, + "industry_life_and_physical_and_social_science": 1463.2726508740054, + "industry_mathematical": 1454.0002670870736, + "industry_medicine_and_healthcare": 1457.0291871056836, + "industry_software_and_it_services": 1456.9146512899904, + "industry_writing_and_literature_and_language": 1435.2636640031615, + "instruction_following": 1427.636125013048, + "japanese": 1391.4848942174788, + "korean": 1423.403874099409, + "longer_query": 1446.4390262081313, + "math": 1435.9865707976865, + "multi_turn": 1452.8774214192476, + "russian": 1436.8936795822808, + "spanish": 1433.9337952389726 + }, + "kimi-k2.5-thinking": { + "overall": 1444.818591305524, + "chinese": 1505.6694611592088, + "coding": 1475.4661196669601, + "creative_writing": 1420.5928461548303, + "english": 1452.1146904090558, + "exclude_ties": 1442.2282034357609, + "expert": 1466.7748878888713, + "french": 1459.9861038773852, + "german": 1443.7996672942056, + "hard_prompts": 1453.7622277802914, + "hard_prompts_english": 1457.9273017207292, + "industry_business_and_management_and_financial_operations": 1426.3970848394456, + "industry_entertainment_and_sports_and_media": 1420.9074740794726, + "industry_legal_and_government": 1436.7353646243655, + "industry_life_and_physical_and_social_science": 1461.4776777075222, + "industry_mathematical": 1475.3113803364747, + "industry_medicine_and_healthcare": 1449.7888118849708, + "industry_software_and_it_services": 1470.8595482632668, + "industry_writing_and_literature_and_language": 1422.8019314828612, + "instruction_following": 1430.1488903596587, + "japanese": 1433.5436204869504, + "korean": 1406.8159624155712, + "longer_query": 1443.0177644703317, + "math": 1470.6015778142826, + "multi_turn": 1444.4623098685458, + "russian": 1432.5736458684873, + "spanish": 1466.4889355436676 + }, + "qwen3.6-max-preview": { + "overall": 1444.377128379033, + "chinese": 1482.575372865726, + "coding": 1463.2481595690795, + "creative_writing": 1429.636678028518, + "english": 1444.2744464435511, + "exclude_ties": 1439.0879648280427, + "expert": 1473.3042338630805, + "hard_prompts": 1452.8392447090334, + "hard_prompts_english": 1447.4816557202778, + "industry_business_and_management_and_financial_operations": 1426.5537833095343, + "industry_entertainment_and_sports_and_media": 1421.7319273362993, + "industry_legal_and_government": 1443.0446000370118, + "industry_life_and_physical_and_social_science": 1468.2618079951824, + "industry_mathematical": 1489.0403383301814, + "industry_medicine_and_healthcare": 1464.830165500663, + "industry_software_and_it_services": 1458.4444974197768, + "industry_writing_and_literature_and_language": 1447.5753491788241, + "instruction_following": 1433.8036281023108, + "longer_query": 1453.0600271045391, + "math": 1478.8452012769317, + "multi_turn": 1453.4519768531268, + "russian": 1444.6029663998504, + "spanish": 1448.3816630331216 + }, + "gemma-4-31b": { + "overall": 1442.38345440772, + "chinese": 1473.5087595232412, + "coding": 1455.193451727307, + "creative_writing": 1417.9388564881413, + "english": 1448.9342076723456, + "exclude_ties": 1440.9038473777696, + "expert": 1451.301660026708, + "french": 1438.28681027508, + "hard_prompts": 1446.7174267159264, + "hard_prompts_english": 1455.9073035358927, + "industry_business_and_management_and_financial_operations": 1424.9942839338755, + "industry_entertainment_and_sports_and_media": 1404.4204105933163, + "industry_legal_and_government": 1446.7971247278313, + "industry_life_and_physical_and_social_science": 1453.3583307026695, + "industry_mathematical": 1464.287732202775, + "industry_medicine_and_healthcare": 1449.1557700589171, + "industry_software_and_it_services": 1460.8068625036499, + "industry_writing_and_literature_and_language": 1429.8591865575877, + "instruction_following": 1432.7431922314, + "longer_query": 1443.7502802555048, + "math": 1464.4044697523989, + "multi_turn": 1451.328287450164, + "russian": 1461.204498162274, + "spanish": 1459.195599837186 + }, + "ernie-5.0-preview-1203": { + "overall": 1442.2557544745762, + "chinese": 1480.2422005740627, + "coding": 1425.8962693623764, + "creative_writing": 1422.181768615961, + "english": 1440.189011314263, + "exclude_ties": 1439.9631782522067, + "expert": 1420.8641776124039, + "french": 1463.9155443160346, + "german": 1457.7607614322303, + "hard_prompts": 1437.545478991836, + "hard_prompts_english": 1431.0738703430852, + "industry_business_and_management_and_financial_operations": 1421.7880268353981, + "industry_entertainment_and_sports_and_media": 1407.446994567368, + "industry_legal_and_government": 1447.5429653717847, + "industry_life_and_physical_and_social_science": 1467.5498153649487, + "industry_mathematical": 1408.9070221280938, + "industry_medicine_and_healthcare": 1474.0286725085014, + "industry_software_and_it_services": 1447.572261657322, + "industry_writing_and_literature_and_language": 1413.141101882976, + "instruction_following": 1401.87897143542, + "korean": 1373.772551510566, + "longer_query": 1409.2148026519847, + "math": 1405.9205459453824, + "multi_turn": 1413.6518817546576, + "russian": 1440.4697073568925, + "spanish": 1475.2238892347254 + }, + "gpt-5.1-high": { + "overall": 1441.3820837064127, + "chinese": 1492.887204233829, + "coding": 1451.8480852191183, + "creative_writing": 1427.9007940752444, + "english": 1448.821472942408, + "exclude_ties": 1438.1477161450873, + "expert": 1468.2564887343985, + "french": 1454.7593062663977, + "german": 1436.731750758989, + "hard_prompts": 1454.6515955152413, + "hard_prompts_english": 1457.7870353499893, + "industry_business_and_management_and_financial_operations": 1429.6738025345956, + "industry_entertainment_and_sports_and_media": 1412.8277795828556, + "industry_legal_and_government": 1455.0031783246652, + "industry_life_and_physical_and_social_science": 1455.9541839673961, + "industry_mathematical": 1446.3013461047558, + "industry_medicine_and_healthcare": 1457.1224940734032, + "industry_software_and_it_services": 1453.0717290261687, + "industry_writing_and_literature_and_language": 1425.9171115246788, + "instruction_following": 1439.6185455895154, + "japanese": 1450.8543179270132, + "korean": 1396.493571170496, + "longer_query": 1445.9834057770329, + "math": 1442.95178284578, + "multi_turn": 1448.3750419672328, + "russian": 1438.3020047453379, + "spanish": 1434.5898176427627 + }, + "qwen3.5-397b-a17b": { + "overall": 1440.9072607298383, + "chinese": 1504.2876953946045, + "coding": 1466.9251276490736, + "creative_writing": 1408.784879479717, + "english": 1448.7539998245609, + "exclude_ties": 1438.8745836139965, + "expert": 1467.4382931242983, + "french": 1456.8221675507048, + "german": 1439.563578639297, + "hard_prompts": 1450.8129538353764, + "hard_prompts_english": 1456.1122232668283, + "industry_business_and_management_and_financial_operations": 1432.377519096852, + "industry_entertainment_and_sports_and_media": 1406.0889098425907, + "industry_legal_and_government": 1435.905752463152, + "industry_life_and_physical_and_social_science": 1460.3941584632805, + "industry_mathematical": 1460.082862314086, + "industry_medicine_and_healthcare": 1456.0175241885588, + "industry_software_and_it_services": 1466.8720178796727, + "industry_writing_and_literature_and_language": 1421.8065996817782, + "instruction_following": 1423.3488558585343, + "japanese": 1437.576099250267, + "korean": 1392.8197556857594, + "longer_query": 1443.197057917655, + "math": 1447.9241138512405, + "multi_turn": 1444.844116464987, + "russian": 1432.6045562735562, + "spanish": 1454.9420573004613 + }, + "glm-4.6": { + "overall": 1440.5358954156661, + "chinese": 1500.3339244294814, + "coding": 1449.721844418114, + "creative_writing": 1413.2969566645047, + "english": 1448.297245495497, + "exclude_ties": 1437.8875573247499, + "expert": 1433.984106007071, + "french": 1463.6843326134654, + "german": 1449.0094838016632, + "hard_prompts": 1440.7850594103884, + "hard_prompts_english": 1443.7957236753962, + "industry_business_and_management_and_financial_operations": 1418.768261940627, + "industry_entertainment_and_sports_and_media": 1408.7026359878032, + "industry_legal_and_government": 1450.0452182705067, + "industry_life_and_physical_and_social_science": 1458.2607547419864, + "industry_mathematical": 1447.6158452386098, + "industry_medicine_and_healthcare": 1459.0674683820794, + "industry_software_and_it_services": 1456.2435157899884, + "industry_writing_and_literature_and_language": 1415.0791576211343, + "instruction_following": 1412.1924866949612, + "japanese": 1392.0630654523973, + "korean": 1387.8523058160326, + "longer_query": 1423.7198820405583, + "math": 1433.0926007365488, + "multi_turn": 1427.2072682089813, + "russian": 1419.2117660099202, + "spanish": 1435.6084217888936 + }, + "gpt-5.2-chat-latest-20260210": { + "overall": 1439.1521202909696, + "chinese": 1468.4480526151035, + "coding": 1448.768824201008, + "creative_writing": 1400.4385936865635, + "english": 1442.7498367734474, + "exclude_ties": 1433.3420026252547, + "expert": 1440.16397773199, + "french": 1439.1823715056394, + "german": 1446.5971514385487, + "hard_prompts": 1445.6711123705297, + "hard_prompts_english": 1447.0236056842814, + "industry_business_and_management_and_financial_operations": 1432.272785419649, + "industry_entertainment_and_sports_and_media": 1399.379180922112, + "industry_legal_and_government": 1435.5774100292733, + "industry_life_and_physical_and_social_science": 1440.095364764141, + "industry_mathematical": 1432.425426406629, + "industry_medicine_and_healthcare": 1447.1371227765394, + "industry_software_and_it_services": 1453.3892231935054, + "industry_writing_and_literature_and_language": 1420.0037332374716, + "instruction_following": 1416.036208465723, + "japanese": 1430.4722731394722, + "korean": 1394.454804578817, + "longer_query": 1427.7422856762203, + "math": 1436.8475854821304, + "multi_turn": 1459.7045813149, + "russian": 1440.5899521543793, + "spanish": 1433.3353959553199 + }, + "qwen3-max-preview": { + "overall": 1438.9001526895204, + "chinese": 1486.3664614109246, + "coding": 1457.160561403031, + "creative_writing": 1401.0726965992253, + "english": 1440.0151949125661, + "exclude_ties": 1435.806736901267, + "expert": 1460.295668117654, + "french": 1460.5472028511354, + "german": 1466.1508882388375, + "hard_prompts": 1449.1149766648766, + "hard_prompts_english": 1445.3236446201531, + "industry_business_and_management_and_financial_operations": 1437.2342190616139, + "industry_entertainment_and_sports_and_media": 1403.7787166684132, + "industry_legal_and_government": 1434.846278072468, + "industry_life_and_physical_and_social_science": 1457.4421545026605, + "industry_mathematical": 1446.4069968758306, + "industry_medicine_and_healthcare": 1467.7372402304647, + "industry_software_and_it_services": 1458.8807841462005, + "industry_writing_and_literature_and_language": 1406.47817441154, + "instruction_following": 1421.081746250695, + "japanese": 1398.2292917402217, + "korean": 1395.6277460298388, + "longer_query": 1438.7106001991888, + "math": 1449.434567357042, + "multi_turn": 1448.8026469676893, + "russian": 1420.5753817249756, + "spanish": 1462.135388876971 + }, + "grok-4.1-thinking": { + "overall": 1437.98574287734, + "chinese": 1455.4515626034536, + "coding": 1445.3317806429288, + "creative_writing": 1407.0511013636983, + "english": 1442.7733703001634, + "exclude_ties": 1432.7835694607184, + "expert": 1421.735602521404, + "french": 1453.0166945465255, + "german": 1435.0994923900907, + "hard_prompts": 1435.0350687477976, + "hard_prompts_english": 1435.0620124885031, + "industry_business_and_management_and_financial_operations": 1418.3497650423053, + "industry_entertainment_and_sports_and_media": 1408.7870485334172, + "industry_legal_and_government": 1442.276148312004, + "industry_life_and_physical_and_social_science": 1438.9292800505714, + "industry_mathematical": 1412.3082383553417, + "industry_medicine_and_healthcare": 1454.6022434612555, + "industry_software_and_it_services": 1452.5374937751233, + "industry_writing_and_literature_and_language": 1405.805208237088, + "instruction_following": 1397.3018096128594, + "japanese": 1379.6740453493974, + "korean": 1399.581976378339, + "longer_query": 1409.501174075454, + "math": 1423.9650109700115, + "multi_turn": 1433.7851417693064, + "russian": 1434.0073310322093, + "spanish": 1439.966742249325 + }, + "qwen3.6-plus": { + "overall": 1437.427158499667, + "chinese": 1477.3716478138829, + "coding": 1463.3443518258543, + "creative_writing": 1399.4654397312095, + "english": 1445.042775851714, + "exclude_ties": 1432.5484745192236, + "expert": 1453.6656783445449, + "french": 1476.7673480430294, + "german": 1452.026388643766, + "hard_prompts": 1448.1393751470214, + "hard_prompts_english": 1451.3649267629553, + "industry_business_and_management_and_financial_operations": 1434.4063517509344, + "industry_entertainment_and_sports_and_media": 1402.6018650465733, + "industry_legal_and_government": 1438.643323016289, + "industry_life_and_physical_and_social_science": 1435.5991245145892, + "industry_mathematical": 1452.6133807242702, + "industry_medicine_and_healthcare": 1450.30944991903, + "industry_software_and_it_services": 1459.0827467985177, + "industry_writing_and_literature_and_language": 1418.977981791858, + "instruction_following": 1427.0930579031588, + "japanese": 1421.6211533048001, + "korean": 1393.0209732647845, + "longer_query": 1438.2093123149057, + "math": 1451.1640972726948, + "multi_turn": 1433.573222371024, + "russian": 1432.599184389651, + "spanish": 1440.7527997767631 + }, + "claude-sonnet-4-5-20250929": { + "overall": 1437.4101523056577, + "chinese": 1462.8113924212248, + "coding": 1486.2665301867532, + "creative_writing": 1441.5099117043658, + "english": 1451.5222239163165, + "exclude_ties": 1432.3930750686438, + "expert": 1473.5373220774711, + "french": 1452.69523403324, + "german": 1423.0058225420628, + "hard_prompts": 1460.9370037536619, + "hard_prompts_english": 1471.0778619664745, + "industry_business_and_management_and_financial_operations": 1438.9975404967738, + "industry_entertainment_and_sports_and_media": 1426.9867781874323, + "industry_legal_and_government": 1447.2766861726957, + "industry_life_and_physical_and_social_science": 1445.5341549651885, + "industry_mathematical": 1444.227100619106, + "industry_medicine_and_healthcare": 1440.3943482143234, + "industry_software_and_it_services": 1472.420952203997, + "industry_writing_and_literature_and_language": 1440.8927710457563, + "instruction_following": 1458.4699891666241, + "japanese": 1388.3661838305966, + "korean": 1395.694358181091, + "longer_query": 1476.4043379056297, + "math": 1419.9588168273401, + "multi_turn": 1464.5628757643494, + "russian": 1434.1062000837717, + "spanish": 1456.189119764732 + }, + "grok-4.1": { + "overall": 1437.065606443339, + "chinese": 1473.1365701949421, + "coding": 1442.7557541056678, + "creative_writing": 1412.4635806610029, + "english": 1446.1736611887522, + "exclude_ties": 1432.3621527156606, + "expert": 1414.1380504281224, + "french": 1445.7546587435002, + "german": 1444.3570969728712, + "hard_prompts": 1436.1978755732162, + "hard_prompts_english": 1438.8021442306556, + "industry_business_and_management_and_financial_operations": 1424.8124862447546, + "industry_entertainment_and_sports_and_media": 1406.0268766832596, + "industry_legal_and_government": 1446.001822625322, + "industry_life_and_physical_and_social_science": 1444.9309392619616, + "industry_mathematical": 1408.8282390005613, + "industry_medicine_and_healthcare": 1445.3024864534336, + "industry_software_and_it_services": 1450.7735332702625, + "industry_writing_and_literature_and_language": 1407.5077760107397, + "instruction_following": 1399.6525104080781, + "japanese": 1401.9250573909458, + "korean": 1411.690848930618, + "longer_query": 1415.125281380252, + "math": 1419.3954379728868, + "multi_turn": 1439.0441225681489, + "russian": 1428.2222085460046, + "spanish": 1422.1261655774842 + }, + "glm-4.7": { + "overall": 1436.054289475202, + "chinese": 1484.7387255893161, + "coding": 1456.0061662497599, + "creative_writing": 1402.3523361168204, + "english": 1453.840088778753, + "exclude_ties": 1432.1179201800105, + "expert": 1419.882103661955, + "french": 1428.3544485875493, + "german": 1418.9907939651468, + "hard_prompts": 1444.0753329513911, + "hard_prompts_english": 1452.0094540647592, + "industry_business_and_management_and_financial_operations": 1428.8261358309846, + "industry_entertainment_and_sports_and_media": 1402.759753522505, + "industry_legal_and_government": 1445.9571883127735, + "industry_life_and_physical_and_social_science": 1455.59532116368, + "industry_mathematical": 1411.5035873753084, + "industry_medicine_and_healthcare": 1463.8498651632724, + "industry_software_and_it_services": 1453.206237583095, + "industry_writing_and_literature_and_language": 1411.4529940944703, + "instruction_following": 1413.606373142191, + "korean": 1407.7262415979278, + "longer_query": 1434.5818150791858, + "math": 1423.877960005397, + "multi_turn": 1448.299347654056, + "russian": 1427.3305238287235, + "spanish": 1416.952342665179 + }, + "mimo-v2-pro": { + "overall": 1435.3556005999194, + "chinese": 1451.2642733034086, + "coding": 1476.5826482637349, + "creative_writing": 1412.6668573795826, + "english": 1453.049708121955, + "exclude_ties": 1429.5754236007285, + "expert": 1477.354778894158, + "french": 1466.39419594889, + "german": 1420.5519700635675, + "hard_prompts": 1456.8796495040053, + "hard_prompts_english": 1472.459360373472, + "industry_business_and_management_and_financial_operations": 1437.7525335100709, + "industry_entertainment_and_sports_and_media": 1413.5065203227173, + "industry_legal_and_government": 1444.8952329352974, + "industry_life_and_physical_and_social_science": 1457.841073844002, + "industry_mathematical": 1465.1927483142993, + "industry_medicine_and_healthcare": 1439.466136060354, + "industry_software_and_it_services": 1467.7428519777523, + "industry_writing_and_literature_and_language": 1422.1807727614394, + "instruction_following": 1444.5510322111636, + "japanese": 1384.5553556007933, + "korean": 1398.0076778113762, + "longer_query": 1453.3339435737444, + "math": 1448.099435278652, + "multi_turn": 1458.129580355945, + "russian": 1427.3184254771845, + "spanish": 1465.548689822809 + }, + "gemma-4-26b-a4b": { + "overall": 1434.748685611606, + "chinese": 1478.1711588632363, + "coding": 1443.7265025307777, + "creative_writing": 1404.5491396074544, + "english": 1441.3475303092982, + "exclude_ties": 1429.792816355246, + "expert": 1449.7827457162841, + "french": 1465.9124720557377, + "hard_prompts": 1438.7830619999813, + "hard_prompts_english": 1443.4482653032571, + "industry_business_and_management_and_financial_operations": 1433.4584048927775, + "industry_entertainment_and_sports_and_media": 1407.2487717036954, + "industry_legal_and_government": 1451.539521978196, + "industry_life_and_physical_and_social_science": 1443.6268776391814, + "industry_mathematical": 1469.254749348355, + "industry_medicine_and_healthcare": 1426.7304345105943, + "industry_software_and_it_services": 1453.0721674592037, + "industry_writing_and_literature_and_language": 1409.5005033564673, + "instruction_following": 1421.1124474685769, + "longer_query": 1429.8014581246675, + "math": 1467.652052935661, + "multi_turn": 1439.4446018902083, + "russian": 1444.1085265829943 + }, + "claude-sonnet-4-5-20250929-thinking-32k": { + "overall": 1431.8533945137433, + "chinese": 1462.5610378200138, + "coding": 1488.3160164646579, + "creative_writing": 1420.6122962129161, + "english": 1449.5419317013884, + "exclude_ties": 1423.6473357470732, + "expert": 1481.1787798253001, + "french": 1443.5365794668808, + "german": 1425.0870617648259, + "hard_prompts": 1459.826836551425, + "hard_prompts_english": 1473.0638064111772, + "industry_business_and_management_and_financial_operations": 1429.058868788109, + "industry_entertainment_and_sports_and_media": 1416.4001678823963, + "industry_legal_and_government": 1436.51631252437, + "industry_life_and_physical_and_social_science": 1436.761389289146, + "industry_mathematical": 1461.7007481545613, + "industry_medicine_and_healthcare": 1426.3156948408607, + "industry_software_and_it_services": 1469.0273841122014, + "industry_writing_and_literature_and_language": 1429.3541308842357, + "instruction_following": 1456.8016222467163, + "japanese": 1371.8703635074048, + "korean": 1360.2619419328612, + "longer_query": 1473.4998721507077, + "math": 1446.615683183308, + "multi_turn": 1449.789746521202, + "russian": 1423.3367655299826, + "spanish": 1436.4803835569376 + }, + "ernie-5.0-preview-1022": { + "overall": 1430.12643552734, + "chinese": 1504.0721257104146, + "coding": 1413.3482775359805, + "creative_writing": 1416.033514826141, + "english": 1430.6182903664733, + "exclude_ties": 1423.7874336509717, + "expert": 1422.8170506207475, + "hard_prompts": 1421.366434474323, + "hard_prompts_english": 1417.6375909365559, + "industry_business_and_management_and_financial_operations": 1416.7937247547093, + "industry_entertainment_and_sports_and_media": 1403.2348176847013, + "industry_legal_and_government": 1462.7582612749627, + "industry_life_and_physical_and_social_science": 1445.991939684419, + "industry_mathematical": 1428.4836133659358, + "industry_medicine_and_healthcare": 1450.4851370363501, + "industry_software_and_it_services": 1435.8318156312885, + "industry_writing_and_literature_and_language": 1410.4327164440413, + "instruction_following": 1394.826529519185, + "longer_query": 1407.414653493977, + "math": 1410.3349209483954, + "multi_turn": 1422.331784367643, + "russian": 1399.9455990651095 + }, + "mistral-large-3": { + "overall": 1430.0971717688246, + "chinese": 1441.2392558986407, + "coding": 1444.4052934533076, + "creative_writing": 1392.3099999553974, + "english": 1440.4747700309156, + "exclude_ties": 1423.1359630036236, + "expert": 1414.3381349426118, + "french": 1477.7725561376026, + "german": 1433.7015602006052, + "hard_prompts": 1430.676558919361, + "hard_prompts_english": 1436.171152527807, + "industry_business_and_management_and_financial_operations": 1424.0783410592053, + "industry_entertainment_and_sports_and_media": 1389.9629685889004, + "industry_legal_and_government": 1447.1978702124804, + "industry_life_and_physical_and_social_science": 1450.4539798782853, + "industry_mathematical": 1423.1243268370981, + "industry_medicine_and_healthcare": 1452.5841825755629, + "industry_software_and_it_services": 1449.5389288460522, + "industry_writing_and_literature_and_language": 1402.7592820247924, + "instruction_following": 1405.8466801344362, + "japanese": 1386.3615806582295, + "korean": 1382.3910020446845, + "longer_query": 1412.0060320344799, + "math": 1414.8899586235784, + "multi_turn": 1430.2159067893106, + "russian": 1409.322688958698, + "spanish": 1446.374489701929 + }, + "glm-4.5": { + "overall": 1428.878099627673, + "chinese": 1469.7978312102973, + "coding": 1433.2885774515587, + "creative_writing": 1395.474460886104, + "english": 1433.104630571291, + "exclude_ties": 1420.9706437772018, + "expert": 1427.5075176053597, + "french": 1428.8538700778795, + "german": 1404.9390053601503, + "hard_prompts": 1429.142107716688, + "hard_prompts_english": 1424.3921371157724, + "industry_business_and_management_and_financial_operations": 1420.3036039802535, + "industry_entertainment_and_sports_and_media": 1404.7726374699391, + "industry_legal_and_government": 1437.7174480756912, + "industry_life_and_physical_and_social_science": 1442.929688962389, + "industry_mathematical": 1426.7380823316994, + "industry_medicine_and_healthcare": 1454.4670550164626, + "industry_software_and_it_services": 1438.5972699708075, + "industry_writing_and_literature_and_language": 1402.1551187107702, + "instruction_following": 1403.5462634184462, + "japanese": 1416.7732327106837, + "korean": 1376.2626498863237, + "longer_query": 1411.0364051660965, + "math": 1426.9027298413828, + "multi_turn": 1416.4400420827465, + "russian": 1414.7251906942963, + "spanish": 1449.9274010124716 + }, + "chatgpt-4o-latest-20250326": { + "overall": 1428.8140612439001, + "chinese": 1452.635081395846, + "coding": 1414.7104246064603, + "creative_writing": 1407.1913212839752, + "english": 1431.9201447442379, + "exclude_ties": 1421.583919434293, + "expert": 1399.4031153445103, + "french": 1446.436129285873, + "german": 1423.882264041629, + "hard_prompts": 1424.6807573464307, + "hard_prompts_english": 1422.7244700325534, + "industry_business_and_management_and_financial_operations": 1424.7460102411212, + "industry_entertainment_and_sports_and_media": 1410.8988401426768, + "industry_legal_and_government": 1443.288518602871, + "industry_life_and_physical_and_social_science": 1442.6199791917877, + "industry_mathematical": 1405.5375049204688, + "industry_medicine_and_healthcare": 1452.4785558263109, + "industry_software_and_it_services": 1430.075110659996, + "industry_writing_and_literature_and_language": 1415.0015459037938, + "instruction_following": 1403.9063647326104, + "japanese": 1407.9450926933837, + "korean": 1402.9593021519179, + "longer_query": 1412.9695253180362, + "math": 1405.9253464643552, + "multi_turn": 1454.4389920302183, + "russian": 1430.2097126471638, + "spanish": 1435.0120777231775 + }, + "deepseek-v4-flash": { + "overall": 1428.1805780153302, + "chinese": 1469.5973926637198, + "coding": 1450.006126711327, + "creative_writing": 1397.546419313576, + "english": 1438.668976942946, + "exclude_ties": 1419.588198403334, + "expert": 1438.9124732125208, + "french": 1423.2056993373533, + "german": 1425.3746993911757, + "hard_prompts": 1438.962477352336, + "hard_prompts_english": 1444.3477207102064, + "industry_business_and_management_and_financial_operations": 1411.7117056882455, + "industry_entertainment_and_sports_and_media": 1394.6102134332355, + "industry_legal_and_government": 1437.740366509098, + "industry_life_and_physical_and_social_science": 1448.0344260399686, + "industry_mathematical": 1442.8705904604587, + "industry_medicine_and_healthcare": 1446.2726542680168, + "industry_software_and_it_services": 1449.1442792945445, + "industry_writing_and_literature_and_language": 1411.0050411329491, + "instruction_following": 1421.7001990983777, + "japanese": 1382.083568125732, + "korean": 1369.2088607417643, + "longer_query": 1430.1897317830346, + "math": 1446.1548007567833, + "multi_turn": 1443.7558978153004, + "russian": 1424.7811212992622, + "spanish": 1440.0495182824275 + }, + "deepseek-r1-0528": { + "overall": 1427.9882245754368, + "chinese": 1459.527163743767, + "coding": 1427.0776852633794, + "creative_writing": 1409.919157048671, + "english": 1434.3433447178047, + "exclude_ties": 1420.2739889153972, + "expert": 1397.1034788058164, + "french": 1420.5018267200992, + "german": 1397.8601083492658, + "hard_prompts": 1416.0275770413205, + "hard_prompts_english": 1417.641127537419, + "industry_business_and_management_and_financial_operations": 1399.0014830059888, + "industry_entertainment_and_sports_and_media": 1402.056677136971, + "industry_legal_and_government": 1419.4984576105824, + "industry_life_and_physical_and_social_science": 1430.6343570855165, + "industry_mathematical": 1418.682643495989, + "industry_medicine_and_healthcare": 1446.105353510259, + "industry_software_and_it_services": 1435.472951962743, + "industry_writing_and_literature_and_language": 1403.373217671038, + "instruction_following": 1382.0842886831017, + "japanese": 1383.0461559067808, + "korean": 1355.6270843549944, + "longer_query": 1391.5396927784666, + "math": 1402.8643226010101, + "multi_turn": 1404.9620935184264, + "russian": 1424.499419528814, + "spanish": 1405.5480142039144 + }, + "mimo-v2.5": { + "overall": 1426.815412360329, + "chinese": 1474.2522035493532, + "coding": 1467.767357300506, + "creative_writing": 1397.1737062820114, + "english": 1448.6831123716602, + "exclude_ties": 1416.4339767735332, + "expert": 1463.1818244583085, + "french": 1434.5699323150714, + "german": 1399.3991285747045, + "hard_prompts": 1450.8011600030113, + "hard_prompts_english": 1461.7354585220412, + "industry_business_and_management_and_financial_operations": 1428.9234633271337, + "industry_entertainment_and_sports_and_media": 1409.6858721439032, + "industry_legal_and_government": 1408.7363728167336, + "industry_life_and_physical_and_social_science": 1436.5274577859861, + "industry_mathematical": 1453.4561651608574, + "industry_medicine_and_healthcare": 1420.7600477677483, + "industry_software_and_it_services": 1462.5139539222337, + "industry_writing_and_literature_and_language": 1411.437217216242, + "instruction_following": 1433.5919536440638, + "korean": 1377.794941133199, + "longer_query": 1445.4103930476278, + "math": 1437.8746414667066, + "multi_turn": 1447.3693202053018, + "russian": 1393.831627289794, + "spanish": 1414.1129536717328 + }, + "mistral-medium-2508": { + "overall": 1425.636544444855, + "chinese": 1448.707325985844, + "coding": 1434.998823732616, + "creative_writing": 1392.93312316002, + "english": 1436.7633130362972, + "exclude_ties": 1416.863147542049, + "expert": 1409.7509123954294, + "french": 1461.4012685355738, + "german": 1431.4308857686544, + "hard_prompts": 1427.4582967261013, + "hard_prompts_english": 1433.2248778836547, + "industry_business_and_management_and_financial_operations": 1419.2041267265317, + "industry_entertainment_and_sports_and_media": 1390.5125331826373, + "industry_legal_and_government": 1441.255681776978, + "industry_life_and_physical_and_social_science": 1447.816593778921, + "industry_mathematical": 1416.1634139006037, + "industry_medicine_and_healthcare": 1450.9658933541884, + "industry_software_and_it_services": 1438.7867234817868, + "industry_writing_and_literature_and_language": 1396.225914502197, + "instruction_following": 1399.6372641795479, + "japanese": 1374.0303064930208, + "korean": 1375.6031068500702, + "longer_query": 1408.0308354531921, + "math": 1410.6204999475146, + "multi_turn": 1420.6977823896016, + "russian": 1412.9379505628974, + "spanish": 1427.7793451247271 + }, + "longcat-flash-chat-2602-exp": { + "overall": 1425.3952877334655, + "chinese": 1464.9458989552902, + "coding": 1473.6330836943453, + "creative_writing": 1386.4138012826766, + "english": 1452.234897206056, + "exclude_ties": 1415.2840200071298, + "expert": 1451.3321059149234, + "french": 1421.0031467309543, + "german": 1377.074254482006, + "hard_prompts": 1438.3552082672563, + "hard_prompts_english": 1460.020913178829, + "industry_business_and_management_and_financial_operations": 1414.0468263255511, + "industry_entertainment_and_sports_and_media": 1392.8390918951661, + "industry_legal_and_government": 1411.4593900866594, + "industry_life_and_physical_and_social_science": 1430.7565878555317, + "industry_mathematical": 1434.506696595692, + "industry_medicine_and_healthcare": 1426.9147884764388, + "industry_software_and_it_services": 1465.68601152813, + "industry_writing_and_literature_and_language": 1388.98420023774, + "instruction_following": 1410.2624552561172, + "japanese": 1388.4501515174318, + "korean": 1370.797990132126, + "longer_query": 1423.0979567913978, + "math": 1435.5791816761339, + "multi_turn": 1413.5844456929187, + "russian": 1383.284424306542, + "spanish": 1415.2234999811146 + }, + "grok-3-preview-02-24": { + "overall": 1424.741258519126, + "chinese": 1447.8668956086465, + "coding": 1432.3191842763933, + "creative_writing": 1414.7568193061165, + "english": 1435.8407303670235, + "exclude_ties": 1417.7805043062083, + "expert": 1419.4578407296217, + "french": 1466.8857163769624, + "german": 1428.152107011855, + "hard_prompts": 1433.3583551776373, + "hard_prompts_english": 1441.039441399619, + "industry_business_and_management_and_financial_operations": 1408.681526983995, + "industry_entertainment_and_sports_and_media": 1415.2762599591156, + "industry_legal_and_government": 1442.955491154813, + "industry_life_and_physical_and_social_science": 1437.3231747495622, + "industry_mathematical": 1405.434508557734, + "industry_medicine_and_healthcare": 1464.9873292014636, + "industry_software_and_it_services": 1437.8167204075478, + "industry_writing_and_literature_and_language": 1410.789354708722, + "instruction_following": 1409.3870383033855, + "japanese": 1396.7501411677463, + "korean": 1371.2877722714957, + "longer_query": 1438.7596660957443, + "math": 1390.1411857337216, + "multi_turn": 1423.1193877562553, + "russian": 1415.9982804146823, + "spanish": 1415.3501221487072 + }, + "deepseek-v3.2-exp-thinking": { + "overall": 1423.7041025323297, + "chinese": 1456.0570390671637, + "coding": 1438.152519281805, + "creative_writing": 1391.5906383422412, + "english": 1440.6526542352792, + "exclude_ties": 1412.881516191257, + "expert": 1417.8171190593105, + "french": 1412.9731171056715, + "german": 1433.7559093900798, + "hard_prompts": 1426.3839310117237, + "hard_prompts_english": 1445.450706593851, + "industry_business_and_management_and_financial_operations": 1394.1205178232872, + "industry_entertainment_and_sports_and_media": 1389.529007030433, + "industry_legal_and_government": 1419.801284052409, + "industry_life_and_physical_and_social_science": 1445.2366609653864, + "industry_mathematical": 1433.864977325182, + "industry_medicine_and_healthcare": 1467.9293986341672, + "industry_software_and_it_services": 1437.2197153613595, + "industry_writing_and_literature_and_language": 1394.3013329580008, + "instruction_following": 1400.2928803275142, + "korean": 1342.5056275682339, + "longer_query": 1407.935773508434, + "math": 1426.1523561252393, + "multi_turn": 1414.3198303221197, + "russian": 1403.7707833722263, + "spanish": 1384.531152391267 + }, + "deepseek-v3.2": { + "overall": 1423.585311680368, + "chinese": 1457.2427636406223, + "coding": 1447.5035736374246, + "creative_writing": 1398.4587251579342, + "english": 1435.0245380359638, + "exclude_ties": 1413.206403839811, + "expert": 1434.5201156736553, + "french": 1432.9956340200395, + "german": 1404.348169474684, + "hard_prompts": 1433.4269173571583, + "hard_prompts_english": 1441.8818878952427, + "industry_business_and_management_and_financial_operations": 1408.788112329173, + "industry_entertainment_and_sports_and_media": 1393.326076747824, + "industry_legal_and_government": 1423.1286318330044, + "industry_life_and_physical_and_social_science": 1443.5802983447172, + "industry_mathematical": 1442.933566161709, + "industry_medicine_and_healthcare": 1437.0401255846048, + "industry_software_and_it_services": 1441.7222109795466, + "industry_writing_and_literature_and_language": 1406.082291386989, + "instruction_following": 1411.43200567568, + "japanese": 1364.9405304880265, + "korean": 1374.0672419267657, + "longer_query": 1426.8331721992097, + "math": 1436.5593993120735, + "multi_turn": 1426.6443169417728, + "russian": 1423.299964471608, + "spanish": 1418.7360404854219 + }, + "deepseek-v3.2-exp": { + "overall": 1422.8707388694424, + "chinese": 1463.9553763147853, + "coding": 1433.878074885695, + "creative_writing": 1404.4846119028684, + "english": 1439.5585324126882, + "exclude_ties": 1414.761086938851, + "expert": 1400.6237417915993, + "french": 1439.1251443270921, + "german": 1433.5957757234735, + "hard_prompts": 1430.1834222205216, + "hard_prompts_english": 1437.8117149579211, + "industry_business_and_management_and_financial_operations": 1413.3860690391432, + "industry_entertainment_and_sports_and_media": 1404.5848922517234, + "industry_legal_and_government": 1408.7772717911153, + "industry_life_and_physical_and_social_science": 1427.882383758227, + "industry_mathematical": 1414.1130452427567, + "industry_medicine_and_healthcare": 1429.049792987191, + "industry_software_and_it_services": 1435.3808305651146, + "industry_writing_and_literature_and_language": 1408.959794866962, + "instruction_following": 1402.4605263715114, + "japanese": 1327.509510629651, + "korean": 1362.2723648366855, + "longer_query": 1418.4891197085426, + "math": 1423.7662946950397, + "multi_turn": 1423.496561850392, + "russian": 1403.98451945383, + "spanish": 1441.0657259025368 + }, + "deepseek-v4-flash-thinking": { + "overall": 1422.692028311801, + "chinese": 1473.6091443639061, + "coding": 1438.6948426878803, + "creative_writing": 1392.0945646268246, + "english": 1434.2342986236577, + "exclude_ties": 1411.7817772618148, + "expert": 1435.5087887719408, + "french": 1440.2276332037513, + "german": 1417.0218750353824, + "hard_prompts": 1430.4168459969576, + "hard_prompts_english": 1435.0885228829316, + "industry_business_and_management_and_financial_operations": 1413.2680025623786, + "industry_entertainment_and_sports_and_media": 1392.3834093060361, + "industry_legal_and_government": 1446.9717402550882, + "industry_life_and_physical_and_social_science": 1439.441684459765, + "industry_mathematical": 1430.745099244137, + "industry_medicine_and_healthcare": 1446.1711887089912, + "industry_software_and_it_services": 1440.5091558043312, + "industry_writing_and_literature_and_language": 1406.8944105722524, + "instruction_following": 1415.9639353771715, + "japanese": 1414.0974890736936, + "korean": 1365.0396406429313, + "longer_query": 1423.3392290366332, + "math": 1425.2274218814628, + "multi_turn": 1430.85282319303, + "russian": 1427.0999941149864, + "spanish": 1422.321667267589 + }, + "gpt-5.1": { + "overall": 1422.196027367246, + "chinese": 1475.6872528800259, + "coding": 1436.2329639089512, + "creative_writing": 1405.0752889682467, + "english": 1424.7239907843548, + "exclude_ties": 1411.497536790274, + "expert": 1441.228365323522, + "french": 1436.51027088833, + "german": 1423.462737010055, + "hard_prompts": 1434.237488299353, + "hard_prompts_english": 1435.8718961659363, + "industry_business_and_management_and_financial_operations": 1419.793869104524, + "industry_entertainment_and_sports_and_media": 1395.8128269462943, + "industry_legal_and_government": 1439.8713349985114, + "industry_life_and_physical_and_social_science": 1439.9626987192203, + "industry_mathematical": 1421.063876279318, + "industry_medicine_and_healthcare": 1444.7988198768521, + "industry_software_and_it_services": 1436.6466731121525, + "industry_writing_and_literature_and_language": 1411.3920689399092, + "instruction_following": 1418.06535603645, + "japanese": 1422.2678649628056, + "korean": 1393.250995733992, + "longer_query": 1430.5476378616943, + "math": 1409.9781515844732, + "multi_turn": 1433.2459627959647, + "russian": 1425.3920796066104, + "spanish": 1424.009452428137 + }, + "longcat-flash-chat": { + "overall": 1421.798011057626, + "chinese": 1430.4967919077978, + "coding": 1467.551289979947, + "creative_writing": 1345.922622203712, + "english": 1440.4109830158654, + "exclude_ties": 1410.5437065445635, + "expert": 1437.4144052394022, + "french": 1460.56491191351, + "german": 1415.820975912204, + "hard_prompts": 1432.748942578086, + "hard_prompts_english": 1445.196911283835, + "industry_business_and_management_and_financial_operations": 1428.2495802827673, + "industry_entertainment_and_sports_and_media": 1360.8576107007273, + "industry_legal_and_government": 1415.6496463924957, + "industry_life_and_physical_and_social_science": 1426.7558847391767, + "industry_mathematical": 1441.2846267564414, + "industry_medicine_and_healthcare": 1461.5313847606985, + "industry_software_and_it_services": 1458.6857027550247, + "industry_writing_and_literature_and_language": 1367.5861100378595, + "instruction_following": 1397.4840301586764, + "japanese": 1309.0247077274153, + "korean": 1360.8443387225577, + "longer_query": 1391.1007272803279, + "math": 1440.6336804418002, + "multi_turn": 1402.4089858385476, + "russian": 1387.0556883671713, + "spanish": 1441.9711545740313 + }, + "qwen3-vl-235b-a22b-instruct": { + "overall": 1421.0485403437456, + "chinese": 1459.7355001521978, + "coding": 1440.2799547997975, + "creative_writing": 1370.5783903408912, + "english": 1430.117586220835, + "exclude_ties": 1408.9359470606287, + "expert": 1446.7116564039509, + "french": 1460.380209786733, + "german": 1423.1920692796837, + "hard_prompts": 1430.1376733226093, + "hard_prompts_english": 1440.3272569782846, + "industry_business_and_management_and_financial_operations": 1431.7052613434078, + "industry_entertainment_and_sports_and_media": 1371.568309631259, + "industry_legal_and_government": 1445.698572220669, + "industry_life_and_physical_and_social_science": 1443.3829532837485, + "industry_mathematical": 1420.0040405583268, + "industry_medicine_and_healthcare": 1435.4318188591556, + "industry_software_and_it_services": 1443.6589106590884, + "industry_writing_and_literature_and_language": 1391.9327356241856, + "instruction_following": 1408.9728619208274, + "japanese": 1382.6945807189554, + "korean": 1399.8631087731708, + "longer_query": 1422.545490968716, + "math": 1424.015472756927, + "multi_turn": 1428.9122319537894, + "russian": 1402.8816889125374, + "spanish": 1408.6138230359725 + }, + "kimi-k2.5-instant": { + "overall": 1420.7102069946668, + "chinese": 1460.784920811445, + "coding": 1484.553248544842, + "creative_writing": 1380.5198865548562, + "english": 1433.0257885690478, + "exclude_ties": 1409.440641431613, + "expert": 1441.0365882122137, + "french": 1405.0652195824407, + "german": 1414.6523995150724, + "hard_prompts": 1444.1116772748833, + "hard_prompts_english": 1448.930428524859, + "industry_business_and_management_and_financial_operations": 1404.9501027957278, + "industry_entertainment_and_sports_and_media": 1391.0899653708273, + "industry_legal_and_government": 1430.5274641268034, + "industry_life_and_physical_and_social_science": 1430.5738072368624, + "industry_mathematical": 1441.2391790025276, + "industry_medicine_and_healthcare": 1407.7653663660003, + "industry_software_and_it_services": 1470.8312024772883, + "industry_writing_and_literature_and_language": 1397.0967120366918, + "instruction_following": 1431.639440510019, + "korean": 1377.9947233986704, + "longer_query": 1437.7384152009774, + "math": 1440.9322274358356, + "multi_turn": 1427.23452446699, + "russian": 1403.9684138040031, + "spanish": 1444.8141005460045 + }, + "gpt-5.5-instant": { + "overall": 1420.5193806661068, + "chinese": 1460.6379006742384, + "coding": 1436.060942700293, + "creative_writing": 1419.2772478082334, + "english": 1420.909624746499, + "exclude_ties": 1409.7393067510334, + "expert": 1412.9155522240044, + "french": 1422.4195301451846, + "german": 1401.8383983960498, + "hard_prompts": 1427.122043526428, + "hard_prompts_english": 1425.0686808500063, + "industry_business_and_management_and_financial_operations": 1408.525578899842, + "industry_entertainment_and_sports_and_media": 1409.2889969222608, + "industry_legal_and_government": 1418.8335991490499, + "industry_life_and_physical_and_social_science": 1427.3631068576135, + "industry_mathematical": 1416.2707021455285, + "industry_medicine_and_healthcare": 1415.3225129452017, + "industry_software_and_it_services": 1433.6574570266434, + "industry_writing_and_literature_and_language": 1424.9742639467677, + "instruction_following": 1407.4644677240412, + "japanese": 1415.6454391849634, + "korean": 1395.5865920563956, + "longer_query": 1421.7461171307589, + "math": 1426.518493831577, + "multi_turn": 1434.131192150662, + "russian": 1430.802919521012, + "spanish": 1424.728231913612 + }, + "amazon-nova-experimental-chat-12-10": { + "overall": 1420.1052463267047, + "chinese": 1449.3841189274212, + "coding": 1429.013330839862, + "creative_writing": 1351.8090842679762, + "english": 1440.1832716191957, + "exclude_ties": 1409.0520466771172, + "expert": 1413.3946437237998, + "hard_prompts": 1428.8821010257232, + "hard_prompts_english": 1448.8717092611275, + "industry_business_and_management_and_financial_operations": 1423.939591521987, + "industry_entertainment_and_sports_and_media": 1345.93303702418, + "industry_legal_and_government": 1434.4675279369712, + "industry_life_and_physical_and_social_science": 1446.7429693445888, + "industry_medicine_and_healthcare": 1467.7410640731214, + "industry_software_and_it_services": 1445.534148911199, + "industry_writing_and_literature_and_language": 1384.7702372363551, + "instruction_following": 1393.295468092594, + "longer_query": 1388.52898368448, + "math": 1420.271327521299, + "multi_turn": 1405.401816455269, + "russian": 1409.1829118925027 + }, + "deepseek-v3.1-terminus-thinking": { + "overall": 1419.6324953485444, + "coding": 1429.0683007590826, + "creative_writing": 1389.6113443401061, + "english": 1430.0572656824534, + "exclude_ties": 1406.8384067419051, + "hard_prompts": 1427.1416471546538, + "hard_prompts_english": 1432.9375495267923, + "industry_business_and_management_and_financial_operations": 1413.291847958619, + "industry_entertainment_and_sports_and_media": 1387.9643600224597, + "industry_legal_and_government": 1458.0879169718182, + "industry_life_and_physical_and_social_science": 1450.660816881976, + "industry_medicine_and_healthcare": 1467.1277943157902, + "industry_software_and_it_services": 1426.6391154328605, + "industry_writing_and_literature_and_language": 1409.1389988446847, + "instruction_following": 1405.5717316269302, + "longer_query": 1420.8778795538256, + "math": 1408.2891199091612, + "multi_turn": 1414.4529944191256, + "russian": 1434.1873458833415 + }, + "deepseek-v3.1": { + "overall": 1419.3236906167401, + "chinese": 1458.3483541674873, + "coding": 1414.5848703036004, + "creative_writing": 1388.3139100785274, + "english": 1430.523763672428, + "exclude_ties": 1405.4219045951913, + "expert": 1403.2083287782798, + "french": 1456.7985755983732, + "german": 1409.300391337692, + "hard_prompts": 1415.3519324876668, + "hard_prompts_english": 1418.296000353332, + "industry_business_and_management_and_financial_operations": 1405.9843227920887, + "industry_entertainment_and_sports_and_media": 1379.70259987216, + "industry_legal_and_government": 1418.2713943388228, + "industry_life_and_physical_and_social_science": 1432.9950790815585, + "industry_mathematical": 1415.3670411181943, + "industry_medicine_and_healthcare": 1432.1044874915383, + "industry_software_and_it_services": 1428.1746927060294, + "industry_writing_and_literature_and_language": 1397.0611777459724, + "instruction_following": 1388.8516914127626, + "japanese": 1367.5452029283908, + "korean": 1331.9314559865886, + "longer_query": 1400.4275510344203, + "math": 1420.0349536248434, + "multi_turn": 1401.4944296815231, + "russian": 1400.5642366109548, + "spanish": 1411.501876372987 + }, + "deepseek-v3.2-thinking": { + "overall": 1419.2080533423075, + "chinese": 1453.0997222721842, + "coding": 1452.1275889088558, + "creative_writing": 1389.0027723577662, + "english": 1440.9737402507396, + "exclude_ties": 1405.873966953379, + "expert": 1430.0293613409006, + "french": 1420.094056080983, + "german": 1407.8693271029053, + "hard_prompts": 1429.5541816215728, + "hard_prompts_english": 1446.9390093628188, + "industry_business_and_management_and_financial_operations": 1407.5281637348685, + "industry_entertainment_and_sports_and_media": 1387.2402720479517, + "industry_legal_and_government": 1420.9259916725798, + "industry_life_and_physical_and_social_science": 1427.4765506244503, + "industry_mathematical": 1433.1880004160985, + "industry_medicine_and_healthcare": 1421.1113900412183, + "industry_software_and_it_services": 1447.9043936308783, + "industry_writing_and_literature_and_language": 1396.1507926074878, + "instruction_following": 1407.2927944733167, + "japanese": 1380.4114171118579, + "korean": 1357.5779286563688, + "longer_query": 1424.3475087646611, + "math": 1418.379893501432, + "multi_turn": 1421.703553016871, + "russian": 1400.5174874783706, + "spanish": 1411.381272831452 + }, + "qwen3-235b-a22b-instruct-2507": { + "overall": 1419.0976145665327, + "chinese": 1465.5870884083763, + "coding": 1444.4790259783738, + "creative_writing": 1376.2980652520914, + "english": 1424.3111255022834, + "exclude_ties": 1407.473479083289, + "expert": 1443.5434501448954, + "french": 1448.5029822666886, + "german": 1435.634349640221, + "hard_prompts": 1433.8276837587275, + "hard_prompts_english": 1435.8896815425026, + "industry_business_and_management_and_financial_operations": 1418.1167887035242, + "industry_entertainment_and_sports_and_media": 1374.9898098940073, + "industry_legal_and_government": 1429.4609975918272, + "industry_life_and_physical_and_social_science": 1437.579326361909, + "industry_mathematical": 1441.1494984523742, + "industry_medicine_and_healthcare": 1445.4515931561289, + "industry_software_and_it_services": 1445.7648621768606, + "industry_writing_and_literature_and_language": 1390.4928703027815, + "instruction_following": 1408.043815440576, + "japanese": 1388.1884612738759, + "korean": 1393.03735790366, + "longer_query": 1426.7377257000005, + "math": 1431.6442887918834, + "multi_turn": 1432.5535864237133, + "russian": 1411.7916679766413, + "spanish": 1426.1756870679026 + }, + "qwen3-next-80b-a3b-instruct": { + "overall": 1418.7752135771811, + "chinese": 1472.9568910831592, + "coding": 1441.1399778538494, + "creative_writing": 1335.5533040124715, + "english": 1424.8297369213817, + "exclude_ties": 1406.335675510209, + "expert": 1409.4221773128677, + "french": 1417.5812060072353, + "german": 1419.8867364709802, + "hard_prompts": 1428.446147734767, + "hard_prompts_english": 1433.5314980637645, + "industry_business_and_management_and_financial_operations": 1422.0476407646247, + "industry_entertainment_and_sports_and_media": 1352.0032632620666, + "industry_legal_and_government": 1435.0276602820938, + "industry_life_and_physical_and_social_science": 1434.8653741444487, + "industry_mathematical": 1433.9229045870272, + "industry_medicine_and_healthcare": 1455.890853764794, + "industry_software_and_it_services": 1449.0609973385792, + "industry_writing_and_literature_and_language": 1356.3978729266387, + "instruction_following": 1389.3265798127384, + "japanese": 1404.365540439503, + "korean": 1359.8828206172716, + "longer_query": 1402.8211320358696, + "math": 1441.5015655871669, + "multi_turn": 1417.926727113949, + "russian": 1400.3186743665976, + "spanish": 1448.377452463392 + }, + "claude-opus-4-1-20250805-thinking-16k": { + "overall": 1418.409133870016, + "chinese": 1420.226939896639, + "coding": 1480.4203855712035, + "creative_writing": 1411.5336743030189, + "english": 1434.1640754752475, + "exclude_ties": 1405.8294080085514, + "expert": 1439.1616756198794, + "french": 1436.2658955360341, + "german": 1414.2887802077537, + "hard_prompts": 1442.6568767906901, + "hard_prompts_english": 1454.1824481083293, + "industry_business_and_management_and_financial_operations": 1404.128732423362, + "industry_entertainment_and_sports_and_media": 1395.2798837926462, + "industry_legal_and_government": 1418.0684695248133, + "industry_life_and_physical_and_social_science": 1421.5041669627908, + "industry_mathematical": 1431.4296644564506, + "industry_medicine_and_healthcare": 1414.1447749493088, + "industry_software_and_it_services": 1455.6120905640764, + "industry_writing_and_literature_and_language": 1417.2301847549693, + "instruction_following": 1436.890895001783, + "japanese": 1379.471167925967, + "korean": 1379.967397903076, + "longer_query": 1455.800499359676, + "math": 1430.319766888593, + "multi_turn": 1444.921495296031, + "russian": 1412.864117828573, + "spanish": 1443.987219193568 + }, + "deepseek-v3.1-terminus": { + "overall": 1417.7111066474727, + "coding": 1406.630470275648, + "creative_writing": 1406.7494722240324, + "english": 1422.703399939655, + "exclude_ties": 1402.741621362435, + "hard_prompts": 1406.0552234989848, + "hard_prompts_english": 1409.3257587577632, + "industry_business_and_management_and_financial_operations": 1386.7582776510262, + "industry_entertainment_and_sports_and_media": 1380.4513693708964, + "industry_legal_and_government": 1407.6813523857427, + "industry_life_and_physical_and_social_science": 1451.5020792429943, + "industry_medicine_and_healthcare": 1447.2511347778839, + "industry_software_and_it_services": 1411.5927345567911, + "industry_writing_and_literature_and_language": 1409.054225654644, + "instruction_following": 1379.427825138919, + "longer_query": 1395.7822756373403, + "math": 1398.6377022620322, + "multi_turn": 1387.9020525260146, + "russian": 1426.508281523235 + }, + "qwen3.5-122b-a10b": { + "overall": 1417.5662898054597, + "chinese": 1467.2841130770057, + "coding": 1431.780401102506, + "creative_writing": 1370.1665872795297, + "english": 1429.000677977339, + "exclude_ties": 1403.1984891179425, + "expert": 1432.3563982092123, + "french": 1438.0018406206518, + "german": 1414.313074850898, + "hard_prompts": 1420.0905554545532, + "hard_prompts_english": 1429.592505397794, + "industry_business_and_management_and_financial_operations": 1413.435000213368, + "industry_entertainment_and_sports_and_media": 1366.8730609555478, + "industry_legal_and_government": 1421.662681974643, + "industry_life_and_physical_and_social_science": 1432.2276297386722, + "industry_mathematical": 1431.3916977365593, + "industry_medicine_and_healthcare": 1428.566926607263, + "industry_software_and_it_services": 1436.4100762584164, + "industry_writing_and_literature_and_language": 1389.5110588130462, + "instruction_following": 1396.6242486052686, + "japanese": 1363.4034128273815, + "korean": 1357.9996791192698, + "longer_query": 1407.1377402964524, + "math": 1426.2676039510495, + "multi_turn": 1413.1255985488838, + "russian": 1393.2524226895061, + "spanish": 1427.478212048584 + }, + "gemini-2.5-flash": { + "overall": 1417.5648721294945, + "chinese": 1452.8000113789078, + "coding": 1423.401478679422, + "creative_writing": 1402.2489067470942, + "english": 1418.2628712511332, + "exclude_ties": 1404.8880203935635, + "expert": 1426.3256761119571, + "french": 1433.7642751844564, + "german": 1417.9197895145264, + "hard_prompts": 1422.9410476447495, + "hard_prompts_english": 1419.8423266359846, + "industry_business_and_management_and_financial_operations": 1403.2470269719274, + "industry_entertainment_and_sports_and_media": 1395.109979582372, + "industry_legal_and_government": 1432.9494391499125, + "industry_life_and_physical_and_social_science": 1433.5118279940286, + "industry_mathematical": 1421.9680594535819, + "industry_medicine_and_healthcare": 1429.3444176956766, + "industry_software_and_it_services": 1425.6850765386248, + "industry_writing_and_literature_and_language": 1407.2830704301807, + "instruction_following": 1405.612695469584, + "japanese": 1399.1044022100739, + "korean": 1390.2478998988872, + "longer_query": 1420.6409031932458, + "math": 1410.731388792275, + "multi_turn": 1407.5077878130062, + "russian": 1417.4959240487103, + "spanish": 1418.30676272539 + }, + "claude-opus-4-1-20250805": { + "overall": 1417.4013916836388, + "chinese": 1424.7236458608602, + "coding": 1472.9450494181185, + "creative_writing": 1410.5212253532493, + "english": 1428.6921517375063, + "exclude_ties": 1403.7593370647376, + "expert": 1425.0049752348018, + "french": 1428.9752556628944, + "german": 1402.8831955116148, + "hard_prompts": 1441.6082049548133, + "hard_prompts_english": 1449.6092910355474, + "industry_business_and_management_and_financial_operations": 1404.7271728953172, + "industry_entertainment_and_sports_and_media": 1397.4943580658364, + "industry_legal_and_government": 1416.861504146776, + "industry_life_and_physical_and_social_science": 1416.1410650132175, + "industry_mathematical": 1425.7779809674425, + "industry_medicine_and_healthcare": 1413.9429085751094, + "industry_software_and_it_services": 1450.7386499478223, + "industry_writing_and_literature_and_language": 1417.2583825286147, + "instruction_following": 1433.915521823988, + "japanese": 1368.9271841236437, + "korean": 1363.3242671445778, + "longer_query": 1445.247384649006, + "math": 1421.68395242177, + "multi_turn": 1442.0510386547512, + "russian": 1422.4545099598454, + "spanish": 1443.7311590955514 + }, + "gpt-4.5-preview-2025-02-27": { + "overall": 1417.3740415750724, + "chinese": 1421.107685918202, + "coding": 1397.067054269768, + "creative_writing": 1394.4744763988033, + "english": 1419.367913690872, + "exclude_ties": 1411.2240800354016, + "expert": 1393.840203697448, + "french": 1420.009324894696, + "german": 1457.0790597908717, + "hard_prompts": 1403.7001091391132, + "hard_prompts_english": 1407.3283612765977, + "industry_business_and_management_and_financial_operations": 1385.6798115697534, + "industry_entertainment_and_sports_and_media": 1399.6822189882143, + "industry_legal_and_government": 1433.464277632126, + "industry_life_and_physical_and_social_science": 1420.4469759616338, + "industry_mathematical": 1410.0655204825941, + "industry_medicine_and_healthcare": 1409.9466501586396, + "industry_software_and_it_services": 1412.9275660430624, + "industry_writing_and_literature_and_language": 1411.5160656556216, + "instruction_following": 1404.1633115645466, + "japanese": 1417.0161346268128, + "korean": 1392.2111285522326, + "longer_query": 1405.890076811113, + "math": 1411.7803246108813, + "multi_turn": 1443.766357357088, + "russian": 1418.099704600647 + }, + "gpt-5.2-high": { + "overall": 1416.9118039038804, + "chinese": 1455.6599982670305, + "coding": 1443.665770126183, + "creative_writing": 1376.309815228957, + "english": 1421.8928479939455, + "exclude_ties": 1403.6494166325883, + "expert": 1448.696849527345, + "french": 1449.3567515963625, + "german": 1444.4648590128809, + "hard_prompts": 1429.0508164117375, + "hard_prompts_english": 1428.7025344271863, + "industry_business_and_management_and_financial_operations": 1415.9531426936378, + "industry_entertainment_and_sports_and_media": 1374.7546254233619, + "industry_legal_and_government": 1427.3552619824684, + "industry_life_and_physical_and_social_science": 1423.4672064561635, + "industry_mathematical": 1442.7719001003625, + "industry_medicine_and_healthcare": 1423.3144149891818, + "industry_software_and_it_services": 1435.5559350713668, + "industry_writing_and_literature_and_language": 1392.2294409564226, + "instruction_following": 1408.9222340005467, + "japanese": 1423.5850363722523, + "korean": 1374.360820906922, + "longer_query": 1410.0541378564278, + "math": 1442.8929335412852, + "multi_turn": 1420.677293074105, + "russian": 1411.1841740651137, + "spanish": 1411.858048010666 + }, + "deepseek-v3.1-thinking": { + "overall": 1416.8642990041321, + "chinese": 1470.2855622306265, + "coding": 1418.60627934422, + "creative_writing": 1403.0254956756248, + "english": 1431.3811649305942, + "exclude_ties": 1403.4458854779084, + "expert": 1402.16899946876, + "french": 1386.5772823307113, + "german": 1409.7491469896827, + "hard_prompts": 1417.0201570858546, + "hard_prompts_english": 1423.7450448253835, + "industry_business_and_management_and_financial_operations": 1408.2684572711066, + "industry_entertainment_and_sports_and_media": 1395.898080795849, + "industry_legal_and_government": 1416.8919714575381, + "industry_life_and_physical_and_social_science": 1428.374300791579, + "industry_mathematical": 1435.2071550672933, + "industry_medicine_and_healthcare": 1441.2568047396423, + "industry_software_and_it_services": 1427.973238407652, + "industry_writing_and_literature_and_language": 1405.8822834272482, + "instruction_following": 1402.33213064557, + "japanese": 1367.3450878474107, + "korean": 1337.406658945932, + "longer_query": 1421.8245683202938, + "math": 1413.3691170745783, + "multi_turn": 1407.0131897719734, + "russian": 1401.1659770509766, + "spanish": 1408.0612061477184 + }, + "gemini-3.1-flash-lite-preview": { + "overall": 1416.5774402836153, + "chinese": 1457.981241702526, + "coding": 1400.2082558796956, + "creative_writing": 1400.7053142462717, + "english": 1414.9984137852734, + "exclude_ties": 1402.7997371349904, + "expert": 1402.3316588606947, + "french": 1419.2422590145998, + "german": 1441.2998229994648, + "hard_prompts": 1406.6253427463826, + "hard_prompts_english": 1401.6789209443025, + "industry_business_and_management_and_financial_operations": 1390.6102025396804, + "industry_entertainment_and_sports_and_media": 1384.9893408415774, + "industry_legal_and_government": 1413.451982318823, + "industry_life_and_physical_and_social_science": 1429.2821514679813, + "industry_mathematical": 1419.4391148610096, + "industry_medicine_and_healthcare": 1429.870307210782, + "industry_software_and_it_services": 1412.904341114501, + "industry_writing_and_literature_and_language": 1409.3474305167006, + "instruction_following": 1379.0745309313934, + "japanese": 1414.4977804312816, + "korean": 1409.4163195224685, + "longer_query": 1394.7599397540728, + "math": 1428.2901454220585, + "multi_turn": 1420.60198488254, + "russian": 1417.3808550072192, + "spanish": 1430.129302638635 + }, + "amazon-nova-experimental-chat-11-10": { + "overall": 1416.5066323559738, + "chinese": 1468.382699120217, + "coding": 1431.7993921463449, + "creative_writing": 1351.0258230437296, + "english": 1422.955980521635, + "exclude_ties": 1403.805455732724, + "expert": 1436.3209982870135, + "french": 1448.29109248817, + "german": 1395.386517298628, + "hard_prompts": 1422.4381855214936, + "hard_prompts_english": 1425.890275064925, + "industry_business_and_management_and_financial_operations": 1418.8645435527194, + "industry_entertainment_and_sports_and_media": 1365.6561636467336, + "industry_legal_and_government": 1430.4939229955094, + "industry_life_and_physical_and_social_science": 1446.1032823157427, + "industry_mathematical": 1427.0536741908263, + "industry_medicine_and_healthcare": 1449.3156312913006, + "industry_software_and_it_services": 1440.8541897755813, + "industry_writing_and_literature_and_language": 1372.2721483818623, + "instruction_following": 1387.8130819342764, + "japanese": 1365.038039516859, + "korean": 1383.5729381784365, + "longer_query": 1395.971691840252, + "math": 1431.7651428545971, + "multi_turn": 1380.446536282687, + "russian": 1394.7853621441698, + "spanish": 1432.4510615179092 + }, + "gpt-5.4-mini-high": { + "overall": 1415.209611864676, + "chinese": 1458.8037872055716, + "coding": 1440.9007038046777, + "creative_writing": 1372.9746929169555, + "english": 1413.7368635491111, + "exclude_ties": 1401.3469477329904, + "expert": 1441.6386006351183, + "french": 1428.8485817464718, + "german": 1406.566396384383, + "hard_prompts": 1429.081095699674, + "hard_prompts_english": 1423.1859216146559, + "industry_business_and_management_and_financial_operations": 1419.2920697669222, + "industry_entertainment_and_sports_and_media": 1376.6501272220898, + "industry_legal_and_government": 1420.318690853849, + "industry_life_and_physical_and_social_science": 1420.7002227452997, + "industry_mathematical": 1435.4172867911793, + "industry_medicine_and_healthcare": 1398.4283556582104, + "industry_software_and_it_services": 1436.295089932425, + "industry_writing_and_literature_and_language": 1394.442028740343, + "instruction_following": 1405.558181477524, + "japanese": 1387.5993836693287, + "korean": 1365.7096860582346, + "longer_query": 1408.294948529095, + "math": 1418.3097170678582, + "multi_turn": 1434.9065886420283, + "russian": 1422.220870195106, + "spanish": 1408.752251779346 + }, + "kimi-k2-thinking-turbo": { + "overall": 1414.2201372452453, + "chinese": 1455.2927918862317, + "coding": 1453.8223521974676, + "creative_writing": 1376.1789297255095, + "english": 1428.600201017702, + "exclude_ties": 1399.7621911323663, + "expert": 1438.1430329960099, + "french": 1428.7275380546455, + "german": 1394.4234594914485, + "hard_prompts": 1427.134072408441, + "hard_prompts_english": 1435.3237310480965, + "industry_business_and_management_and_financial_operations": 1398.22599067673, + "industry_entertainment_and_sports_and_media": 1377.7138306545712, + "industry_legal_and_government": 1416.8871235571596, + "industry_life_and_physical_and_social_science": 1420.5150356753406, + "industry_mathematical": 1441.2366096520173, + "industry_medicine_and_healthcare": 1419.1409584379037, + "industry_software_and_it_services": 1445.3197921301173, + "industry_writing_and_literature_and_language": 1380.4507263145151, + "instruction_following": 1401.901819894684, + "japanese": 1362.886166163012, + "korean": 1331.456508394574, + "longer_query": 1414.513259099256, + "math": 1432.317620487137, + "multi_turn": 1412.8679003813254, + "russian": 1389.889239039451, + "spanish": 1410.3953238041738 + }, + "qwen3-235b-a22b-thinking-2507": { + "overall": 1413.8706660554603, + "chinese": 1472.7097084801633, + "coding": 1423.4126341227538, + "creative_writing": 1386.4166417199522, + "english": 1426.900685186842, + "exclude_ties": 1399.944440768685, + "expert": 1456.807821285528, + "german": 1404.004904338273, + "hard_prompts": 1415.9710197115323, + "hard_prompts_english": 1430.0216045313687, + "industry_business_and_management_and_financial_operations": 1411.484422950588, + "industry_entertainment_and_sports_and_media": 1370.4207082861537, + "industry_legal_and_government": 1407.7883364848399, + "industry_life_and_physical_and_social_science": 1431.7599565774756, + "industry_mathematical": 1435.6264430179338, + "industry_medicine_and_healthcare": 1450.322998176284, + "industry_software_and_it_services": 1430.1559321339096, + "industry_writing_and_literature_and_language": 1387.8465422621148, + "instruction_following": 1385.1143905381423, + "japanese": 1383.4900174418644, + "korean": 1360.5881917424813, + "longer_query": 1397.856367335275, + "math": 1412.5108290976757, + "multi_turn": 1406.2595799932856, + "russian": 1398.7732014353928, + "spanish": 1390.0457857406393 + }, + "qwen3-max-2025-09-23": { + "overall": 1413.0314410566318, + "chinese": 1426.408112187219, + "coding": 1439.597095256102, + "creative_writing": 1382.4387251598253, + "english": 1420.433651102799, + "exclude_ties": 1398.118710064139, + "expert": 1398.3345663747873, + "german": 1437.7292026776395, + "hard_prompts": 1425.0069803402105, + "hard_prompts_english": 1434.1047124747033, + "industry_business_and_management_and_financial_operations": 1395.9655675897566, + "industry_entertainment_and_sports_and_media": 1379.5628873290634, + "industry_legal_and_government": 1406.3422414675977, + "industry_life_and_physical_and_social_science": 1413.6616162527755, + "industry_mathematical": 1412.3058829317893, + "industry_medicine_and_healthcare": 1406.4274552419038, + "industry_software_and_it_services": 1437.748715603109, + "industry_writing_and_literature_and_language": 1388.8663088596056, + "instruction_following": 1398.9935852705612, + "korean": 1356.7976044743493, + "longer_query": 1412.2257883383263, + "math": 1437.9022686288106, + "multi_turn": 1436.931517697866, + "russian": 1418.2263005639081, + "spanish": 1397.1364887853135 + }, + "mimo-v2-flash (non-thinking)": { + "overall": 1411.5651811997964, + "chinese": 1463.5317911502823, + "coding": 1440.3009090319647, + "creative_writing": 1379.1528651811948, + "english": 1430.1730989648254, + "exclude_ties": 1396.9354003102605, + "expert": 1424.7983301926054, + "french": 1416.6594389951124, + "german": 1381.957259474611, + "hard_prompts": 1420.1963640715373, + "hard_prompts_english": 1435.2105871630267, + "industry_business_and_management_and_financial_operations": 1400.7972313962516, + "industry_entertainment_and_sports_and_media": 1376.4975307998081, + "industry_legal_and_government": 1412.77677048696, + "industry_life_and_physical_and_social_science": 1422.2846556324107, + "industry_mathematical": 1415.246428764367, + "industry_medicine_and_healthcare": 1417.0027313784376, + "industry_software_and_it_services": 1442.088272697584, + "industry_writing_and_literature_and_language": 1386.3836697144995, + "instruction_following": 1391.5617471259172, + "japanese": 1338.746961875843, + "korean": 1358.6779822861292, + "longer_query": 1408.2561331617094, + "math": 1396.6670280090334, + "multi_turn": 1403.2742178158217, + "russian": 1387.1604342696098, + "spanish": 1417.5130923552263 + }, + "gpt-5.2": { + "overall": 1411.442555651884, + "chinese": 1455.818160283588, + "coding": 1431.9236400867533, + "creative_writing": 1375.1210294866582, + "english": 1411.3636513398174, + "exclude_ties": 1396.9821463114088, + "expert": 1430.3023253226631, + "french": 1444.013202910601, + "german": 1417.5453831585812, + "hard_prompts": 1425.9016022029398, + "hard_prompts_english": 1420.9111111646293, + "industry_business_and_management_and_financial_operations": 1408.8122583304462, + "industry_entertainment_and_sports_and_media": 1379.6479731042775, + "industry_legal_and_government": 1428.5615793403706, + "industry_life_and_physical_and_social_science": 1425.6216373039251, + "industry_mathematical": 1416.5621765949256, + "industry_medicine_and_healthcare": 1428.1681991548996, + "industry_software_and_it_services": 1430.1191606483255, + "industry_writing_and_literature_and_language": 1390.9870727480998, + "instruction_following": 1402.2855043786014, + "japanese": 1385.9021237315458, + "korean": 1374.0480133928306, + "longer_query": 1412.240071315077, + "math": 1418.581388126864, + "multi_turn": 1420.358857503838, + "russian": 1420.1156851136407, + "spanish": 1399.575237234654 + }, + "grok-4-0709": { + "overall": 1409.8832004340782, + "chinese": 1430.427732544091, + "coding": 1409.2444593367197, + "creative_writing": 1398.5700322988343, + "english": 1418.4246000108594, + "exclude_ties": 1393.672257797035, + "expert": 1418.047006931603, + "french": 1426.5274859000956, + "german": 1426.2524095482845, + "hard_prompts": 1408.5294446857267, + "hard_prompts_english": 1412.7764223625757, + "industry_business_and_management_and_financial_operations": 1387.8827192247281, + "industry_entertainment_and_sports_and_media": 1392.2019711028447, + "industry_legal_and_government": 1428.3994893675742, + "industry_life_and_physical_and_social_science": 1435.770096832797, + "industry_mathematical": 1424.4312878186065, + "industry_medicine_and_healthcare": 1435.7783216141434, + "industry_software_and_it_services": 1410.446196077345, + "industry_writing_and_literature_and_language": 1398.910447972512, + "instruction_following": 1387.1347922921768, + "japanese": 1387.7939179183697, + "korean": 1365.4584722801765, + "longer_query": 1409.9814219721584, + "math": 1424.05806229203, + "multi_turn": 1415.5967292993487, + "russian": 1412.8853195659867, + "spanish": 1412.841102319994 + }, + "o3-2025-04-16": { + "overall": 1409.379066607526, + "chinese": 1437.1959341669924, + "coding": 1408.0436928315228, + "creative_writing": 1359.5857364081276, + "english": 1414.2268870536864, + "exclude_ties": 1392.5625182426234, + "expert": 1399.8393271097718, + "french": 1446.6437175170252, + "german": 1419.5867777171504, + "hard_prompts": 1401.9806804808177, + "hard_prompts_english": 1406.3084981103, + "industry_business_and_management_and_financial_operations": 1392.3509414803948, + "industry_entertainment_and_sports_and_media": 1373.661979768718, + "industry_legal_and_government": 1428.2604149543638, + "industry_life_and_physical_and_social_science": 1428.1085469528905, + "industry_mathematical": 1422.3249404865912, + "industry_medicine_and_healthcare": 1448.029770685736, + "industry_software_and_it_services": 1414.4157960603497, + "industry_writing_and_literature_and_language": 1371.8520286028374, + "instruction_following": 1367.6758973882236, + "japanese": 1405.7893597738387, + "korean": 1369.7453724269167, + "longer_query": 1370.7878046397448, + "math": 1424.611471293845, + "multi_turn": 1404.5324504093846, + "russian": 1408.2563017415216, + "spanish": 1383.4636345546082 + }, + "grok-4-fast-chat": { + "overall": 1409.199949177996, + "chinese": 1475.5098481479329, + "coding": 1430.3282110392447, + "creative_writing": 1389.687651564458, + "english": 1416.2501505489324, + "exclude_ties": 1393.4249529065944, + "expert": 1404.6359897976836, + "hard_prompts": 1413.7791043429625, + "hard_prompts_english": 1425.2752557714484, + "industry_business_and_management_and_financial_operations": 1383.0986497024298, + "industry_entertainment_and_sports_and_media": 1383.2656952863724, + "industry_legal_and_government": 1410.835237792602, + "industry_life_and_physical_and_social_science": 1413.825718478974, + "industry_mathematical": 1433.3340414189604, + "industry_medicine_and_healthcare": 1414.7232740580616, + "industry_software_and_it_services": 1434.4437627409304, + "industry_writing_and_literature_and_language": 1391.7276056763164, + "instruction_following": 1392.6615041367506, + "korean": 1344.00253111743, + "longer_query": 1418.4797363419527, + "math": 1417.374619153377, + "multi_turn": 1415.8223636518226, + "russian": 1382.3872216670293, + "spanish": 1421.1341803175094 + }, + "grok-4.3": { + "overall": 1408.7183303990425, + "chinese": 1443.9647182354302, + "coding": 1426.9518520844854, + "creative_writing": 1397.6903108476117, + "english": 1414.247052735991, + "exclude_ties": 1392.3176248321097, + "expert": 1384.9179207373718, + "french": 1413.901515441789, + "german": 1398.347621879819, + "hard_prompts": 1406.7251324278338, + "hard_prompts_english": 1409.6945164043698, + "industry_business_and_management_and_financial_operations": 1389.3143551591497, + "industry_entertainment_and_sports_and_media": 1386.2432096125103, + "industry_legal_and_government": 1397.3971753729995, + "industry_life_and_physical_and_social_science": 1417.7892197429073, + "industry_mathematical": 1417.621066674241, + "industry_medicine_and_healthcare": 1412.1774834622163, + "industry_software_and_it_services": 1424.5453182045326, + "industry_writing_and_literature_and_language": 1396.8999429555372, + "instruction_following": 1378.59904214357, + "korean": 1368.915794233481, + "longer_query": 1402.6798301893148, + "math": 1398.3659231081356, + "multi_turn": 1427.6445173021052, + "russian": 1423.3523970578967, + "spanish": 1398.2082840861012 + }, + "grok-4-1-fast-reasoning": { + "overall": 1408.4058384868265, + "chinese": 1443.1306312160602, + "coding": 1412.31545263177, + "creative_writing": 1392.017270646775, + "english": 1417.9206347304232, + "exclude_ties": 1391.6748834071582, + "expert": 1403.0983849365234, + "french": 1418.8582553885155, + "german": 1396.9953939188376, + "hard_prompts": 1407.4783359335934, + "hard_prompts_english": 1409.6644001567593, + "industry_business_and_management_and_financial_operations": 1384.1848793318559, + "industry_entertainment_and_sports_and_media": 1380.585824715104, + "industry_legal_and_government": 1401.5687673737161, + "industry_life_and_physical_and_social_science": 1420.7791017178213, + "industry_mathematical": 1404.016998571296, + "industry_medicine_and_healthcare": 1414.3330916964032, + "industry_software_and_it_services": 1418.5026756703471, + "industry_writing_and_literature_and_language": 1385.1614130149123, + "instruction_following": 1375.6130741346499, + "japanese": 1349.0667189756673, + "korean": 1361.109071097796, + "longer_query": 1391.537212849618, + "math": 1406.5348686850036, + "multi_turn": 1391.377526985841, + "russian": 1381.0432370917588, + "spanish": 1411.9944883814837 + }, + "hunyuan-hy3-preview": { + "overall": 1408.0995039639658, + "chinese": 1463.949000303251, + "coding": 1430.6956085624677, + "creative_writing": 1355.3998421057627, + "english": 1413.616410950442, + "exclude_ties": 1390.9353944937634, + "expert": 1428.670508547858, + "french": 1414.2864793890658, + "hard_prompts": 1421.546098958404, + "hard_prompts_english": 1422.059418748566, + "industry_business_and_management_and_financial_operations": 1398.5650557508204, + "industry_entertainment_and_sports_and_media": 1364.3939621822663, + "industry_legal_and_government": 1426.347376545201, + "industry_life_and_physical_and_social_science": 1422.7778456356702, + "industry_mathematical": 1442.2633473792514, + "industry_medicine_and_healthcare": 1413.1398000283561, + "industry_software_and_it_services": 1428.1451600622731, + "industry_writing_and_literature_and_language": 1384.3044962353124, + "instruction_following": 1394.0970892703829, + "longer_query": 1410.9333764730823, + "math": 1421.2504032048573, + "multi_turn": 1414.3036921381652, + "russian": 1404.0212064851053, + "spanish": 1394.2059967835596 + }, + "qwen3.5-27b": { + "overall": 1408.0101616573982, + "chinese": 1473.861605600606, + "coding": 1423.0387557468437, + "creative_writing": 1362.0681444659635, + "english": 1421.7066977828656, + "exclude_ties": 1389.332838857934, + "expert": 1426.1849625528519, + "french": 1401.094216051424, + "german": 1404.2376157042797, + "hard_prompts": 1412.9683461184582, + "hard_prompts_english": 1423.2537733989532, + "industry_business_and_management_and_financial_operations": 1397.0904421234002, + "industry_entertainment_and_sports_and_media": 1358.9065741793856, + "industry_legal_and_government": 1407.3735400163507, + "industry_life_and_physical_and_social_science": 1434.282418170731, + "industry_mathematical": 1441.0707491298142, + "industry_medicine_and_healthcare": 1424.7285594359, + "industry_software_and_it_services": 1428.2192995720936, + "industry_writing_and_literature_and_language": 1379.727216763099, + "instruction_following": 1391.5155776625793, + "japanese": 1367.4761914338687, + "korean": 1360.8876479229307, + "longer_query": 1412.0765596882316, + "math": 1432.1342049583438, + "multi_turn": 1411.7942491846663, + "russian": 1388.0609484252718, + "spanish": 1409.6907198097515 + }, + "gemini-2.5-flash-preview-09-2025": { + "overall": 1406.980721164846, + "chinese": 1451.7912803690142, + "coding": 1401.825990080487, + "creative_writing": 1388.238875413789, + "english": 1411.1269575527226, + "exclude_ties": 1389.406423492688, + "expert": 1424.5531587511762, + "french": 1438.6084851781354, + "german": 1422.5800986348097, + "hard_prompts": 1409.2632225755476, + "hard_prompts_english": 1408.8190074908428, + "industry_business_and_management_and_financial_operations": 1397.771979254042, + "industry_entertainment_and_sports_and_media": 1385.7553040951984, + "industry_legal_and_government": 1428.1596962922404, + "industry_life_and_physical_and_social_science": 1435.4189035540633, + "industry_mathematical": 1421.2665548274879, + "industry_medicine_and_healthcare": 1423.8624462482553, + "industry_software_and_it_services": 1409.3726216064147, + "industry_writing_and_literature_and_language": 1395.1322195581151, + "instruction_following": 1394.3587238674804, + "japanese": 1407.112897983229, + "korean": 1391.0470567803013, + "longer_query": 1405.9072490921603, + "math": 1416.5954088554697, + "multi_turn": 1395.4999817729636, + "russian": 1398.641434752616, + "spanish": 1397.563148133827 + }, + "hunyuan-vision-1.5-thinking": { + "overall": 1406.108556091495, + "coding": 1420.622926641967, + "creative_writing": 1387.7311329470454, + "english": 1416.627417022974, + "exclude_ties": 1387.7246979796187, + "hard_prompts": 1417.7240307290704, + "hard_prompts_english": 1422.0311229907725, + "industry_business_and_management_and_financial_operations": 1413.3210446053204, + "industry_entertainment_and_sports_and_media": 1387.6284918681142, + "industry_life_and_physical_and_social_science": 1409.2219245129645, + "industry_software_and_it_services": 1422.4691461061527, + "industry_writing_and_literature_and_language": 1387.4159106879256, + "instruction_following": 1395.374107265761, + "longer_query": 1400.2000413541402, + "multi_turn": 1420.6809085460682 + }, + "gpt-5-high": { + "overall": 1405.2822469450632, + "chinese": 1424.584611588798, + "coding": 1434.8071667055342, + "creative_writing": 1365.556820531429, + "english": 1408.8283657134882, + "exclude_ties": 1386.1924558412632, + "expert": 1420.2299159145439, + "french": 1406.7858014264104, + "german": 1412.7931654703968, + "hard_prompts": 1414.805989919595, + "hard_prompts_english": 1420.3870636313727, + "industry_business_and_management_and_financial_operations": 1377.9713588696882, + "industry_entertainment_and_sports_and_media": 1378.1946112423102, + "industry_legal_and_government": 1420.271213988567, + "industry_life_and_physical_and_social_science": 1401.0960392224783, + "industry_mathematical": 1404.0224563987929, + "industry_medicine_and_healthcare": 1413.285241176731, + "industry_software_and_it_services": 1419.2955920982863, + "industry_writing_and_literature_and_language": 1379.050775817579, + "instruction_following": 1387.6469796978954, + "japanese": 1396.8885189873095, + "korean": 1365.574932243281, + "longer_query": 1387.448144104665, + "math": 1397.9535117872713, + "multi_turn": 1400.5170785223995, + "russian": 1400.0912164217473, + "spanish": 1378.3089182836977 + }, + "gpt-5-chat": { + "overall": 1403.66215878652, + "chinese": 1420.8124442680664, + "coding": 1400.4834239871811, + "creative_writing": 1368.1715206911979, + "english": 1407.1827990514703, + "exclude_ties": 1384.4143553260174, + "expert": 1407.6120474642348, + "french": 1413.5802937330716, + "german": 1406.338209653178, + "hard_prompts": 1405.2688112807448, + "hard_prompts_english": 1404.0694093655648, + "industry_business_and_management_and_financial_operations": 1398.808155999831, + "industry_entertainment_and_sports_and_media": 1365.372611253008, + "industry_legal_and_government": 1430.0566344978279, + "industry_life_and_physical_and_social_science": 1417.197900074837, + "industry_mathematical": 1397.6622134182007, + "industry_medicine_and_healthcare": 1413.10087240275, + "industry_software_and_it_services": 1409.1612948756958, + "industry_writing_and_literature_and_language": 1385.226137410588, + "instruction_following": 1383.6262701307417, + "japanese": 1405.614978870235, + "korean": 1358.9730368974842, + "longer_query": 1400.6407680684167, + "math": 1408.478923187389, + "multi_turn": 1426.2684296637967, + "russian": 1403.5839978983763, + "spanish": 1386.9444587837445 + }, + "step-3.5-flash": { + "overall": 1403.0275835037587, + "chinese": 1442.1777099719195, + "coding": 1433.5284384110148, + "creative_writing": 1356.0776287835465, + "english": 1418.1672036185498, + "exclude_ties": 1383.5274727175304, + "expert": 1419.6322242007711, + "french": 1428.2355040103853, + "german": 1394.9616061369375, + "hard_prompts": 1410.3055579971115, + "hard_prompts_english": 1422.6960674908985, + "industry_business_and_management_and_financial_operations": 1394.094098824135, + "industry_entertainment_and_sports_and_media": 1357.9287173427456, + "industry_legal_and_government": 1408.5774769058735, + "industry_life_and_physical_and_social_science": 1416.2330971638362, + "industry_mathematical": 1416.1998523637458, + "industry_medicine_and_healthcare": 1409.3203439414601, + "industry_software_and_it_services": 1433.2669852240674, + "industry_writing_and_literature_and_language": 1369.0482399952068, + "instruction_following": 1386.046340110734, + "japanese": 1339.8649728328614, + "korean": 1344.3517968061512, + "longer_query": 1405.1317697905838, + "math": 1404.7124625943632, + "multi_turn": 1405.1379152078118, + "russian": 1384.3475098590498, + "spanish": 1417.3188957720913 + }, + "mimo-v2-omni": { + "overall": 1401.6046313088584, + "coding": 1441.2357849245116, + "creative_writing": 1370.2047632762763, + "english": 1419.2308702460587, + "exclude_ties": 1385.3242921332078, + "expert": 1420.3375961195545, + "hard_prompts": 1422.884707942357, + "hard_prompts_english": 1432.7359912378424, + "industry_business_and_management_and_financial_operations": 1420.1479803069064, + "industry_entertainment_and_sports_and_media": 1369.1871813563712, + "industry_legal_and_government": 1392.8930491245367, + "industry_life_and_physical_and_social_science": 1399.2709583825874, + "industry_software_and_it_services": 1442.322563481546, + "industry_writing_and_literature_and_language": 1375.6181649576288, + "instruction_following": 1396.358503927541, + "longer_query": 1420.7307154376156, + "multi_turn": 1397.1068049699202, + "russian": 1409.579808598273 + }, + "qwen3-vl-235b-a22b-thinking": { + "overall": 1401.0157384719805, + "chinese": 1451.3418598822104, + "coding": 1427.976488169301, + "creative_writing": 1344.0751396406695, + "english": 1417.8176715895268, + "exclude_ties": 1379.9853006707503, + "expert": 1421.6305111051552, + "german": 1421.8745587874027, + "hard_prompts": 1408.0054714605521, + "hard_prompts_english": 1415.2825001741292, + "industry_business_and_management_and_financial_operations": 1392.0529694845827, + "industry_entertainment_and_sports_and_media": 1369.9145508471497, + "industry_legal_and_government": 1409.3320030609607, + "industry_life_and_physical_and_social_science": 1412.2926014020838, + "industry_mathematical": 1437.5866003418114, + "industry_medicine_and_healthcare": 1416.5133921304737, + "industry_software_and_it_services": 1427.0458422678328, + "industry_writing_and_literature_and_language": 1365.0694020152253, + "instruction_following": 1374.9306365240022, + "korean": 1364.6537486465256, + "longer_query": 1393.5613277353823, + "math": 1414.65968805011, + "multi_turn": 1389.7308479998824, + "russian": 1360.0585512124953, + "spanish": 1384.3263403738886 + }, + "minimax-m2.7": { + "overall": 1400.0684104145614, + "chinese": 1423.51367524292, + "coding": 1447.542067393573, + "creative_writing": 1342.075466712713, + "english": 1421.6400051926948, + "exclude_ties": 1380.9059347682498, + "expert": 1433.2102537446397, + "french": 1423.6847345297072, + "german": 1378.456374569328, + "hard_prompts": 1417.3381866114994, + "hard_prompts_english": 1432.544395905819, + "industry_business_and_management_and_financial_operations": 1404.2705291949374, + "industry_entertainment_and_sports_and_media": 1352.7022479423379, + "industry_legal_and_government": 1400.5753202344124, + "industry_life_and_physical_and_social_science": 1402.3491735861635, + "industry_mathematical": 1431.4976059133253, + "industry_medicine_and_healthcare": 1400.6195193793956, + "industry_software_and_it_services": 1442.2174450705545, + "industry_writing_and_literature_and_language": 1364.7855317488393, + "instruction_following": 1401.4900775556473, + "japanese": 1226.1853924213633, + "korean": 1306.908849236382, + "longer_query": 1415.2884959763514, + "math": 1410.6547817711942, + "multi_turn": 1408.8154996589608, + "russian": 1373.1591525130266, + "spanish": 1416.2804480545597 + }, + "hunyuan-t1-20250711": { + "overall": 1399.735305861502, + "chinese": 1431.29568143437, + "coding": 1387.3083915031527, + "creative_writing": 1394.1126796883025, + "english": 1395.2862266809843, + "exclude_ties": 1378.9334333258391, + "expert": 1389.4926328374538, + "hard_prompts": 1398.815380613213, + "hard_prompts_english": 1394.225019726882, + "industry_business_and_management_and_financial_operations": 1373.20468714893, + "industry_entertainment_and_sports_and_media": 1354.633478330973, + "industry_legal_and_government": 1419.4194705610657, + "industry_life_and_physical_and_social_science": 1431.8815954805987, + "industry_mathematical": 1388.6956881489123, + "industry_medicine_and_healthcare": 1452.2016564785642, + "industry_software_and_it_services": 1408.075894903491, + "industry_writing_and_literature_and_language": 1383.0378698289592, + "instruction_following": 1376.3121525014067, + "korean": 1399.7401423726471, + "longer_query": 1383.7559743796594, + "math": 1409.45957398825, + "multi_turn": 1393.412561109144, + "russian": 1385.4187849240047 + }, + "amazon-nova-experimental-chat-26-01-10": { + "overall": 1398.9357060144578, + "coding": 1445.0411473959164, + "creative_writing": 1336.956490907206, + "english": 1415.4940339361515, + "exclude_ties": 1376.4903654053978, + "expert": 1445.0235554841395, + "hard_prompts": 1419.3198555999352, + "hard_prompts_english": 1439.1371934311362, + "industry_business_and_management_and_financial_operations": 1393.6754070031523, + "industry_entertainment_and_sports_and_media": 1348.7838130743144, + "industry_legal_and_government": 1382.8615764571111, + "industry_life_and_physical_and_social_science": 1408.3029111764527, + "industry_medicine_and_healthcare": 1405.7938070788032, + "industry_software_and_it_services": 1439.2852643636027, + "industry_writing_and_literature_and_language": 1354.4841310255892, + "instruction_following": 1385.1077889317673, + "longer_query": 1401.145556197703, + "math": 1402.4937737155126, + "multi_turn": 1381.8657297557793, + "russian": 1393.5515940147618 + }, + "grok-4-fast-reasoning": { + "overall": 1397.502611550541, + "chinese": 1432.793262078857, + "coding": 1418.56173634249, + "creative_writing": 1374.8665018833105, + "english": 1406.8514648694525, + "exclude_ties": 1376.643598387414, + "expert": 1407.612069385253, + "french": 1434.5869309758166, + "german": 1386.6971175612775, + "hard_prompts": 1403.3309177394997, + "hard_prompts_english": 1414.5663971553133, + "industry_business_and_management_and_financial_operations": 1392.4184091388368, + "industry_entertainment_and_sports_and_media": 1366.3930220892203, + "industry_legal_and_government": 1427.414185191892, + "industry_life_and_physical_and_social_science": 1423.3187306256896, + "industry_mathematical": 1399.9918695665697, + "industry_medicine_and_healthcare": 1419.470133726866, + "industry_software_and_it_services": 1412.7222741879734, + "industry_writing_and_literature_and_language": 1379.9232961124062, + "instruction_following": 1381.4977584318638, + "japanese": 1352.8136788487177, + "korean": 1354.1352143137387, + "longer_query": 1410.6477904653311, + "math": 1403.5731665145067, + "multi_turn": 1404.5861783531861, + "russian": 1390.3278400557301, + "spanish": 1410.579680793365 + }, + "qwen3.5-flash": { + "overall": 1397.1972791528028, + "chinese": 1465.5066470304084, + "coding": 1408.5939652119034, + "creative_writing": 1349.4095075478178, + "english": 1404.009868160905, + "exclude_ties": 1375.5483061726213, + "expert": 1409.6848967411786, + "french": 1403.614762445976, + "german": 1386.591114206093, + "hard_prompts": 1402.3069415622172, + "hard_prompts_english": 1407.0110726918608, + "industry_business_and_management_and_financial_operations": 1395.4238929140563, + "industry_entertainment_and_sports_and_media": 1345.4614816305511, + "industry_legal_and_government": 1393.2978461547455, + "industry_life_and_physical_and_social_science": 1415.6512044421538, + "industry_mathematical": 1411.9793283017084, + "industry_medicine_and_healthcare": 1408.3521869848305, + "industry_software_and_it_services": 1415.3738121400368, + "industry_writing_and_literature_and_language": 1370.7856303361586, + "instruction_following": 1370.5279891218622, + "japanese": 1376.243833467052, + "korean": 1359.9457678310034, + "longer_query": 1392.3395453749458, + "math": 1407.2185325202997, + "multi_turn": 1389.3693373722806, + "russian": 1382.87045149807, + "spanish": 1411.8653462923724 + }, + "qwen3.5-35b-a3b": { + "overall": 1396.3181993382923, + "chinese": 1468.6771910457953, + "coding": 1411.3910411112593, + "creative_writing": 1348.264239928617, + "english": 1408.3389039515446, + "exclude_ties": 1374.4552416918937, + "expert": 1407.6346169576216, + "french": 1418.1811427484754, + "german": 1358.3724027562407, + "hard_prompts": 1402.1461467011768, + "hard_prompts_english": 1409.1554343407443, + "industry_business_and_management_and_financial_operations": 1394.7309616350933, + "industry_entertainment_and_sports_and_media": 1348.6522532709, + "industry_legal_and_government": 1398.9953930167878, + "industry_life_and_physical_and_social_science": 1409.5213092199238, + "industry_mathematical": 1408.7947485983948, + "industry_medicine_and_healthcare": 1423.6947543592203, + "industry_software_and_it_services": 1417.846851908168, + "industry_writing_and_literature_and_language": 1366.6795737224923, + "instruction_following": 1382.4127307729943, + "japanese": 1337.8036756102292, + "korean": 1356.9269293542307, + "longer_query": 1393.3030938618858, + "math": 1406.0498103143132, + "multi_turn": 1390.1480130594675, + "russian": 1377.2299394768431, + "spanish": 1393.3116589169772 + }, + "mimo-v2-flash (thinking)": { + "overall": 1395.3493263820815, + "chinese": 1430.6394585834369, + "coding": 1417.8270265639644, + "creative_writing": 1348.6676980092875, + "english": 1410.658615736261, + "exclude_ties": 1371.2350597678653, + "expert": 1403.7931660510014, + "french": 1428.7476674155007, + "german": 1391.5706684886802, + "hard_prompts": 1405.1486595885729, + "hard_prompts_english": 1416.050657988767, + "industry_business_and_management_and_financial_operations": 1387.4558747753854, + "industry_entertainment_and_sports_and_media": 1355.6990168059747, + "industry_legal_and_government": 1405.6213813248482, + "industry_life_and_physical_and_social_science": 1401.5695693840178, + "industry_mathematical": 1389.3285983509234, + "industry_medicine_and_healthcare": 1407.5929724805155, + "industry_software_and_it_services": 1419.995098484175, + "industry_writing_and_literature_and_language": 1363.2595389329729, + "instruction_following": 1376.444019184194, + "korean": 1286.3041438379773, + "longer_query": 1395.0654771463123, + "math": 1378.7413028044584, + "multi_turn": 1378.7363322205213, + "russian": 1372.0384762456335, + "spanish": 1384.8740600036142 + }, + "amazon-nova-experimental-chat-10-20": { + "overall": 1394.3692198880403, + "chinese": 1408.6004995312767, + "coding": 1413.073928050357, + "creative_writing": 1317.5757166600656, + "english": 1413.2270621986684, + "exclude_ties": 1373.8833452271847, + "expert": 1385.7656733652955, + "french": 1427.7576636452661, + "german": 1391.5267047096806, + "hard_prompts": 1396.0994427063338, + "hard_prompts_english": 1411.1328487890628, + "industry_business_and_management_and_financial_operations": 1402.6546788672943, + "industry_entertainment_and_sports_and_media": 1340.1113371557797, + "industry_legal_and_government": 1387.1262919206758, + "industry_life_and_physical_and_social_science": 1426.535657236623, + "industry_mathematical": 1413.4658426754934, + "industry_medicine_and_healthcare": 1406.6530836593986, + "industry_software_and_it_services": 1422.351120081445, + "industry_writing_and_literature_and_language": 1348.9084311709987, + "instruction_following": 1364.2883302513765, + "korean": 1332.0353328386109, + "longer_query": 1369.6623705673182, + "math": 1427.1042909914775, + "multi_turn": 1367.1180761044707, + "russian": 1368.7963948562901, + "spanish": 1366.3504176929816 + }, + "qwen3-235b-a22b-no-thinking": { + "overall": 1393.6856031278687, + "chinese": 1426.3938895388892, + "coding": 1398.91779110512, + "creative_writing": 1358.225608705892, + "english": 1396.2566267349343, + "exclude_ties": 1371.0388819313475, + "expert": 1368.2722774789352, + "french": 1374.1466518430743, + "german": 1381.1413715808474, + "hard_prompts": 1393.9548603705753, + "hard_prompts_english": 1391.7432126521214, + "industry_business_and_management_and_financial_operations": 1391.5705043051198, + "industry_entertainment_and_sports_and_media": 1350.544176739764, + "industry_legal_and_government": 1394.24709085753, + "industry_life_and_physical_and_social_science": 1406.3356356924141, + "industry_mathematical": 1401.4309001113481, + "industry_medicine_and_healthcare": 1407.9160093645778, + "industry_software_and_it_services": 1409.3363371728326, + "industry_writing_and_literature_and_language": 1367.59858058218, + "instruction_following": 1361.0242884661607, + "japanese": 1368.9548499996683, + "korean": 1360.3735617213408, + "longer_query": 1389.7142311985874, + "math": 1398.534645668667, + "multi_turn": 1398.432132544485, + "russian": 1391.9270125400444, + "spanish": 1404.1133917799134 + }, + "claude-haiku-4-5-20251001": { + "overall": 1391.3091248566454, + "chinese": 1414.55934244049, + "coding": 1451.2295116503888, + "creative_writing": 1368.330562185542, + "english": 1411.311619042987, + "exclude_ties": 1368.8927562666952, + "expert": 1439.3363333488023, + "french": 1401.441122344762, + "german": 1369.3121031141977, + "hard_prompts": 1416.616318712152, + "hard_prompts_english": 1432.588798325188, + "industry_business_and_management_and_financial_operations": 1395.4280765450821, + "industry_entertainment_and_sports_and_media": 1362.4404951922106, + "industry_legal_and_government": 1392.7234995870285, + "industry_life_and_physical_and_social_science": 1397.2743959041547, + "industry_mathematical": 1418.952020732446, + "industry_medicine_and_healthcare": 1382.287910150693, + "industry_software_and_it_services": 1434.1968313680457, + "industry_writing_and_literature_and_language": 1382.8253508402722, + "instruction_following": 1411.350413841516, + "japanese": 1319.677276962585, + "korean": 1329.3528872837342, + "longer_query": 1427.9483203312936, + "math": 1388.2245269366294, + "multi_turn": 1407.898553072915, + "russian": 1372.6996200733993, + "spanish": 1417.8320428619918 + }, + "minimax-m2.1-preview": { + "overall": 1391.002489577083, + "chinese": 1428.4971501321197, + "coding": 1422.6872136430352, + "creative_writing": 1362.1713803568591, + "english": 1405.83499179147, + "exclude_ties": 1366.939368461805, + "expert": 1440.632759398303, + "french": 1404.6443546767039, + "german": 1376.7958980926567, + "hard_prompts": 1410.3084546338885, + "hard_prompts_english": 1422.7766415041265, + "industry_business_and_management_and_financial_operations": 1390.4001498712971, + "industry_entertainment_and_sports_and_media": 1362.001524914056, + "industry_legal_and_government": 1411.2330035922087, + "industry_life_and_physical_and_social_science": 1415.8921424977596, + "industry_mathematical": 1419.799477944982, + "industry_medicine_and_healthcare": 1388.4855117607872, + "industry_software_and_it_services": 1415.1315562922018, + "industry_writing_and_literature_and_language": 1372.1895618019153, + "instruction_following": 1398.1987785381873, + "japanese": 1289.0269706039794, + "korean": 1289.3954552771534, + "longer_query": 1417.4847063521008, + "math": 1394.8514648716953, + "multi_turn": 1393.8853333120185, + "russian": 1385.6727508461347, + "spanish": 1401.4758626343253 + }, + "gpt-5.3-chat-latest": { + "overall": 1387.3613694037595, + "chinese": 1423.105969401508, + "coding": 1405.348889326289, + "creative_writing": 1355.546796072321, + "english": 1388.8963382440947, + "exclude_ties": 1364.0861379873427, + "expert": 1396.4346567095095, + "french": 1393.6776419991725, + "german": 1375.1299559013232, + "hard_prompts": 1397.2810194501853, + "hard_prompts_english": 1394.620531270558, + "industry_business_and_management_and_financial_operations": 1379.2774566037917, + "industry_entertainment_and_sports_and_media": 1355.970123065538, + "industry_legal_and_government": 1389.3521056907116, + "industry_life_and_physical_and_social_science": 1381.9573906509481, + "industry_mathematical": 1382.4378368876867, + "industry_medicine_and_healthcare": 1389.8421326439716, + "industry_software_and_it_services": 1406.3641730834995, + "industry_writing_and_literature_and_language": 1375.8569213869234, + "instruction_following": 1375.4054768102055, + "japanese": 1357.8467265151853, + "korean": 1350.7049435959418, + "longer_query": 1393.717668523104, + "math": 1386.7560549239417, + "multi_turn": 1411.230588787225, + "russian": 1400.0691463528256, + "spanish": 1367.216137107554 + }, + "qwen3-30b-a3b-instruct-2507": { + "overall": 1383.8331699150904, + "chinese": 1434.399799595034, + "coding": 1417.7743916851975, + "creative_writing": 1319.3736106518716, + "english": 1391.0944402449438, + "exclude_ties": 1357.9895948790831, + "expert": 1391.3865509809955, + "french": 1421.5351765720504, + "german": 1369.627661041584, + "hard_prompts": 1398.0004453666834, + "hard_prompts_english": 1405.5222830698042, + "industry_business_and_management_and_financial_operations": 1388.878947759055, + "industry_entertainment_and_sports_and_media": 1336.12863067926, + "industry_legal_and_government": 1384.9987240234198, + "industry_life_and_physical_and_social_science": 1391.4784392817764, + "industry_mathematical": 1409.0596352076209, + "industry_medicine_and_healthcare": 1391.4655554766364, + "industry_software_and_it_services": 1417.841792009612, + "industry_writing_and_literature_and_language": 1337.4773081398469, + "instruction_following": 1362.3399194941303, + "japanese": 1337.2198312982655, + "korean": 1328.4122765656525, + "longer_query": 1378.3784895444787, + "math": 1394.4308738088744, + "multi_turn": 1381.4277725683128, + "russian": 1363.6777392978956, + "spanish": 1392.1410794111328 + }, + "glm-4.5-air": { + "overall": 1382.5880602873613, + "chinese": 1431.2877533556093, + "coding": 1396.4987358966694, + "creative_writing": 1342.7023070067755, + "english": 1395.1390357799246, + "exclude_ties": 1356.1283530762155, + "expert": 1369.097148979606, + "french": 1409.349967984753, + "german": 1369.8871170489688, + "hard_prompts": 1378.3612590787156, + "hard_prompts_english": 1386.99001401874, + "industry_business_and_management_and_financial_operations": 1370.9669260109074, + "industry_entertainment_and_sports_and_media": 1338.775576396989, + "industry_legal_and_government": 1385.8139284145727, + "industry_life_and_physical_and_social_science": 1407.760767351817, + "industry_mathematical": 1393.933070354049, + "industry_medicine_and_healthcare": 1403.6075766830036, + "industry_software_and_it_services": 1400.3903786936146, + "industry_writing_and_literature_and_language": 1353.9630273055554, + "instruction_following": 1353.6419683675877, + "japanese": 1343.6861653231397, + "korean": 1301.9779104685676, + "longer_query": 1365.3080752191618, + "math": 1397.7773443770943, + "multi_turn": 1371.934964723917, + "russian": 1369.0393124761706, + "spanish": 1371.829816794659 + }, + "gpt-4.1-2025-04-14": { + "overall": 1381.930745489823, + "chinese": 1373.5996573585644, + "coding": 1390.3613910716747, + "creative_writing": 1363.7831403657087, + "english": 1390.1334914639306, + "exclude_ties": 1354.7161590280061, + "expert": 1363.9991897925595, + "french": 1383.9253360564862, + "german": 1381.1483775967256, + "hard_prompts": 1382.5350333679705, + "hard_prompts_english": 1389.9083868868202, + "industry_business_and_management_and_financial_operations": 1368.2842475244472, + "industry_entertainment_and_sports_and_media": 1363.8672985325172, + "industry_legal_and_government": 1399.6258851178147, + "industry_life_and_physical_and_social_science": 1381.369602434198, + "industry_mathematical": 1358.8384348342722, + "industry_medicine_and_healthcare": 1379.1852472106345, + "industry_software_and_it_services": 1390.6579583023927, + "industry_writing_and_literature_and_language": 1370.1452308570372, + "instruction_following": 1366.2868789917065, + "japanese": 1314.266683398068, + "korean": 1349.9933270751185, + "longer_query": 1382.90668559478, + "math": 1367.5930620080194, + "multi_turn": 1397.1315613367226, + "russian": 1373.173772286955, + "spanish": 1367.9519907179524 + }, + "kimi-k2-0905-preview": { + "overall": 1379.5560541752943, + "chinese": 1412.9385125648585, + "coding": 1399.6363677336851, + "creative_writing": 1349.6953867098794, + "english": 1381.741111854827, + "exclude_ties": 1351.2097169541237, + "expert": 1366.2105041097348, + "french": 1379.7775588594595, + "german": 1384.392313009711, + "hard_prompts": 1382.6111717453587, + "hard_prompts_english": 1382.9469498938251, + "industry_business_and_management_and_financial_operations": 1353.2995827560273, + "industry_entertainment_and_sports_and_media": 1342.1781020742817, + "industry_legal_and_government": 1405.1213259349163, + "industry_life_and_physical_and_social_science": 1386.8686127670433, + "industry_mathematical": 1398.0861864631875, + "industry_medicine_and_healthcare": 1382.8558010590077, + "industry_software_and_it_services": 1393.8822911857103, + "industry_writing_and_literature_and_language": 1345.207235023846, + "instruction_following": 1348.1644974366022, + "japanese": 1344.604993835874, + "korean": 1312.6177714811963, + "longer_query": 1351.216457578577, + "math": 1397.3029520240661, + "multi_turn": 1365.125476236674, + "russian": 1372.9242727673704, + "spanish": 1380.102344402019 + }, + "gemini-2.5-flash-lite-preview-09-2025-no-thinking": { + "overall": 1378.907910368329, + "chinese": 1404.0380697659832, + "coding": 1371.3634126972702, + "creative_writing": 1359.108795882312, + "english": 1381.4533380715238, + "exclude_ties": 1349.0703559122996, + "expert": 1373.1141431664987, + "french": 1388.100108383454, + "german": 1389.0173020122018, + "hard_prompts": 1376.7997186074442, + "hard_prompts_english": 1380.6597341160016, + "industry_business_and_management_and_financial_operations": 1367.4307061114653, + "industry_entertainment_and_sports_and_media": 1343.3223909335215, + "industry_legal_and_government": 1392.6203307561107, + "industry_life_and_physical_and_social_science": 1395.0454753672793, + "industry_mathematical": 1375.297316838559, + "industry_medicine_and_healthcare": 1387.5816277975134, + "industry_software_and_it_services": 1384.38716220317, + "industry_writing_and_literature_and_language": 1367.069417754317, + "instruction_following": 1355.9727754523228, + "japanese": 1363.3048487038834, + "korean": 1346.5051383494217, + "longer_query": 1371.9134586352816, + "math": 1373.3654568291622, + "multi_turn": 1365.856668850782, + "russian": 1371.6762102865112, + "spanish": 1398.028093511605 + }, + "nvidia-nemotron-3-super-120b-a12b": { + "overall": 1378.0801471821649, + "chinese": 1430.7071876803193, + "coding": 1404.3188641336938, + "creative_writing": 1316.801422814665, + "english": 1400.1971880026192, + "exclude_ties": 1346.880206003708, + "expert": 1400.6759639522043, + "french": 1407.3215827156769, + "german": 1341.2901892600212, + "hard_prompts": 1388.304907431998, + "hard_prompts_english": 1399.556170156661, + "industry_business_and_management_and_financial_operations": 1363.7675743216573, + "industry_entertainment_and_sports_and_media": 1337.8281277450883, + "industry_legal_and_government": 1390.0532234145612, + "industry_life_and_physical_and_social_science": 1387.9844524306366, + "industry_mathematical": 1401.3456192311407, + "industry_medicine_and_healthcare": 1385.790363474596, + "industry_software_and_it_services": 1410.2415476359533, + "industry_writing_and_literature_and_language": 1332.2102086102507, + "instruction_following": 1350.278986595788, + "longer_query": 1365.6570618978567, + "math": 1380.790843500599, + "multi_turn": 1367.3025445392414, + "russian": 1338.4024409843596, + "spanish": 1416.908802917588 + }, + "hunyuan-turbos-20250416": { + "overall": 1375.8331118803274, + "chinese": 1419.7924232411665, + "coding": 1361.3738184311715, + "creative_writing": 1358.4743092390186, + "english": 1383.851099947032, + "exclude_ties": 1347.5093206211411, + "expert": 1337.8373540890489, + "french": 1383.7348758886717, + "german": 1352.7622406354856, + "hard_prompts": 1372.1747081564474, + "hard_prompts_english": 1372.6984006592136, + "industry_business_and_management_and_financial_operations": 1382.8167090505594, + "industry_entertainment_and_sports_and_media": 1332.9438854061443, + "industry_legal_and_government": 1371.2311326539739, + "industry_life_and_physical_and_social_science": 1398.9137573348, + "industry_mathematical": 1360.6678557303858, + "industry_medicine_and_healthcare": 1416.626000100621, + "industry_software_and_it_services": 1391.2222185080852, + "industry_writing_and_literature_and_language": 1349.231518383528, + "instruction_following": 1341.8564259198947, + "japanese": 1330.1700248448362, + "korean": 1360.9305674500567, + "longer_query": 1361.301661953239, + "math": 1358.804085095639, + "multi_turn": 1385.933703697627, + "russian": 1370.709158681935 + }, + "claude-opus-4-20250514-thinking-16k": { + "overall": 1375.4874321272423, + "chinese": 1388.0846788473211, + "coding": 1442.280020207358, + "creative_writing": 1384.4096665611671, + "english": 1386.33669611226, + "exclude_ties": 1344.5690482069058, + "expert": 1383.6280374834948, + "french": 1385.7950801473094, + "german": 1384.1308771132594, + "hard_prompts": 1398.7915971799575, + "hard_prompts_english": 1408.0777823482704, + "industry_business_and_management_and_financial_operations": 1344.381259083258, + "industry_entertainment_and_sports_and_media": 1367.5738502566435, + "industry_legal_and_government": 1366.0791044985067, + "industry_life_and_physical_and_social_science": 1366.6264964747704, + "industry_mathematical": 1391.8471848366016, + "industry_medicine_and_healthcare": 1356.0755671978227, + "industry_software_and_it_services": 1406.7185139283142, + "industry_writing_and_literature_and_language": 1390.6783186285284, + "instruction_following": 1405.9915378669557, + "japanese": 1332.8456502782, + "korean": 1314.4157173182687, + "longer_query": 1421.4815430690935, + "math": 1390.2753860431787, + "multi_turn": 1394.0172799090772, + "russian": 1388.9968324102144, + "spanish": 1377.6533271014496 + }, + "deepseek-v3-0324": { + "overall": 1374.8414474136384, + "chinese": 1388.2581636393875, + "coding": 1368.93431577504, + "creative_writing": 1364.9626266367784, + "english": 1386.40518373116, + "exclude_ties": 1345.7814182422785, + "expert": 1351.6153067929777, + "french": 1390.6611534519984, + "german": 1380.2395554152522, + "hard_prompts": 1365.8066461735216, + "hard_prompts_english": 1376.0085485615468, + "industry_business_and_management_and_financial_operations": 1350.1871984252366, + "industry_entertainment_and_sports_and_media": 1356.8168376813867, + "industry_legal_and_government": 1379.9694538418462, + "industry_life_and_physical_and_social_science": 1381.5325588440621, + "industry_mathematical": 1365.4604041702037, + "industry_medicine_and_healthcare": 1382.478490219422, + "industry_software_and_it_services": 1379.7708074976383, + "industry_writing_and_literature_and_language": 1364.7845017168581, + "instruction_following": 1345.4637569477718, + "japanese": 1326.577962134234, + "korean": 1317.4542106213407, + "longer_query": 1352.7668609974412, + "math": 1373.8094142863915, + "multi_turn": 1387.5958206007158, + "russian": 1372.1012113161119, + "spanish": 1344.5887743527255 + }, + "glm-4.6v": { + "overall": 1374.7635300560628, + "coding": 1393.8329074938088, + "creative_writing": 1342.0559780795998, + "english": 1391.660471887265, + "exclude_ties": 1345.297782061201, + "hard_prompts": 1367.4721602667123, + "hard_prompts_english": 1387.8813916678243, + "industry_business_and_management_and_financial_operations": 1357.5281259749704, + "industry_entertainment_and_sports_and_media": 1343.5352503639715, + "industry_legal_and_government": 1397.470326164104, + "industry_life_and_physical_and_social_science": 1392.8464422741579, + "industry_software_and_it_services": 1394.5941029061441, + "industry_writing_and_literature_and_language": 1346.2902525959007, + "instruction_following": 1352.905838801668, + "longer_query": 1357.0066311677053, + "multi_turn": 1357.530328992515, + "russian": 1340.2053926095177 + }, + "gpt-5-mini-high": { + "overall": 1373.7254578956865, + "chinese": 1388.5574566277483, + "coding": 1407.1699262415948, + "creative_writing": 1326.8173805118145, + "english": 1379.831525389419, + "exclude_ties": 1343.063808617822, + "expert": 1380.7371486035124, + "french": 1382.1929270043433, + "german": 1368.396224433841, + "hard_prompts": 1380.558459567497, + "hard_prompts_english": 1386.318875772804, + "industry_business_and_management_and_financial_operations": 1357.9536283938169, + "industry_entertainment_and_sports_and_media": 1341.3514278613482, + "industry_legal_and_government": 1374.9950406002988, + "industry_life_and_physical_and_social_science": 1370.1839266501884, + "industry_mathematical": 1381.105520026867, + "industry_medicine_and_healthcare": 1377.4118671341544, + "industry_software_and_it_services": 1395.0116982209001, + "industry_writing_and_literature_and_language": 1345.6287097814309, + "instruction_following": 1358.5450403248954, + "japanese": 1343.7389446374873, + "korean": 1305.0209169937884, + "longer_query": 1355.891564465307, + "math": 1376.5946191482171, + "multi_turn": 1364.9112148240797, + "russian": 1357.7163873150266, + "spanish": 1359.8332358048724 + }, + "gpt-5.4-nano-high": { + "overall": 1373.5514510582343, + "chinese": 1406.6600380888647, + "coding": 1403.5032156835514, + "creative_writing": 1315.6000220200876, + "english": 1383.3200321956601, + "exclude_ties": 1345.511934611561, + "expert": 1394.7499922227269, + "french": 1390.0246693254676, + "german": 1363.0830001045156, + "hard_prompts": 1381.6969367249076, + "hard_prompts_english": 1390.0151968851944, + "industry_business_and_management_and_financial_operations": 1361.4549409511553, + "industry_entertainment_and_sports_and_media": 1325.6282544376281, + "industry_legal_and_government": 1363.3199257069582, + "industry_life_and_physical_and_social_science": 1376.2850247812387, + "industry_mathematical": 1422.5930797009237, + "industry_medicine_and_healthcare": 1378.1795300129604, + "industry_software_and_it_services": 1401.2546130094079, + "industry_writing_and_literature_and_language": 1337.8247300771618, + "instruction_following": 1360.257026123362, + "japanese": 1360.2796197794323, + "korean": 1294.0531931289572, + "longer_query": 1365.492327403373, + "math": 1418.100074547276, + "multi_turn": 1382.2031282080393, + "russian": 1366.6225748610748, + "spanish": 1361.0643550716977 + }, + "deepseek-r1": { + "overall": 1372.697602781603, + "chinese": 1400.0796259581257, + "coding": 1371.9101020321893, + "creative_writing": 1354.8814440548017, + "english": 1384.7827525963162, + "exclude_ties": 1346.1282473781891, + "expert": 1337.0959236640192, + "french": 1368.1694574203525, + "german": 1384.5673484166205, + "hard_prompts": 1361.4362802668902, + "hard_prompts_english": 1375.5065662613088, + "industry_business_and_management_and_financial_operations": 1347.59825060834, + "industry_entertainment_and_sports_and_media": 1347.1248880838227, + "industry_legal_and_government": 1371.0564464373847, + "industry_life_and_physical_and_social_science": 1375.4179123146293, + "industry_mathematical": 1389.0438972780194, + "industry_medicine_and_healthcare": 1369.702752055246, + "industry_software_and_it_services": 1375.8106121430344, + "industry_writing_and_literature_and_language": 1364.3116364723987, + "instruction_following": 1357.7499328439415, + "japanese": 1325.0224999235515, + "korean": 1329.794578652026, + "longer_query": 1355.028969765825, + "math": 1392.2805667057005, + "multi_turn": 1390.3291719348174, + "russian": 1353.5698857626098, + "spanish": 1374.9776844772803 + }, + "kimi-k2-0711-preview": { + "overall": 1370.774271159434, + "chinese": 1398.3559115328053, + "coding": 1377.7559758721716, + "creative_writing": 1323.84962033027, + "english": 1376.3700660676059, + "exclude_ties": 1338.2043342842867, + "expert": 1345.725757616987, + "french": 1369.583534556968, + "german": 1373.9008189071167, + "hard_prompts": 1365.1815798896885, + "hard_prompts_english": 1368.154067133864, + "industry_business_and_management_and_financial_operations": 1341.2338297261922, + "industry_entertainment_and_sports_and_media": 1332.407456447334, + "industry_legal_and_government": 1355.5342090446916, + "industry_life_and_physical_and_social_science": 1363.7596292948992, + "industry_mathematical": 1356.978125566153, + "industry_medicine_and_healthcare": 1367.6676092595176, + "industry_software_and_it_services": 1384.302171663605, + "industry_writing_and_literature_and_language": 1326.3990687319904, + "instruction_following": 1323.2109996524732, + "japanese": 1327.5900790682647, + "korean": 1286.239185634206, + "longer_query": 1325.7419634874198, + "math": 1367.1388775868245, + "multi_turn": 1372.2095279704586, + "russian": 1352.0838769298953, + "spanish": 1325.086240508078 + }, + "mistral-medium-2505": { + "overall": 1369.2996054829764, + "chinese": 1373.742871160749, + "coding": 1386.2612205821977, + "creative_writing": 1343.9998975822584, + "english": 1384.228227331833, + "exclude_ties": 1339.357651212583, + "expert": 1343.6867092333496, + "french": 1372.5572995418586, + "german": 1377.9369231885087, + "hard_prompts": 1364.8066854316062, + "hard_prompts_english": 1375.976357082607, + "industry_business_and_management_and_financial_operations": 1351.2507911586647, + "industry_entertainment_and_sports_and_media": 1334.9702461094726, + "industry_legal_and_government": 1383.4874273922533, + "industry_life_and_physical_and_social_science": 1379.7570317525553, + "industry_mathematical": 1354.0874164667825, + "industry_medicine_and_healthcare": 1379.9109221150004, + "industry_software_and_it_services": 1385.4878089178028, + "industry_writing_and_literature_and_language": 1351.5427178114687, + "instruction_following": 1339.0317256360236, + "japanese": 1313.2203607987485, + "korean": 1301.9983732481774, + "longer_query": 1358.5692821712184, + "math": 1351.801039399284, + "multi_turn": 1383.0153485258218, + "russian": 1357.2380283328148, + "spanish": 1364.6797409521953 + }, + "gemini-2.5-flash-lite-preview-06-17-thinking": { + "overall": 1368.4514580751118, + "chinese": 1401.6112050028528, + "coding": 1373.8055384159172, + "creative_writing": 1365.7813502646911, + "english": 1372.538219063858, + "exclude_ties": 1335.4330471629983, + "expert": 1365.127266519894, + "french": 1388.2792068906138, + "german": 1371.7172128468007, + "hard_prompts": 1372.6894658206381, + "hard_prompts_english": 1374.0437905024555, + "industry_business_and_management_and_financial_operations": 1354.352123064996, + "industry_entertainment_and_sports_and_media": 1354.0909916445141, + "industry_legal_and_government": 1372.1868543393016, + "industry_life_and_physical_and_social_science": 1372.3605231508268, + "industry_mathematical": 1366.2287519243732, + "industry_medicine_and_healthcare": 1386.8440988063726, + "industry_software_and_it_services": 1376.301849873712, + "industry_writing_and_literature_and_language": 1373.8319390195577, + "instruction_following": 1366.450118199757, + "japanese": 1349.6131326112493, + "korean": 1359.338451359557, + "longer_query": 1373.3224240951101, + "math": 1362.8203875998784, + "multi_turn": 1360.033873488268, + "russian": 1367.4864105404022, + "spanish": 1361.3058303111766 + }, + "qwen3-next-80b-a3b-thinking": { + "overall": 1367.7476631774657, + "chinese": 1416.5164079626559, + "coding": 1391.2292219277263, + "creative_writing": 1311.800360544552, + "english": 1392.8380295027646, + "exclude_ties": 1333.4386721913675, + "expert": 1368.0114468022132, + "french": 1358.409678714287, + "german": 1360.4403591767375, + "hard_prompts": 1370.5264610732333, + "hard_prompts_english": 1386.34642457053, + "industry_business_and_management_and_financial_operations": 1361.404610410873, + "industry_entertainment_and_sports_and_media": 1313.2105200113647, + "industry_legal_and_government": 1363.4998100530963, + "industry_life_and_physical_and_social_science": 1382.2352393627461, + "industry_mathematical": 1387.86524594386, + "industry_medicine_and_healthcare": 1389.8959005147317, + "industry_software_and_it_services": 1393.1157295771982, + "industry_writing_and_literature_and_language": 1327.6601232563949, + "instruction_following": 1342.8766976233853, + "japanese": 1281.6714919171327, + "korean": 1309.4613368449602, + "longer_query": 1352.7024886457298, + "math": 1394.4817887078343, + "multi_turn": 1345.3658797725097, + "russian": 1338.0706501146815, + "spanish": 1348.1560110608043 + }, + "grok-3-mini-high": { + "overall": 1366.675855330357, + "chinese": 1378.6557274329134, + "coding": 1375.8605410896698, + "creative_writing": 1330.283114085792, + "english": 1376.5507913700746, + "exclude_ties": 1331.713948369471, + "expert": 1389.8401143599592, + "french": 1352.4158946303264, + "german": 1354.2272965026846, + "hard_prompts": 1373.156259449323, + "hard_prompts_english": 1377.476103097485, + "industry_business_and_management_and_financial_operations": 1341.467876080604, + "industry_entertainment_and_sports_and_media": 1332.6991822396988, + "industry_legal_and_government": 1386.2541679710623, + "industry_life_and_physical_and_social_science": 1379.2556788267605, + "industry_mathematical": 1365.4761616044073, + "industry_medicine_and_healthcare": 1390.4175617120045, + "industry_software_and_it_services": 1375.1056862353098, + "industry_writing_and_literature_and_language": 1347.12695573296, + "instruction_following": 1356.9923986065498, + "japanese": 1314.7262151800978, + "korean": 1341.409252938796, + "longer_query": 1372.1122713303976, + "math": 1383.653714067016, + "multi_turn": 1352.6176222612175, + "russian": 1354.2324302710076, + "spanish": 1355.0834623720007 + }, + "qwen2.5-max": { + "overall": 1366.5473310294171, + "chinese": 1380.0924093115295, + "coding": 1359.824877843147, + "creative_writing": 1339.4870487302628, + "english": 1374.893123937637, + "exclude_ties": 1337.012885470466, + "expert": 1335.2049333621744, + "french": 1395.4006441748209, + "german": 1349.7535461757975, + "hard_prompts": 1359.2769501389807, + "hard_prompts_english": 1362.3360651753264, + "industry_business_and_management_and_financial_operations": 1349.9337640671843, + "industry_entertainment_and_sports_and_media": 1326.872171911044, + "industry_legal_and_government": 1374.2565430072045, + "industry_life_and_physical_and_social_science": 1381.176328864617, + "industry_mathematical": 1366.7551155987953, + "industry_medicine_and_healthcare": 1383.0272456702735, + "industry_software_and_it_services": 1372.7428696514564, + "industry_writing_and_literature_and_language": 1346.081604543421, + "instruction_following": 1334.5080236957856, + "japanese": 1300.0353648377195, + "korean": 1305.308846383163, + "longer_query": 1356.963275553537, + "math": 1368.840341540682, + "multi_turn": 1363.964141056429, + "russian": 1349.6600654293256, + "spanish": 1377.5744242356784 + }, + "o1-2024-12-17": { + "overall": 1365.8721638791403, + "chinese": 1393.6447771988967, + "coding": 1367.4241083149682, + "creative_writing": 1348.0516434863878, + "english": 1372.0673286259275, + "exclude_ties": 1333.9441516093625, + "expert": 1360.967972506838, + "french": 1345.3709870289156, + "german": 1337.1915161510133, + "hard_prompts": 1371.7449099548694, + "hard_prompts_english": 1373.3239310181573, + "industry_business_and_management_and_financial_operations": 1327.5404788771505, + "industry_entertainment_and_sports_and_media": 1358.4491026161409, + "industry_legal_and_government": 1368.253167777445, + "industry_life_and_physical_and_social_science": 1369.0868123138034, + "industry_mathematical": 1389.3730623499305, + "industry_medicine_and_healthcare": 1353.900122773158, + "industry_software_and_it_services": 1367.3079802874502, + "industry_writing_and_literature_and_language": 1359.924813740378, + "instruction_following": 1367.6487789921505, + "japanese": 1346.811201169663, + "korean": 1396.2301071660322, + "longer_query": 1377.8728570036901, + "math": 1388.1933432752587, + "multi_turn": 1354.6039938111144, + "russian": 1354.7319779563877, + "spanish": 1341.2449309523413 + }, + "qwen3-235b-a22b": { + "overall": 1365.7566571708007, + "chinese": 1379.9110855711613, + "coding": 1385.7802886034974, + "creative_writing": 1316.5635436264424, + "english": 1376.5363332960428, + "exclude_ties": 1331.8443923776995, + "expert": 1348.3373971490876, + "french": 1368.4673602672542, + "german": 1386.1758151063345, + "hard_prompts": 1362.295041798875, + "hard_prompts_english": 1372.055896054326, + "industry_business_and_management_and_financial_operations": 1348.7939501943931, + "industry_entertainment_and_sports_and_media": 1316.1752694480779, + "industry_legal_and_government": 1354.9686496087265, + "industry_life_and_physical_and_social_science": 1371.721174228821, + "industry_mathematical": 1397.0806406492297, + "industry_medicine_and_healthcare": 1362.86990354124, + "industry_software_and_it_services": 1386.861151586945, + "industry_writing_and_literature_and_language": 1336.503231564315, + "instruction_following": 1333.1607800695774, + "japanese": 1307.5956850009072, + "korean": 1314.6213239144306, + "longer_query": 1352.1110579522754, + "math": 1395.3742421898237, + "multi_turn": 1362.1276501406044, + "russian": 1338.3677670349653, + "spanish": 1362.4251037598024 + }, + "gpt-oss-120b": { + "overall": 1365.527982625054, + "chinese": 1377.715543174236, + "coding": 1380.4243394384548, + "creative_writing": 1273.9373696013547, + "english": 1373.8458979749307, + "exclude_ties": 1330.7650133033976, + "expert": 1357.8200372643191, + "french": 1372.6279528395712, + "german": 1351.980139341184, + "hard_prompts": 1363.7462881652934, + "hard_prompts_english": 1367.9678983013089, + "industry_business_and_management_and_financial_operations": 1357.6828726054166, + "industry_entertainment_and_sports_and_media": 1292.6497065424292, + "industry_legal_and_government": 1357.2266752457735, + "industry_life_and_physical_and_social_science": 1381.502605856943, + "industry_mathematical": 1384.102946000642, + "industry_medicine_and_healthcare": 1389.8184132765273, + "industry_software_and_it_services": 1387.791033855696, + "industry_writing_and_literature_and_language": 1305.4473371589008, + "instruction_following": 1319.1751633711988, + "japanese": 1323.532215469373, + "korean": 1278.532416241701, + "longer_query": 1319.097608461962, + "math": 1388.043867898504, + "multi_turn": 1339.8287538189322, + "russian": 1338.2137165399877, + "spanish": 1383.782822624211 + }, + "claude-opus-4-20250514": { + "overall": 1364.2857615943894, + "chinese": 1387.4026735172808, + "coding": 1401.775822692937, + "creative_writing": 1375.1338353877495, + "english": 1376.294971165187, + "exclude_ties": 1329.791529730854, + "expert": 1371.1253060868873, + "french": 1365.4101990293907, + "german": 1372.4470940796505, + "hard_prompts": 1375.9110792649124, + "hard_prompts_english": 1384.2718964778371, + "industry_business_and_management_and_financial_operations": 1335.8640873382876, + "industry_entertainment_and_sports_and_media": 1348.9138938694168, + "industry_legal_and_government": 1372.6493080148257, + "industry_life_and_physical_and_social_science": 1362.6116018993318, + "industry_mathematical": 1376.339395045655, + "industry_medicine_and_healthcare": 1364.4179147774773, + "industry_software_and_it_services": 1378.4419670958937, + "industry_writing_and_literature_and_language": 1380.9766042221904, + "instruction_following": 1372.8092401393533, + "japanese": 1329.8508271159162, + "korean": 1303.4816141500482, + "longer_query": 1401.9862425225854, + "math": 1373.0621630665742, + "multi_turn": 1385.9946217895156, + "russian": 1381.26303710258, + "spanish": 1350.4355251903785 + }, + "amazon-nova-experimental-chat-10-09": { + "overall": 1364.1591008388023, + "coding": 1366.161710397271, + "creative_writing": 1306.4291479047638, + "english": 1382.516837039848, + "exclude_ties": 1328.5488634766682, + "hard_prompts": 1353.6898896538255, + "hard_prompts_english": 1361.341609593141, + "industry_business_and_management_and_financial_operations": 1376.9356514151793, + "industry_entertainment_and_sports_and_media": 1308.347572223939, + "industry_legal_and_government": 1367.1067841671868, + "industry_life_and_physical_and_social_science": 1407.1396092892735, + "industry_software_and_it_services": 1377.1843817911565, + "industry_writing_and_literature_and_language": 1321.4952606840102, + "instruction_following": 1315.8279842651573, + "longer_query": 1332.256459377016, + "multi_turn": 1353.9363699392297 + }, + "nova-2-lite": { + "overall": 1362.9348351352505, + "chinese": 1367.0807709490055, + "coding": 1388.6899439247459, + "creative_writing": 1294.277476701012, + "english": 1383.6093515102013, + "exclude_ties": 1327.3053111759455, + "expert": 1366.866027529423, + "french": 1377.4546774766711, + "german": 1353.7355147361961, + "hard_prompts": 1365.2058033131163, + "hard_prompts_english": 1381.667888661171, + "industry_business_and_management_and_financial_operations": 1368.0232175413732, + "industry_entertainment_and_sports_and_media": 1307.068555006551, + "industry_legal_and_government": 1354.7881589542972, + "industry_life_and_physical_and_social_science": 1381.1222403263314, + "industry_mathematical": 1370.0586477583859, + "industry_medicine_and_healthcare": 1385.5973751254219, + "industry_software_and_it_services": 1394.5264215537559, + "industry_writing_and_literature_and_language": 1318.6104232299076, + "instruction_following": 1335.2841644689156, + "korean": 1271.885894856906, + "longer_query": 1336.1996688110453, + "math": 1360.440584180144, + "multi_turn": 1337.3137976353134, + "russian": 1338.5424657210392, + "spanish": 1360.4461340890493 + }, + "ling-flash-2.0": { + "overall": 1362.7393379843973, + "chinese": 1410.4859441224708, + "coding": 1390.9860880467309, + "creative_writing": 1288.257300294805, + "english": 1382.8582898087436, + "exclude_ties": 1325.5942358077632, + "expert": 1355.6220527579546, + "hard_prompts": 1363.5066541708693, + "hard_prompts_english": 1382.8382272396757, + "industry_business_and_management_and_financial_operations": 1358.196212934592, + "industry_entertainment_and_sports_and_media": 1294.1814771304307, + "industry_legal_and_government": 1369.5444998769249, + "industry_life_and_physical_and_social_science": 1383.6994274342326, + "industry_mathematical": 1370.1125452056465, + "industry_medicine_and_healthcare": 1360.9597473552053, + "industry_software_and_it_services": 1403.0357266931471, + "industry_writing_and_literature_and_language": 1293.6514385249934, + "instruction_following": 1318.6048745570079, + "korean": 1332.6270588319467, + "longer_query": 1323.5184338638253, + "math": 1371.5139611962034, + "multi_turn": 1327.0617377826886, + "russian": 1304.6636053907853, + "spanish": 1401.5186266343717 + }, + "grok-3-mini-beta": { + "overall": 1362.6189905861668, + "chinese": 1374.7668024685458, + "coding": 1366.118650609882, + "creative_writing": 1340.0138082028932, + "english": 1372.9491884260751, + "exclude_ties": 1327.6216669946493, + "expert": 1360.1085423231511, + "french": 1358.719507871311, + "german": 1344.3209039404276, + "hard_prompts": 1363.2963864928893, + "hard_prompts_english": 1373.538021948687, + "industry_business_and_management_and_financial_operations": 1344.4789084857534, + "industry_entertainment_and_sports_and_media": 1328.1242479996931, + "industry_legal_and_government": 1367.5531014973494, + "industry_life_and_physical_and_social_science": 1373.4844494517135, + "industry_mathematical": 1358.4089122748805, + "industry_medicine_and_healthcare": 1364.4160118921054, + "industry_software_and_it_services": 1371.7910490540132, + "industry_writing_and_literature_and_language": 1344.8760514959872, + "instruction_following": 1345.4047058267117, + "japanese": 1337.403668276491, + "korean": 1305.9284891992772, + "longer_query": 1356.5991274632322, + "math": 1370.2380108355999, + "multi_turn": 1348.8668541330599, + "russian": 1350.49764228866, + "spanish": 1383.7073682084888 + }, + "minimax-m2.5": { + "overall": 1359.8946025342593, + "chinese": 1396.3612799998934, + "coding": 1379.8256157069065, + "creative_writing": 1332.9191322857118, + "english": 1383.9121375692102, + "exclude_ties": 1325.3585986650298, + "expert": 1376.7221877472548, + "french": 1366.8755595210637, + "german": 1355.2590131113748, + "hard_prompts": 1371.170823211611, + "hard_prompts_english": 1385.7956810178482, + "industry_business_and_management_and_financial_operations": 1356.4203444999705, + "industry_entertainment_and_sports_and_media": 1328.4475573618465, + "industry_legal_and_government": 1366.082082365347, + "industry_life_and_physical_and_social_science": 1366.2512677900959, + "industry_mathematical": 1380.1009187759817, + "industry_medicine_and_healthcare": 1359.950822013282, + "industry_software_and_it_services": 1382.5366355698088, + "industry_writing_and_literature_and_language": 1348.2435146037485, + "instruction_following": 1353.9297700769614, + "japanese": 1169.8747283472105, + "korean": 1234.9877922946994, + "longer_query": 1364.6244683415298, + "math": 1383.2136206217774, + "multi_turn": 1361.983143300423, + "russian": 1353.6417114953056, + "spanish": 1351.079414852887 + }, + "gemma-3-27b-it": { + "overall": 1358.1735435502378, + "chinese": 1346.5856057310857, + "coding": 1322.4896700142954, + "creative_writing": 1344.9244662478186, + "english": 1374.2188350477318, + "exclude_ties": 1322.181397847335, + "expert": 1302.8743233704704, + "french": 1371.4148378280163, + "german": 1362.466037212583, + "hard_prompts": 1339.7962073474575, + "hard_prompts_english": 1349.4319779227617, + "industry_business_and_management_and_financial_operations": 1359.6019275962135, + "industry_entertainment_and_sports_and_media": 1320.6014371772771, + "industry_legal_and_government": 1374.9089657586417, + "industry_life_and_physical_and_social_science": 1383.1706715122666, + "industry_mathematical": 1326.8012847795035, + "industry_medicine_and_healthcare": 1389.3388431416695, + "industry_software_and_it_services": 1355.8257667334874, + "industry_writing_and_literature_and_language": 1347.1290242280459, + "instruction_following": 1321.5135400069219, + "japanese": 1286.6186582853325, + "korean": 1309.2799343775307, + "longer_query": 1333.4155982876046, + "math": 1311.2858229417764, + "multi_turn": 1345.2087338216172, + "russian": 1348.6501372866671, + "spanish": 1341.5129345089074 + }, + "mercury-2": { + "overall": 1356.4462535030877, + "coding": 1391.9294365076853, + "creative_writing": 1294.9924760762883, + "english": 1377.6125859893464, + "exclude_ties": 1313.7689355363414, + "expert": 1355.8134536823784, + "hard_prompts": 1360.7305598354055, + "hard_prompts_english": 1375.8729714012331, + "industry_business_and_management_and_financial_operations": 1348.8222592312256, + "industry_entertainment_and_sports_and_media": 1316.2012356911316, + "industry_life_and_physical_and_social_science": 1344.2282869105347, + "industry_medicine_and_healthcare": 1370.5072479192206, + "industry_software_and_it_services": 1384.7213637483617, + "industry_writing_and_literature_and_language": 1302.2869398633306, + "instruction_following": 1321.2824585747394, + "longer_query": 1327.2260514703325, + "multi_turn": 1357.084320843991, + "russian": 1302.7591884766239 + }, + "qwen3-coder-480b-a35b-instruct": { + "overall": 1356.3135313933833, + "chinese": 1356.6674820601584, + "coding": 1412.5265706880111, + "creative_writing": 1335.6537423281725, + "english": 1363.6067990566353, + "exclude_ties": 1318.330473310536, + "expert": 1336.9203166330658, + "french": 1395.4061117051804, + "german": 1322.0248204408767, + "hard_prompts": 1371.9767970300165, + "hard_prompts_english": 1384.1813533670465, + "industry_business_and_management_and_financial_operations": 1343.6361392711628, + "industry_entertainment_and_sports_and_media": 1318.6813523195942, + "industry_legal_and_government": 1357.650337645412, + "industry_life_and_physical_and_social_science": 1348.7968359262054, + "industry_mathematical": 1365.9890960432756, + "industry_medicine_and_healthcare": 1346.4294612616172, + "industry_software_and_it_services": 1392.2744105056754, + "industry_writing_and_literature_and_language": 1340.4898948401797, + "instruction_following": 1356.8934808761778, + "japanese": 1303.6764811629514, + "korean": 1305.4888068875275, + "longer_query": 1379.0548996955206, + "math": 1365.6405319298353, + "multi_turn": 1366.9214924647172, + "russian": 1361.0351439359372, + "spanish": 1350.8728359108786 + }, + "intellect-3": { + "overall": 1356.1337979667167, + "chinese": 1364.3119787098913, + "coding": 1367.963692523403, + "creative_writing": 1318.7994616486599, + "english": 1380.3859940356324, + "exclude_ties": 1316.8660015718808, + "expert": 1319.639425777837, + "hard_prompts": 1349.5575463238083, + "hard_prompts_english": 1372.9968358716596, + "industry_business_and_management_and_financial_operations": 1348.7008452729797, + "industry_entertainment_and_sports_and_media": 1307.0599231016422, + "industry_legal_and_government": 1324.7332717242812, + "industry_life_and_physical_and_social_science": 1362.4310715074396, + "industry_mathematical": 1377.2220641870551, + "industry_medicine_and_healthcare": 1325.8000916216206, + "industry_software_and_it_services": 1373.509760187585, + "industry_writing_and_literature_and_language": 1338.4134279770542, + "instruction_following": 1312.4602516158448, + "longer_query": 1327.8729588677588, + "math": 1384.3457826080933, + "multi_turn": 1352.6405765977265, + "russian": 1336.340091460816 + }, + "gemini-2.0-flash-001": { + "overall": 1354.0260234783718, + "chinese": 1378.5255944580886, + "coding": 1351.6586278282998, + "creative_writing": 1340.578438683938, + "english": 1365.20500248303, + "exclude_ties": 1317.649210973117, + "expert": 1339.4659180617552, + "french": 1392.5564375876877, + "german": 1353.2509067201438, + "hard_prompts": 1346.5315577956928, + "hard_prompts_english": 1356.5381348804615, + "industry_business_and_management_and_financial_operations": 1336.6784370172659, + "industry_entertainment_and_sports_and_media": 1333.2801706268735, + "industry_legal_and_government": 1372.1247246631435, + "industry_life_and_physical_and_social_science": 1361.047114990096, + "industry_mathematical": 1356.8542627820536, + "industry_medicine_and_healthcare": 1367.6877294605704, + "industry_software_and_it_services": 1362.2064530032462, + "industry_writing_and_literature_and_language": 1347.314972754642, + "instruction_following": 1335.8767260449508, + "japanese": 1293.5089949763471, + "korean": 1312.3876744076363, + "longer_query": 1344.8648469441441, + "math": 1351.5520130522887, + "multi_turn": 1349.669365759414, + "russian": 1352.692219935991, + "spanish": 1360.8842632847263 + }, + "glm-4.7-flash": { + "overall": 1353.3877317042368, + "chinese": 1425.553411240891, + "coding": 1383.0185812541583, + "creative_writing": 1302.75857973447, + "english": 1375.8436891503088, + "exclude_ties": 1310.884437762606, + "expert": 1361.1952824033046, + "french": 1347.0209088323713, + "german": 1335.2415479786562, + "hard_prompts": 1358.703136474828, + "hard_prompts_english": 1372.283536945439, + "industry_business_and_management_and_financial_operations": 1344.8120480013667, + "industry_entertainment_and_sports_and_media": 1307.3422860836026, + "industry_legal_and_government": 1360.3837661377484, + "industry_life_and_physical_and_social_science": 1362.547237179192, + "industry_mathematical": 1375.6243217201225, + "industry_medicine_and_healthcare": 1346.5066586498374, + "industry_software_and_it_services": 1392.128157856101, + "industry_writing_and_literature_and_language": 1321.7380737750505, + "instruction_following": 1329.388644050346, + "korean": 1286.0994144018873, + "longer_query": 1348.1654692633022, + "math": 1358.1887604763215, + "multi_turn": 1344.8379860591322, + "russian": 1337.516091270282, + "spanish": 1357.642248510558 + }, + "o1-preview": { + "overall": 1352.8573218239003, + "chinese": 1324.5700308853716, + "coding": 1367.1451249661313, + "creative_writing": 1318.695465887855, + "english": 1382.7150064586244, + "exclude_ties": 1314.3955372398882, + "expert": 1337.1204143477435, + "french": 1344.9454576187104, + "german": 1309.223999302335, + "hard_prompts": 1354.5270322814295, + "hard_prompts_english": 1377.3895701320669, + "industry_business_and_management_and_financial_operations": 1304.1618603061613, + "industry_entertainment_and_sports_and_media": 1331.3078808976998, + "industry_legal_and_government": 1346.4507191908626, + "industry_life_and_physical_and_social_science": 1348.7401663825935, + "industry_mathematical": 1380.0031754946242, + "industry_medicine_and_healthcare": 1305.9467041786975, + "industry_software_and_it_services": 1362.6779226848464, + "industry_writing_and_literature_and_language": 1339.3927517844818, + "instruction_following": 1341.9611544414581, + "japanese": 1294.7924837640412, + "korean": 1290.2389017209275, + "longer_query": 1344.076629044811, + "math": 1372.8657694340761, + "multi_turn": 1368.4391547411353, + "russian": 1314.993363036803, + "spanish": 1300.4077978830524 + }, + "o4-mini-2025-04-16": { + "overall": 1352.7034972912745, + "chinese": 1350.4763965441807, + "coding": 1368.8243375379493, + "creative_writing": 1296.0042415113599, + "english": 1366.1466956701604, + "exclude_ties": 1314.1946684448435, + "expert": 1344.024401851815, + "french": 1360.7001029496457, + "german": 1336.4908640053236, + "hard_prompts": 1351.189907795835, + "hard_prompts_english": 1364.412566426309, + "industry_business_and_management_and_financial_operations": 1324.6084389454686, + "industry_entertainment_and_sports_and_media": 1309.604717089705, + "industry_legal_and_government": 1350.3680042091446, + "industry_life_and_physical_and_social_science": 1352.97546818815, + "industry_mathematical": 1375.7727692925814, + "industry_medicine_and_healthcare": 1361.2342723103845, + "industry_software_and_it_services": 1367.5073912550702, + "industry_writing_and_literature_and_language": 1315.7901367507889, + "instruction_following": 1321.1686044084208, + "japanese": 1301.9672310288938, + "korean": 1317.4849673320703, + "longer_query": 1315.016923205102, + "math": 1386.811656337085, + "multi_turn": 1349.310856971521, + "russian": 1335.407972721859, + "spanish": 1344.5052002702687 + }, + "step-3": { + "overall": 1349.417931240205, + "chinese": 1398.4586213286616, + "coding": 1365.4063169035765, + "creative_writing": 1318.232052309501, + "english": 1370.19068469687, + "exclude_ties": 1310.1062815737148, + "expert": 1330.5151809058734, + "german": 1379.6281234413466, + "hard_prompts": 1354.3007230837966, + "hard_prompts_english": 1374.822378048069, + "industry_business_and_management_and_financial_operations": 1343.5296130166798, + "industry_entertainment_and_sports_and_media": 1328.467271650586, + "industry_legal_and_government": 1338.616256323191, + "industry_life_and_physical_and_social_science": 1354.9005785844329, + "industry_mathematical": 1375.4167022384217, + "industry_medicine_and_healthcare": 1357.647631288801, + "industry_software_and_it_services": 1365.5971153558105, + "industry_writing_and_literature_and_language": 1324.371350610213, + "instruction_following": 1331.4310246876337, + "korean": 1263.5199067012582, + "longer_query": 1326.6588327043544, + "math": 1366.9197356262925, + "multi_turn": 1337.501267928867, + "russian": 1325.3063065861365, + "spanish": 1379.9024284618872 + }, + "nvidia-nemotron-3-nano-30b-a3b-bf16": { + "overall": 1349.2985952263987, + "chinese": 1393.5644120940665, + "coding": 1379.2111248642007, + "creative_writing": 1275.7445045776185, + "english": 1378.3556832861436, + "exclude_ties": 1305.6118290980917, + "expert": 1367.3483652593945, + "french": 1397.9496007880202, + "german": 1349.9627642706805, + "hard_prompts": 1350.0725230976905, + "hard_prompts_english": 1373.2954620449716, + "industry_business_and_management_and_financial_operations": 1346.8999696674841, + "industry_entertainment_and_sports_and_media": 1298.8211992310507, + "industry_legal_and_government": 1347.689560774385, + "industry_life_and_physical_and_social_science": 1357.6986632256549, + "industry_mathematical": 1376.9314251986675, + "industry_medicine_and_healthcare": 1350.188904747926, + "industry_software_and_it_services": 1383.349590592527, + "industry_writing_and_literature_and_language": 1287.9340669809608, + "instruction_following": 1303.6451545390023, + "japanese": 1257.2279757521048, + "korean": 1236.1112682384764, + "longer_query": 1305.0197406040134, + "math": 1373.672333038005, + "multi_turn": 1327.6342545391105, + "russian": 1296.8422126873581, + "spanish": 1369.0366772707027 + }, + "claude-sonnet-4-20250514-thinking-32k": { + "overall": 1348.441408687781, + "chinese": 1352.9394153604237, + "coding": 1413.198718576513, + "creative_writing": 1345.8750491756932, + "english": 1366.0169695277184, + "exclude_ties": 1308.1277822780316, + "expert": 1372.8163521187255, + "french": 1360.38731753097, + "german": 1314.5633262643848, + "hard_prompts": 1371.028096315131, + "hard_prompts_english": 1390.4680416268677, + "industry_business_and_management_and_financial_operations": 1314.0830121405104, + "industry_entertainment_and_sports_and_media": 1332.8516001630683, + "industry_legal_and_government": 1336.36214082152, + "industry_life_and_physical_and_social_science": 1344.7011782609388, + "industry_mathematical": 1379.043625497152, + "industry_medicine_and_healthcare": 1324.943582340347, + "industry_software_and_it_services": 1380.5317815824244, + "industry_writing_and_literature_and_language": 1356.2866801175437, + "instruction_following": 1375.4880884545978, + "japanese": 1289.748117363043, + "korean": 1269.0985755247332, + "longer_query": 1396.775505390101, + "math": 1374.4507520914506, + "multi_turn": 1375.1581177543549, + "russian": 1356.2937733634894, + "spanish": 1350.8569414796807 + }, + "trinity-large-thinking": { + "overall": 1345.4194201743612, + "chinese": 1384.5746702052254, + "coding": 1362.6642962149044, + "creative_writing": 1313.3370778496146, + "english": 1361.1145324539934, + "exclude_ties": 1307.9039366861475, + "expert": 1360.4327125546647, + "french": 1382.0992735563025, + "german": 1349.4932284140905, + "hard_prompts": 1346.0339197282383, + "hard_prompts_english": 1359.2820755227121, + "industry_business_and_management_and_financial_operations": 1334.6120686883348, + "industry_entertainment_and_sports_and_media": 1303.5045360476333, + "industry_legal_and_government": 1344.3380784572742, + "industry_life_and_physical_and_social_science": 1362.6364208725336, + "industry_mathematical": 1363.8929021074412, + "industry_medicine_and_healthcare": 1358.0298083195923, + "industry_software_and_it_services": 1368.0229183942838, + "industry_writing_and_literature_and_language": 1323.5323519159313, + "instruction_following": 1329.1307771805937, + "japanese": 1343.113170721163, + "korean": 1301.0674241647675, + "longer_query": 1320.7629846177795, + "math": 1372.0132366018772, + "multi_turn": 1324.816679163368, + "russian": 1338.6513732439516, + "spanish": 1339.4892743875898 + }, + "minimax-m1": { + "overall": 1342.3191689062471, + "chinese": 1359.9692493232592, + "coding": 1358.9980540158472, + "creative_writing": 1296.5763989502175, + "english": 1361.8457488021586, + "exclude_ties": 1298.361168174024, + "expert": 1315.5293617061027, + "french": 1387.3201668337874, + "german": 1347.6198625813317, + "hard_prompts": 1339.8018406082665, + "hard_prompts_english": 1351.9940214542985, + "industry_business_and_management_and_financial_operations": 1319.0369114819161, + "industry_entertainment_and_sports_and_media": 1302.907490380113, + "industry_legal_and_government": 1340.114447804926, + "industry_life_and_physical_and_social_science": 1355.954502405582, + "industry_mathematical": 1364.5486215971564, + "industry_medicine_and_healthcare": 1356.7451947062666, + "industry_software_and_it_services": 1359.1671540203306, + "industry_writing_and_literature_and_language": 1310.4561516881126, + "instruction_following": 1312.1053292253764, + "japanese": 1212.7265702400641, + "korean": 1253.30170586744, + "longer_query": 1328.213338303472, + "math": 1361.3726860279849, + "multi_turn": 1335.0022219221976, + "russian": 1325.6908674836036, + "spanish": 1340.3196090026304 + }, + "minimax-m2": { + "overall": 1342.228059592529, + "chinese": 1384.922992018155, + "coding": 1373.8653906506918, + "creative_writing": 1289.6360683203322, + "english": 1363.820320717097, + "exclude_ties": 1299.0479792222552, + "expert": 1321.8488969349214, + "hard_prompts": 1359.034072215493, + "hard_prompts_english": 1374.9112842739787, + "industry_business_and_management_and_financial_operations": 1338.182989068146, + "industry_entertainment_and_sports_and_media": 1301.8264668696233, + "industry_legal_and_government": 1345.8178095563467, + "industry_life_and_physical_and_social_science": 1349.4372144927956, + "industry_mathematical": 1368.4093195829894, + "industry_medicine_and_healthcare": 1365.9529532791407, + "industry_software_and_it_services": 1376.400532060201, + "industry_writing_and_literature_and_language": 1298.9385428078767, + "instruction_following": 1333.539174861724, + "longer_query": 1335.3813004799717, + "math": 1352.8408407467034, + "multi_turn": 1367.6416916520845, + "russian": 1328.8514228067297, + "spanish": 1318.429693258257 + }, + "gpt-4.1-mini-2025-04-14": { + "overall": 1340.457375274113, + "chinese": 1329.0571090706003, + "coding": 1367.5270125384454, + "creative_writing": 1300.857619875678, + "english": 1356.3251835836481, + "exclude_ties": 1296.56980370676, + "expert": 1337.3130651540428, + "french": 1357.6323347322584, + "german": 1347.8692162863736, + "hard_prompts": 1348.94318986075, + "hard_prompts_english": 1361.4564968987834, + "industry_business_and_management_and_financial_operations": 1321.5124039231694, + "industry_entertainment_and_sports_and_media": 1300.8744495132378, + "industry_legal_and_government": 1344.8068117879561, + "industry_life_and_physical_and_social_science": 1327.8454036010598, + "industry_mathematical": 1346.2676925751273, + "industry_medicine_and_healthcare": 1325.702912784916, + "industry_software_and_it_services": 1360.6897326890448, + "industry_writing_and_literature_and_language": 1318.6499827787577, + "instruction_following": 1332.1334487623812, + "japanese": 1285.7391973470926, + "korean": 1297.4425724765533, + "longer_query": 1343.4893432345793, + "math": 1343.317317375524, + "multi_turn": 1352.8727181068894, + "russian": 1317.32333822285, + "spanish": 1307.5747477302116 + }, + "qwen3-32b": { + "overall": 1340.0551171644636, + "chinese": 1357.6648364717798, + "coding": 1358.1636542854812, + "creative_writing": 1298.1103196116555, + "english": 1358.519009221768, + "exclude_ties": 1296.5237667979495, + "expert": 1362.021266971823, + "german": 1341.5329623121827, + "hard_prompts": 1334.6538963250018, + "hard_prompts_english": 1349.3398040338445, + "industry_business_and_management_and_financial_operations": 1318.1635840840095, + "industry_entertainment_and_sports_and_media": 1279.477097934524, + "industry_legal_and_government": 1324.8101607607755, + "industry_life_and_physical_and_social_science": 1372.098630534169, + "industry_mathematical": 1419.3116979253982, + "industry_medicine_and_healthcare": 1367.8945212911524, + "industry_software_and_it_services": 1366.6566516681737, + "industry_writing_and_literature_and_language": 1298.3711710957282, + "instruction_following": 1305.2464860503128, + "longer_query": 1327.1709703872007, + "math": 1398.176384728083, + "multi_turn": 1330.6282680826444, + "russian": 1310.8795130612496 + }, + "mistral-small-2506": { + "overall": 1339.080973468133, + "chinese": 1339.7894367321992, + "coding": 1363.1496788496784, + "creative_writing": 1305.1394260491793, + "english": 1362.0484139227117, + "exclude_ties": 1293.284949976307, + "expert": 1295.5794728202588, + "french": 1341.1535492773528, + "german": 1349.3308335564511, + "hard_prompts": 1336.2834538614534, + "hard_prompts_english": 1360.2414353959357, + "industry_business_and_management_and_financial_operations": 1326.5768800757573, + "industry_entertainment_and_sports_and_media": 1304.6596109473417, + "industry_legal_and_government": 1338.802538211407, + "industry_life_and_physical_and_social_science": 1341.7809165420588, + "industry_mathematical": 1354.7040946115433, + "industry_medicine_and_healthcare": 1344.6242471417975, + "industry_software_and_it_services": 1361.5756487351773, + "industry_writing_and_literature_and_language": 1310.8353080790837, + "instruction_following": 1311.4436063816363, + "japanese": 1281.2789774626126, + "korean": 1269.539729355136, + "longer_query": 1328.6668674342757, + "math": 1341.089680302584, + "multi_turn": 1345.3185611128597, + "russian": 1327.5339537219188, + "spanish": 1346.914551365036 + }, + "claude-sonnet-4-20250514": { + "overall": 1337.7991881618168, + "chinese": 1343.6314921450944, + "coding": 1379.827484879643, + "creative_writing": 1339.1828305984561, + "english": 1345.5853081837467, + "exclude_ties": 1293.3158597956829, + "expert": 1334.3320030895663, + "french": 1365.1208499876377, + "german": 1335.794121511767, + "hard_prompts": 1354.4452936629043, + "hard_prompts_english": 1362.5582542400075, + "industry_business_and_management_and_financial_operations": 1317.0560729438384, + "industry_entertainment_and_sports_and_media": 1319.6004415986201, + "industry_legal_and_government": 1342.6320863638757, + "industry_life_and_physical_and_social_science": 1339.6458377376641, + "industry_mathematical": 1356.2941689643114, + "industry_medicine_and_healthcare": 1329.4211383342317, + "industry_software_and_it_services": 1359.314653162021, + "industry_writing_and_literature_and_language": 1348.8564840757974, + "instruction_following": 1350.401641644779, + "japanese": 1291.1496961131793, + "korean": 1289.4106145556862, + "longer_query": 1379.9830667795627, + "math": 1357.9658405600899, + "multi_turn": 1363.6556504807168, + "russian": 1345.338793554908, + "spanish": 1350.6086238403811 + }, + "nvidia-llama-3.3-nemotron-super-49b-v1.5": { + "overall": 1337.6366152162677, + "coding": 1356.6314912884845, + "creative_writing": 1309.6789874376757, + "english": 1356.4287937369145, + "exclude_ties": 1292.1415959757128, + "expert": 1319.794488121639, + "hard_prompts": 1332.606813134466, + "hard_prompts_english": 1336.2811095167508, + "industry_business_and_management_and_financial_operations": 1309.8418809805385, + "industry_entertainment_and_sports_and_media": 1280.1577451569617, + "industry_legal_and_government": 1345.1446144394608, + "industry_life_and_physical_and_social_science": 1364.488011266962, + "industry_mathematical": 1401.0818470266393, + "industry_medicine_and_healthcare": 1364.7571175780047, + "industry_software_and_it_services": 1358.7868030499512, + "industry_writing_and_literature_and_language": 1292.774251337597, + "instruction_following": 1298.6603410372854, + "japanese": 1296.01373687748, + "longer_query": 1315.3632084897417, + "math": 1394.0816282246458, + "multi_turn": 1333.583898837883 + }, + "trinity-large-preview": { + "overall": 1337.2218516696341, + "chinese": 1350.4452964096363, + "coding": 1377.5818821218456, + "creative_writing": 1319.1424418901902, + "english": 1355.9007266061176, + "exclude_ties": 1292.8680743697296, + "expert": 1349.1297304251634, + "french": 1378.9954477593997, + "german": 1326.3515491002304, + "hard_prompts": 1348.3347391401553, + "hard_prompts_english": 1363.8676178794444, + "industry_business_and_management_and_financial_operations": 1322.8256012732147, + "industry_entertainment_and_sports_and_media": 1314.6654319040654, + "industry_legal_and_government": 1340.0882707815858, + "industry_life_and_physical_and_social_science": 1352.70754662955, + "industry_mathematical": 1350.0091302830012, + "industry_medicine_and_healthcare": 1348.0702698612324, + "industry_software_and_it_services": 1367.2561511879503, + "industry_writing_and_literature_and_language": 1328.4834027932154, + "instruction_following": 1333.921135962974, + "japanese": 1304.5685820644705, + "korean": 1266.0451167944834, + "longer_query": 1354.278465991752, + "math": 1334.6531909744112, + "multi_turn": 1343.0096960228402, + "russian": 1322.460822519674, + "spanish": 1348.7152313342026 + }, + "o3-mini-high": { + "overall": 1336.58974097813, + "chinese": 1378.3771173723662, + "coding": 1378.6815238866468, + "creative_writing": 1286.4403579135112, + "english": 1350.7719866612347, + "exclude_ties": 1291.6452178575537, + "expert": 1363.7685363956373, + "french": 1335.5779039949837, + "german": 1303.5609045432298, + "hard_prompts": 1365.7170391284897, + "hard_prompts_english": 1376.1917927913266, + "industry_business_and_management_and_financial_operations": 1312.9180648458273, + "industry_entertainment_and_sports_and_media": 1283.9239603320434, + "industry_legal_and_government": 1328.5097140289267, + "industry_life_and_physical_and_social_science": 1339.418566798604, + "industry_mathematical": 1398.2311018786363, + "industry_medicine_and_healthcare": 1329.362363613362, + "industry_software_and_it_services": 1364.6218778672292, + "industry_writing_and_literature_and_language": 1313.1007725408726, + "instruction_following": 1337.7198984755644, + "japanese": 1286.5892343335781, + "korean": 1313.8503864609117, + "longer_query": 1343.0627045862361, + "math": 1395.4877061265092, + "multi_turn": 1320.22476163797, + "russian": 1303.1479625340953, + "spanish": 1317.0732407498022 + }, + "step-1o-turbo-202506": { + "overall": 1335.0266961380607, + "chinese": 1362.4379460052032, + "coding": 1340.340939128403, + "creative_writing": 1308.2747835733358, + "english": 1351.8208266775882, + "exclude_ties": 1287.697127546303, + "expert": 1299.47444978781, + "german": 1320.9622045872986, + "hard_prompts": 1335.258147325872, + "hard_prompts_english": 1354.2674859491478, + "industry_business_and_management_and_financial_operations": 1327.3948163045075, + "industry_entertainment_and_sports_and_media": 1294.3828976935065, + "industry_legal_and_government": 1351.195690675317, + "industry_life_and_physical_and_social_science": 1357.2763255940647, + "industry_mathematical": 1340.99811036726, + "industry_medicine_and_healthcare": 1347.3918642067501, + "industry_software_and_it_services": 1353.2712123752904, + "industry_writing_and_literature_and_language": 1309.0329430610013, + "instruction_following": 1310.253169420644, + "longer_query": 1346.7208424385767, + "math": 1318.1976466597637, + "multi_turn": 1338.2925451697333, + "russian": 1326.6227618664716 + }, + "gemma-3-12b-it": { + "overall": 1334.1984471045157, + "coding": 1281.7382661097215, + "creative_writing": 1332.1102138495085, + "english": 1347.9677543047517, + "exclude_ties": 1289.2998947417639, + "expert": 1247.62254160274, + "german": 1370.7063666052782, + "hard_prompts": 1309.5757464466183, + "hard_prompts_english": 1318.304927466127, + "industry_business_and_management_and_financial_operations": 1376.3887299527432, + "industry_entertainment_and_sports_and_media": 1298.0919082321286, + "industry_legal_and_government": 1381.0456596137449, + "industry_life_and_physical_and_social_science": 1365.902622161261, + "industry_mathematical": 1335.563369073973, + "industry_medicine_and_healthcare": 1333.9085075508651, + "industry_software_and_it_services": 1313.6630594462722, + "industry_writing_and_literature_and_language": 1307.1801087517124, + "instruction_following": 1299.6539037551925, + "longer_query": 1316.9127385834095, + "math": 1307.0951002994466, + "multi_turn": 1333.4680921657728, + "russian": 1335.6698699256535 + }, + "glm-4.5v": { + "overall": 1333.6463992518784, + "chinese": 1326.2637639498994, + "coding": 1349.5551051538473, + "creative_writing": 1295.401644275138, + "english": 1360.1932555576716, + "exclude_ties": 1284.5939852413676, + "expert": 1356.3772712304917, + "hard_prompts": 1337.0135139594772, + "hard_prompts_english": 1357.0907582397817, + "industry_business_and_management_and_financial_operations": 1308.6923326529059, + "industry_entertainment_and_sports_and_media": 1303.8767307383043, + "industry_legal_and_government": 1337.7932987113768, + "industry_life_and_physical_and_social_science": 1329.9553992521496, + "industry_mathematical": 1328.829398245098, + "industry_medicine_and_healthcare": 1347.990133327231, + "industry_software_and_it_services": 1355.941334348107, + "industry_writing_and_literature_and_language": 1304.7186883229815, + "instruction_following": 1314.2533954643527, + "longer_query": 1306.6136801883067, + "math": 1349.0177976638247, + "multi_turn": 1334.4433460154125, + "russian": 1295.662406476726, + "spanish": 1333.895933084645 + }, + "deepseek-v3": { + "overall": 1332.5533769545727, + "chinese": 1337.7045253116044, + "coding": 1325.4858041504178, + "creative_writing": 1329.5948368997574, + "english": 1345.7816506151516, + "exclude_ties": 1288.9966990442153, + "expert": 1305.1081580151867, + "french": 1343.943908871045, + "german": 1323.6941212030865, + "hard_prompts": 1312.3074301812708, + "hard_prompts_english": 1324.9419482478777, + "industry_business_and_management_and_financial_operations": 1306.8937669780307, + "industry_entertainment_and_sports_and_media": 1324.283250979625, + "industry_legal_and_government": 1341.4601006405555, + "industry_life_and_physical_and_social_science": 1332.9482209996922, + "industry_mathematical": 1313.7112002083686, + "industry_medicine_and_healthcare": 1344.246408104772, + "industry_software_and_it_services": 1337.40178251628, + "industry_writing_and_literature_and_language": 1331.0609744716276, + "instruction_following": 1315.3328945213652, + "japanese": 1267.0235873125134, + "korean": 1248.3979509992778, + "longer_query": 1343.1881949174622, + "math": 1310.511930165188, + "multi_turn": 1348.9612806866535, + "russian": 1323.0401031141623, + "spanish": 1349.5463842363488 + }, + "ring-flash-2.0": { + "overall": 1331.774085349572, + "chinese": 1405.094032636259, + "coding": 1365.7935149570035, + "creative_writing": 1273.5191449798529, + "english": 1364.9414393552033, + "exclude_ties": 1283.2345064448084, + "expert": 1339.345998976447, + "hard_prompts": 1343.4818371181025, + "hard_prompts_english": 1375.4125818711661, + "industry_business_and_management_and_financial_operations": 1326.027523110173, + "industry_entertainment_and_sports_and_media": 1286.62886742999, + "industry_legal_and_government": 1308.1375332513278, + "industry_life_and_physical_and_social_science": 1339.4025730242706, + "industry_mathematical": 1359.5050428924483, + "industry_medicine_and_healthcare": 1333.11180902679, + "industry_software_and_it_services": 1366.0189265274116, + "industry_writing_and_literature_and_language": 1296.2173290013889, + "instruction_following": 1313.318460570305, + "korean": 1254.04100849514, + "longer_query": 1319.4238153865067, + "math": 1347.2849023831498, + "multi_turn": 1290.3323017222979, + "russian": 1220.985258381144, + "spanish": 1380.7455363264764 + }, + "command-a-03-2025": { + "overall": 1331.318480317182, + "chinese": 1323.8613392053144, + "coding": 1331.1079396849123, + "creative_writing": 1319.6564510290348, + "english": 1342.8809798969346, + "exclude_ties": 1281.9602106091147, + "expert": 1291.5610664904693, + "french": 1354.2945061224764, + "german": 1342.025402131383, + "hard_prompts": 1326.4614020218758, + "hard_prompts_english": 1335.5919242753785, + "industry_business_and_management_and_financial_operations": 1314.5581087110754, + "industry_entertainment_and_sports_and_media": 1306.3974425090933, + "industry_legal_and_government": 1347.5840495047642, + "industry_life_and_physical_and_social_science": 1337.9289346832766, + "industry_mathematical": 1310.6537274325285, + "industry_medicine_and_healthcare": 1333.3225738586243, + "industry_software_and_it_services": 1337.6564213261745, + "industry_writing_and_literature_and_language": 1327.6272003955205, + "instruction_following": 1308.8590421425022, + "japanese": 1289.607861121669, + "korean": 1285.0157652372307, + "longer_query": 1333.861588821299, + "math": 1299.0911514378263, + "multi_turn": 1337.5075569709893, + "russian": 1316.169758926102, + "spanish": 1335.3973614735507 + }, + "glm-4-plus-0111": { + "overall": 1330.8857947878782, + "chinese": 1388.3538474377674, + "coding": 1287.1452668335637, + "creative_writing": 1311.4392807887998, + "english": 1340.3276412120615, + "exclude_ties": 1284.1293443302334, + "expert": 1278.821918546916, + "german": 1341.0960455418424, + "hard_prompts": 1293.876768017467, + "hard_prompts_english": 1293.2884598918263, + "industry_business_and_management_and_financial_operations": 1326.4262445783331, + "industry_entertainment_and_sports_and_media": 1291.5563016429146, + "industry_legal_and_government": 1362.305432693821, + "industry_life_and_physical_and_social_science": 1350.6974344508071, + "industry_mathematical": 1290.7332182840673, + "industry_medicine_and_healthcare": 1389.045951038519, + "industry_software_and_it_services": 1320.6098297160902, + "industry_writing_and_literature_and_language": 1326.2421009524114, + "instruction_following": 1292.7835328562985, + "japanese": 1225.6933157233627, + "longer_query": 1315.8973238446695, + "math": 1287.223163417389, + "multi_turn": 1323.616802319717, + "russian": 1311.530950635395 + }, + "gemini-2.0-flash-lite-preview-02-05": { + "overall": 1329.5910326171752, + "chinese": 1338.4706439329098, + "coding": 1322.4837895731755, + "creative_writing": 1319.5901700753911, + "english": 1332.7768501299943, + "exclude_ties": 1281.1154291541357, + "expert": 1304.1921373793703, + "french": 1348.6331085557981, + "german": 1305.7119059658212, + "hard_prompts": 1324.5544644870379, + "hard_prompts_english": 1330.6856082530862, + "industry_business_and_management_and_financial_operations": 1334.739458235028, + "industry_entertainment_and_sports_and_media": 1297.596232465414, + "industry_legal_and_government": 1344.5257787316355, + "industry_life_and_physical_and_social_science": 1331.7918489910562, + "industry_mathematical": 1317.5865840301558, + "industry_medicine_and_healthcare": 1344.7824700715591, + "industry_software_and_it_services": 1336.0460839940383, + "industry_writing_and_literature_and_language": 1321.4373333101419, + "instruction_following": 1305.023395323407, + "japanese": 1302.0132073044176, + "korean": 1325.8999846369416, + "longer_query": 1320.5568528375734, + "math": 1308.9649208496608, + "multi_turn": 1307.0763555919134, + "russian": 1326.8403247597603, + "spanish": 1309.8806968793895 + }, + "qwq-32b": { + "overall": 1329.1913491984387, + "chinese": 1378.424815285088, + "coding": 1334.533857577099, + "creative_writing": 1288.8645386599949, + "english": 1349.000347756123, + "exclude_ties": 1279.548894445357, + "expert": 1324.7166464831014, + "french": 1340.9370354526918, + "german": 1311.5030671835887, + "hard_prompts": 1326.4009160740939, + "hard_prompts_english": 1339.5086690546614, + "industry_business_and_management_and_financial_operations": 1321.122223800703, + "industry_entertainment_and_sports_and_media": 1279.356093527474, + "industry_legal_and_government": 1336.9465637811281, + "industry_life_and_physical_and_social_science": 1346.8222522749998, + "industry_mathematical": 1355.7544169773766, + "industry_medicine_and_healthcare": 1340.6239907169374, + "industry_software_and_it_services": 1347.631996067726, + "industry_writing_and_literature_and_language": 1301.3534287161492, + "instruction_following": 1298.1708630833305, + "japanese": 1260.6275833063532, + "korean": 1280.1631208112676, + "longer_query": 1309.3530102396855, + "math": 1359.3491191483308, + "multi_turn": 1314.1612895569635, + "russian": 1293.5649581120174, + "spanish": 1343.1832959545245 + }, + "qwen-plus-0125": { + "overall": 1326.459972869111, + "chinese": 1346.6543000593902, + "coding": 1328.5459351074428, + "creative_writing": 1293.0912594051392, + "english": 1338.5479036647057, + "exclude_ties": 1280.399305961845, + "expert": 1327.3232657030808, + "hard_prompts": 1316.838570853467, + "hard_prompts_english": 1334.54967076148, + "industry_business_and_management_and_financial_operations": 1303.3277215212615, + "industry_entertainment_and_sports_and_media": 1270.422607928464, + "industry_legal_and_government": 1348.7868821195466, + "industry_life_and_physical_and_social_science": 1352.7649462848167, + "industry_mathematical": 1332.2308934975044, + "industry_medicine_and_healthcare": 1349.0752779791173, + "industry_software_and_it_services": 1343.683980068391, + "industry_writing_and_literature_and_language": 1295.8826188714388, + "instruction_following": 1303.2201764372915, + "japanese": 1251.5222050382044, + "longer_query": 1324.552682206861, + "math": 1325.7454656122536, + "multi_turn": 1336.0284079407788, + "russian": 1322.0833083450154 + }, + "step-2-16k-exp-202412": { + "overall": 1320.9944751939888, + "chinese": 1330.173621627499, + "coding": 1316.7203703763894, + "creative_writing": 1328.169489706552, + "english": 1344.5123753952569, + "exclude_ties": 1272.4354386289035, + "expert": 1278.0395531813292, + "hard_prompts": 1299.4742805450146, + "hard_prompts_english": 1310.3519615416853, + "industry_business_and_management_and_financial_operations": 1282.9789773065595, + "industry_entertainment_and_sports_and_media": 1291.7578890721668, + "industry_legal_and_government": 1336.6025611700584, + "industry_life_and_physical_and_social_science": 1330.3032480343493, + "industry_mathematical": 1310.0652049100759, + "industry_medicine_and_healthcare": 1353.7373212389969, + "industry_software_and_it_services": 1316.092931608202, + "industry_writing_and_literature_and_language": 1303.3768797084879, + "instruction_following": 1287.0709719856936, + "longer_query": 1306.6747444993177, + "math": 1304.350928005985, + "multi_turn": 1292.2867479344636, + "russian": 1322.8676021204856 + }, + "gpt-5-nano-high": { + "overall": 1320.0454461514146, + "chinese": 1351.9244486555033, + "coding": 1351.6500976746092, + "creative_writing": 1250.557332538993, + "english": 1323.1246255968458, + "exclude_ties": 1268.725885693747, + "expert": 1321.0213904008156, + "german": 1329.7309370860694, + "hard_prompts": 1328.4123447218421, + "hard_prompts_english": 1338.5423893117982, + "industry_business_and_management_and_financial_operations": 1309.0728694390455, + "industry_entertainment_and_sports_and_media": 1268.7702242214773, + "industry_legal_and_government": 1301.2156768471687, + "industry_life_and_physical_and_social_science": 1318.0074505850826, + "industry_mathematical": 1324.4015408028874, + "industry_medicine_and_healthcare": 1332.6727692414272, + "industry_software_and_it_services": 1346.6080867604292, + "industry_writing_and_literature_and_language": 1283.990829188049, + "instruction_following": 1305.294378575871, + "korean": 1263.9072764144344, + "longer_query": 1311.3519081783493, + "math": 1315.6498203795725, + "multi_turn": 1311.9964074885975, + "russian": 1297.3671936809892, + "spanish": 1365.003919410881 + }, + "hunyuan-turbos-20250226": { + "overall": 1319.7615969089736, + "chinese": 1323.2793367004467, + "coding": 1343.0518390369532, + "creative_writing": 1271.0007520088132, + "english": 1333.785056360653, + "exclude_ties": 1265.5994074229102, + "hard_prompts": 1325.001949851291, + "hard_prompts_english": 1338.2468364610932, + "industry_business_and_management_and_financial_operations": 1324.3122362577024, + "industry_entertainment_and_sports_and_media": 1253.241993380975, + "industry_life_and_physical_and_social_science": 1318.0370771389757, + "industry_mathematical": 1289.7375430477764, + "industry_software_and_it_services": 1353.9048298778148, + "industry_writing_and_literature_and_language": 1301.8433670193822, + "instruction_following": 1315.9764207642847, + "longer_query": 1320.3605155183834, + "math": 1293.7485205997634, + "multi_turn": 1319.3770773154815, + "russian": 1312.074063541856 + }, + "llama-3.1-nemotron-ultra-253b-v1": { + "overall": 1319.4727821583058, + "coding": 1312.4330838653352, + "creative_writing": 1314.9258416700923, + "english": 1344.3528635421114, + "exclude_ties": 1263.6377404662865, + "hard_prompts": 1316.1043834741326, + "hard_prompts_english": 1336.0456075223995, + "industry_business_and_management_and_financial_operations": 1301.1973046765067, + "industry_entertainment_and_sports_and_media": 1276.7654732514507, + "industry_life_and_physical_and_social_science": 1323.8830870293334, + "industry_software_and_it_services": 1327.411024554385, + "industry_writing_and_literature_and_language": 1300.151964324315, + "instruction_following": 1308.1031408802892, + "longer_query": 1298.8315832011776, + "math": 1359.4162714707138, + "multi_turn": 1316.9796876709308, + "russian": 1283.2310192413088 + }, + "gemini-1.5-pro-002": { + "overall": 1319.2013847882347, + "chinese": 1330.456205147118, + "coding": 1294.8388669191263, + "creative_writing": 1333.2390431508122, + "english": 1322.5637930648998, + "exclude_ties": 1266.3437846174143, + "expert": 1277.946882620156, + "french": 1302.4328464139194, + "german": 1286.2886469216028, + "hard_prompts": 1296.637723846788, + "hard_prompts_english": 1300.4414634053828, + "industry_business_and_management_and_financial_operations": 1285.792376060198, + "industry_entertainment_and_sports_and_media": 1305.6068040000573, + "industry_legal_and_government": 1328.4357317588815, + "industry_life_and_physical_and_social_science": 1320.9693388292467, + "industry_mathematical": 1324.365984421954, + "industry_medicine_and_healthcare": 1304.7940727290415, + "industry_software_and_it_services": 1304.5344146371378, + "industry_writing_and_literature_and_language": 1326.1373956941134, + "instruction_following": 1297.1827972290093, + "japanese": 1292.884107661622, + "korean": 1297.6645473334652, + "longer_query": 1307.9749624277933, + "math": 1314.579899233307, + "multi_turn": 1296.0209453908103, + "russian": 1318.6494815179142, + "spanish": 1308.72599474553 + }, + "o3-mini": { + "overall": 1318.9808325159527, + "chinese": 1323.6505510509808, + "coding": 1362.1317557646141, + "creative_writing": 1273.2626401587959, + "english": 1333.842397349521, + "exclude_ties": 1265.2260873905539, + "expert": 1330.667207946196, + "french": 1320.99731895623, + "german": 1302.6683057994505, + "hard_prompts": 1332.3553985845006, + "hard_prompts_english": 1344.3686202208596, + "industry_business_and_management_and_financial_operations": 1288.2641755143563, + "industry_entertainment_and_sports_and_media": 1272.6130112222522, + "industry_legal_and_government": 1305.5066730107133, + "industry_life_and_physical_and_social_science": 1315.355365686158, + "industry_mathematical": 1378.4440515611525, + "industry_medicine_and_healthcare": 1298.09852857434, + "industry_software_and_it_services": 1343.6898129279853, + "industry_writing_and_literature_and_language": 1294.034504625166, + "instruction_following": 1314.9580389353966, + "japanese": 1266.2087493436227, + "korean": 1261.4716157486823, + "longer_query": 1326.9458201735795, + "math": 1373.243749725922, + "multi_turn": 1312.9146538798518, + "russian": 1298.5753525197783, + "spanish": 1299.0468541211308 + }, + "o1-mini": { + "overall": 1317.1516003310458, + "chinese": 1313.285801584617, + "coding": 1362.816166021806, + "creative_writing": 1244.8111647608343, + "english": 1340.1373750538119, + "exclude_ties": 1262.9766680566963, + "expert": 1315.2817790784848, + "french": 1293.848971289364, + "german": 1278.0297134479933, + "hard_prompts": 1333.526346834645, + "hard_prompts_english": 1353.0546070300343, + "industry_business_and_management_and_financial_operations": 1286.8965698256225, + "industry_entertainment_and_sports_and_media": 1257.2599722635898, + "industry_legal_and_government": 1315.4167556942102, + "industry_life_and_physical_and_social_science": 1316.1894992708067, + "industry_mathematical": 1366.1962791067972, + "industry_medicine_and_healthcare": 1295.707656320541, + "industry_software_and_it_services": 1354.769310859524, + "industry_writing_and_literature_and_language": 1279.8358560753595, + "instruction_following": 1304.3216531688809, + "japanese": 1245.2964370607986, + "korean": 1222.6355283176335, + "longer_query": 1320.550648196248, + "math": 1358.051475184835, + "multi_turn": 1313.8728991421422, + "russian": 1281.3921290822796, + "spanish": 1301.001542846447 + }, + "qwen3-30b-a3b": { + "overall": 1317.067031182889, + "chinese": 1355.7536663289443, + "coding": 1337.9332732725252, + "creative_writing": 1272.5257859081576, + "english": 1334.9753479814408, + "exclude_ties": 1260.531311701277, + "expert": 1313.7905060102544, + "french": 1355.2130754473496, + "german": 1308.3537051676576, + "hard_prompts": 1315.2718402674545, + "hard_prompts_english": 1331.0796718037732, + "industry_business_and_management_and_financial_operations": 1308.486981119777, + "industry_entertainment_and_sports_and_media": 1259.261079067279, + "industry_legal_and_government": 1313.7586230587733, + "industry_life_and_physical_and_social_science": 1321.2667177071617, + "industry_mathematical": 1364.544856402427, + "industry_medicine_and_healthcare": 1317.6012172832843, + "industry_software_and_it_services": 1340.297154453503, + "industry_writing_and_literature_and_language": 1290.9659131621893, + "instruction_following": 1285.5298035874757, + "japanese": 1251.6924269258539, + "korean": 1261.962513571777, + "longer_query": 1313.1946093431493, + "math": 1354.686370553975, + "multi_turn": 1307.6167374132406, + "russian": 1288.7348903475659, + "spanish": 1318.0584081898833 + }, + "claude-3-7-sonnet-20250219-thinking-32k": { + "overall": 1314.4667956609196, + "chinese": 1306.1435449343098, + "coding": 1361.4632772713692, + "creative_writing": 1332.7450527283177, + "english": 1327.4038116817624, + "exclude_ties": 1261.131654078038, + "expert": 1319.679180965978, + "french": 1281.4610919764068, + "german": 1297.3662623037062, + "hard_prompts": 1332.5956065197397, + "hard_prompts_english": 1343.7120276369196, + "industry_business_and_management_and_financial_operations": 1268.750011241118, + "industry_entertainment_and_sports_and_media": 1313.0624887506087, + "industry_legal_and_government": 1301.9199183057526, + "industry_life_and_physical_and_social_science": 1299.2673919812485, + "industry_mathematical": 1337.5397655880142, + "industry_medicine_and_healthcare": 1286.9558914915474, + "industry_software_and_it_services": 1324.9408870151242, + "industry_writing_and_literature_and_language": 1336.9563183390433, + "instruction_following": 1351.8339361059811, + "japanese": 1275.334936702932, + "korean": 1246.5976793390537, + "longer_query": 1372.4269385698606, + "math": 1336.530173968892, + "multi_turn": 1336.1566532103714, + "russian": 1310.8525169968811, + "spanish": 1304.7635516840205 + }, + "olmo-3.1-32b-instruct": { + "overall": 1311.8536294905834, + "chinese": 1298.8956234177285, + "coding": 1348.808144206821, + "creative_writing": 1263.583718808392, + "english": 1348.7908476527077, + "exclude_ties": 1250.7469740509732, + "expert": 1310.8095335867301, + "french": 1332.7616874056152, + "german": 1273.417424574258, + "hard_prompts": 1322.6902662197188, + "hard_prompts_english": 1348.0314641270743, + "industry_business_and_management_and_financial_operations": 1315.9158687256909, + "industry_entertainment_and_sports_and_media": 1271.8525960227462, + "industry_legal_and_government": 1329.6325860443621, + "industry_life_and_physical_and_social_science": 1314.150240383943, + "industry_mathematical": 1349.199275468439, + "industry_medicine_and_healthcare": 1288.2826588229987, + "industry_software_and_it_services": 1350.2689964754231, + "industry_writing_and_literature_and_language": 1276.2386798267285, + "instruction_following": 1299.4188719374258, + "korean": 1206.8697835681592, + "longer_query": 1312.3705971554946, + "math": 1311.235609015299, + "multi_turn": 1308.126947295687, + "russian": 1274.797504979681, + "spanish": 1339.6550762265897 + }, + "hunyuan-turbo-0110": { + "overall": 1311.4527251197624, + "chinese": 1357.37938858814, + "coding": 1319.843213455545, + "creative_writing": 1269.5675860131778, + "english": 1328.580118001857, + "exclude_ties": 1248.9041501499414, + "hard_prompts": 1308.2790180666038, + "hard_prompts_english": 1327.7986839788311, + "industry_business_and_management_and_financial_operations": 1308.67863043355, + "industry_entertainment_and_sports_and_media": 1267.0468789588315, + "industry_life_and_physical_and_social_science": 1327.8956941007746, + "industry_mathematical": 1298.327902455453, + "industry_software_and_it_services": 1332.871008899287, + "industry_writing_and_literature_and_language": 1290.4789762376945, + "instruction_following": 1278.7304064039813, + "longer_query": 1308.9974758593783, + "math": 1272.6835820771066, + "multi_turn": 1303.2794734520517, + "russian": 1304.0610641343994 + }, + "llama-3.3-nemotron-49b-super-v1": { + "overall": 1308.4065667679743, + "chinese": 1276.643915935318, + "coding": 1296.114624739725, + "creative_writing": 1288.5554031801798, + "english": 1353.5860587810876, + "exclude_ties": 1252.0973476123168, + "hard_prompts": 1311.443757838856, + "hard_prompts_english": 1338.6949353349778, + "industry_business_and_management_and_financial_operations": 1286.4634079474254, + "industry_entertainment_and_sports_and_media": 1289.3414077336158, + "industry_life_and_physical_and_social_science": 1332.7088040550977, + "industry_mathematical": 1343.9227078115555, + "industry_software_and_it_services": 1301.3125340206125, + "industry_writing_and_literature_and_language": 1300.4081428676545, + "instruction_following": 1293.7509035938501, + "longer_query": 1299.5450360074146, + "multi_turn": 1314.9572504198632, + "russian": 1268.382851985938 + }, + "gemma-3n-e4b-it": { + "overall": 1306.2326659277942, + "chinese": 1308.2274893507422, + "coding": 1269.7130502347293, + "creative_writing": 1287.797695189666, + "english": 1319.9496787104413, + "exclude_ties": 1246.353831263254, + "expert": 1246.348041150654, + "french": 1326.434147442972, + "german": 1308.5274891316371, + "hard_prompts": 1284.6249489150882, + "hard_prompts_english": 1292.5127888990987, + "industry_business_and_management_and_financial_operations": 1295.5452378535592, + "industry_entertainment_and_sports_and_media": 1263.0828419090808, + "industry_legal_and_government": 1309.8199785702861, + "industry_life_and_physical_and_social_science": 1323.5753996767523, + "industry_mathematical": 1272.7679299199972, + "industry_medicine_and_healthcare": 1322.3121766932843, + "industry_software_and_it_services": 1309.4581109563155, + "industry_writing_and_literature_and_language": 1287.0363364010777, + "instruction_following": 1254.9986862186827, + "japanese": 1271.2092093603628, + "korean": 1261.7136850810418, + "longer_query": 1276.6899943971016, + "math": 1250.527666442155, + "multi_turn": 1277.744270934323, + "russian": 1297.6571589913674, + "spanish": 1296.043290555811 + }, + "grok-2-2024-08-13": { + "overall": 1304.6596254015146, + "chinese": 1288.3963475507803, + "coding": 1287.9648604631807, + "creative_writing": 1284.8193303901076, + "english": 1320.7560924044888, + "exclude_ties": 1244.3968123216407, + "expert": 1253.1648081953845, + "french": 1318.3534539493464, + "german": 1286.7029027618169, + "hard_prompts": 1272.4364429400998, + "hard_prompts_english": 1281.441963616655, + "industry_business_and_management_and_financial_operations": 1265.0116224510105, + "industry_entertainment_and_sports_and_media": 1282.4524457347102, + "industry_legal_and_government": 1325.8117722457114, + "industry_life_and_physical_and_social_science": 1314.811566491469, + "industry_mathematical": 1283.6318562983884, + "industry_medicine_and_healthcare": 1305.5230868925928, + "industry_software_and_it_services": 1295.4287523356456, + "industry_writing_and_literature_and_language": 1291.7858201638435, + "instruction_following": 1270.18409426621, + "japanese": 1244.5307834615633, + "korean": 1236.57803837268, + "longer_query": 1276.3976531354465, + "math": 1283.0194338344204, + "multi_turn": 1290.0747895869404, + "russian": 1284.6521536041296, + "spanish": 1279.8038455747546 + }, + "yi-lightning": { + "overall": 1301.6060082244844, + "chinese": 1318.9145525756248, + "coding": 1312.8754255466106, + "creative_writing": 1280.5915397956655, + "english": 1329.8207975818455, + "exclude_ties": 1241.6617001123138, + "expert": 1284.9518566125846, + "french": 1305.9536619993469, + "german": 1266.544836308828, + "hard_prompts": 1302.5242314996415, + "hard_prompts_english": 1323.8775014603723, + "industry_business_and_management_and_financial_operations": 1271.02150942262, + "industry_entertainment_and_sports_and_media": 1272.5666236574716, + "industry_legal_and_government": 1327.8969075352165, + "industry_life_and_physical_and_social_science": 1315.907692668135, + "industry_mathematical": 1306.3038388934872, + "industry_medicine_and_healthcare": 1305.0109603857202, + "industry_software_and_it_services": 1314.8805138287648, + "industry_writing_and_literature_and_language": 1276.1837365292915, + "instruction_following": 1278.5868695407044, + "japanese": 1228.3810037221688, + "korean": 1192.161455677723, + "longer_query": 1296.9957759535966, + "math": 1299.5058486432904, + "multi_turn": 1310.9967822953777, + "russian": 1253.951478295087, + "spanish": 1314.0690150910386 + }, + "gpt-4o-2024-05-13": { + "overall": 1300.5178926175286, + "chinese": 1276.2859599693093, + "coding": 1297.655320132622, + "creative_writing": 1291.9806220937971, + "english": 1312.428418964244, + "exclude_ties": 1238.668510009331, + "expert": 1249.417419397242, + "french": 1304.7967827799034, + "german": 1282.3041821113316, + "hard_prompts": 1281.1873854868445, + "hard_prompts_english": 1294.4616925493751, + "industry_business_and_management_and_financial_operations": 1261.441569145009, + "industry_entertainment_and_sports_and_media": 1289.8344659829784, + "industry_legal_and_government": 1313.772488701828, + "industry_life_and_physical_and_social_science": 1295.285945263459, + "industry_mathematical": 1284.20235910554, + "industry_medicine_and_healthcare": 1276.8847585845717, + "industry_software_and_it_services": 1298.2464051478128, + "industry_writing_and_literature_and_language": 1297.6384333321603, + "instruction_following": 1277.9490433011463, + "japanese": 1257.1311364655126, + "korean": 1233.5902077763717, + "longer_query": 1288.7010131333082, + "math": 1284.2173639339403, + "multi_turn": 1301.5355465147568, + "russian": 1284.3920483576433, + "spanish": 1290.9132244472721 + }, + "claude-3-7-sonnet-20250219": { + "overall": 1299.2695921378713, + "chinese": 1298.6184334722043, + "coding": 1338.932604182462, + "creative_writing": 1316.9239508445157, + "english": 1310.8165542125075, + "exclude_ties": 1240.8033300475972, + "expert": 1300.1121809711774, + "french": 1304.5909860173833, + "german": 1280.8718943250344, + "hard_prompts": 1315.0926595774686, + "hard_prompts_english": 1321.9356241422097, + "industry_business_and_management_and_financial_operations": 1273.382971791513, + "industry_entertainment_and_sports_and_media": 1300.2647739688473, + "industry_legal_and_government": 1299.6209194924554, + "industry_life_and_physical_and_social_science": 1284.5807933729398, + "industry_mathematical": 1313.7438416804548, + "industry_medicine_and_healthcare": 1258.2778631148133, + "industry_software_and_it_services": 1314.161935394603, + "industry_writing_and_literature_and_language": 1323.7904809721776, + "instruction_following": 1324.856813436807, + "japanese": 1267.5167896437001, + "korean": 1242.5074048643692, + "longer_query": 1354.8085117247274, + "math": 1318.4244431782167, + "multi_turn": 1338.3012393150398, + "russian": 1309.8494747664886, + "spanish": 1288.5229835647358 + }, + "qwen2.5-plus-1127": { + "overall": 1299.2302826116616, + "chinese": 1312.9861850918005, + "coding": 1314.7553159599186, + "creative_writing": 1262.3983176640272, + "english": 1322.0833192640018, + "exclude_ties": 1236.6507643847028, + "expert": 1288.7070248049827, + "german": 1230.847659355503, + "hard_prompts": 1299.6959472105336, + "hard_prompts_english": 1319.3738203615187, + "industry_business_and_management_and_financial_operations": 1288.739422641239, + "industry_entertainment_and_sports_and_media": 1244.6638726345125, + "industry_legal_and_government": 1306.7839649699956, + "industry_life_and_physical_and_social_science": 1317.6911656012949, + "industry_mathematical": 1304.8303801682487, + "industry_medicine_and_healthcare": 1300.790652957777, + "industry_software_and_it_services": 1317.0937125490468, + "industry_writing_and_literature_and_language": 1273.348699183864, + "instruction_following": 1274.8960536715674, + "japanese": 1207.0501035537118, + "longer_query": 1291.9621108576366, + "math": 1297.369415390096, + "multi_turn": 1299.1527215254155, + "russian": 1269.4901272432785 + }, + "olmo-3-32b-think": { + "overall": 1298.5250407794283, + "chinese": 1297.9804485229624, + "coding": 1320.706795418631, + "creative_writing": 1253.7152397348038, + "english": 1343.2162914166852, + "exclude_ties": 1237.213451451044, + "expert": 1286.153137495852, + "german": 1285.0583696915678, + "hard_prompts": 1302.4611627539066, + "hard_prompts_english": 1333.880666284047, + "industry_business_and_management_and_financial_operations": 1290.908213521801, + "industry_entertainment_and_sports_and_media": 1256.454329078035, + "industry_legal_and_government": 1295.2624479468182, + "industry_life_and_physical_and_social_science": 1311.2991426308115, + "industry_mathematical": 1330.806650724231, + "industry_medicine_and_healthcare": 1311.7307671611206, + "industry_software_and_it_services": 1332.7877098041822, + "industry_writing_and_literature_and_language": 1271.2045253621754, + "instruction_following": 1274.8529294085522, + "longer_query": 1293.9993791734998, + "math": 1315.227322714395, + "multi_turn": 1290.1646731637047, + "russian": 1255.3886346090553 + }, + "claude-3-5-sonnet-20241022": { + "overall": 1297.6697747391308, + "chinese": 1270.5507624335305, + "coding": 1342.8659965387371, + "creative_writing": 1291.831887693027, + "english": 1307.6844536713052, + "exclude_ties": 1236.0670206180976, + "expert": 1264.1189913079916, + "french": 1303.1080349190283, + "german": 1295.4790896228324, + "hard_prompts": 1305.6269765523916, + "hard_prompts_english": 1314.687134503219, + "industry_business_and_management_and_financial_operations": 1254.4822820547479, + "industry_entertainment_and_sports_and_media": 1278.9813687566157, + "industry_legal_and_government": 1277.4148691705227, + "industry_life_and_physical_and_social_science": 1270.6492196208826, + "industry_mathematical": 1306.6469456869272, + "industry_medicine_and_healthcare": 1255.755126127846, + "industry_software_and_it_services": 1316.673164784026, + "industry_writing_and_literature_and_language": 1313.0249068194785, + "instruction_following": 1297.309210161106, + "japanese": 1233.6877138880918, + "korean": 1197.318052440463, + "longer_query": 1311.5493000171132, + "math": 1306.447216959621, + "multi_turn": 1325.4569012638153, + "russian": 1303.89613020608, + "spanish": 1283.017506015668 + }, + "granite-4.1-8b": { + "overall": 1296.2363759616117, + "coding": 1314.3438965063, + "creative_writing": 1254.161154191293, + "english": 1320.7818603495248, + "exclude_ties": 1234.3636824939072, + "expert": 1306.299637786421, + "hard_prompts": 1301.7903662749134, + "hard_prompts_english": 1322.1283158792987, + "industry_business_and_management_and_financial_operations": 1282.00354484825, + "industry_entertainment_and_sports_and_media": 1245.4392855507872, + "industry_legal_and_government": 1346.1686225984317, + "industry_life_and_physical_and_social_science": 1305.637933324526, + "industry_mathematical": 1333.7273199511637, + "industry_medicine_and_healthcare": 1306.3927263049673, + "industry_software_and_it_services": 1326.8559572790418, + "industry_writing_and_literature_and_language": 1264.0790383172025, + "instruction_following": 1277.1768854541556, + "longer_query": 1286.5700403444514, + "math": 1308.1020255663902, + "multi_turn": 1282.1997183744518, + "russian": 1249.6856971273628 + }, + "molmo-2-8b": { + "overall": 1294.3580108913927, + "english": 1307.04076887506, + "exclude_ties": 1227.9588066514523, + "hard_prompts": 1293.9070481894373, + "hard_prompts_english": 1297.5526954443594, + "industry_software_and_it_services": 1300.3249311595678, + "instruction_following": 1275.1712570346658 + }, + "deepseek-v2.5-1210": { + "overall": 1294.0260877342448, + "chinese": 1316.5975126584603, + "coding": 1309.9406171230266, + "creative_writing": 1285.677113268979, + "english": 1309.647919257912, + "exclude_ties": 1229.211884552989, + "expert": 1265.6401977433752, + "german": 1258.184709481475, + "hard_prompts": 1289.639185518423, + "hard_prompts_english": 1300.6891237329883, + "industry_business_and_management_and_financial_operations": 1259.1586220707868, + "industry_entertainment_and_sports_and_media": 1275.4870935334993, + "industry_legal_and_government": 1278.6757558778922, + "industry_life_and_physical_and_social_science": 1288.6013865813388, + "industry_mathematical": 1278.097073302781, + "industry_medicine_and_healthcare": 1290.9078399785567, + "industry_software_and_it_services": 1314.2086270002014, + "industry_writing_and_literature_and_language": 1282.9850613627286, + "instruction_following": 1279.9414139589119, + "japanese": 1228.2227910243291, + "longer_query": 1300.9394614304422, + "math": 1288.349795781157, + "multi_turn": 1296.6146893374703, + "russian": 1288.1566444160871 + }, + "athene-v2-chat": { + "overall": 1291.539046888554, + "chinese": 1301.0150434953716, + "coding": 1312.6799397135933, + "creative_writing": 1234.8968623841106, + "english": 1306.8983794631645, + "exclude_ties": 1224.086653057576, + "expert": 1267.6873713772475, + "french": 1333.8507012612124, + "german": 1260.7852969633207, + "hard_prompts": 1292.8724152515942, + "hard_prompts_english": 1305.6840206351276, + "industry_business_and_management_and_financial_operations": 1273.6091888943033, + "industry_entertainment_and_sports_and_media": 1229.1362851725833, + "industry_legal_and_government": 1299.2297100424294, + "industry_life_and_physical_and_social_science": 1307.4762032049148, + "industry_mathematical": 1305.125573666715, + "industry_medicine_and_healthcare": 1294.8989407834067, + "industry_software_and_it_services": 1311.0293795201385, + "industry_writing_and_literature_and_language": 1254.03076615288, + "instruction_following": 1270.8542881626965, + "japanese": 1215.9600977625068, + "korean": 1205.5417888888387, + "longer_query": 1289.6215494866308, + "math": 1300.1155634176685, + "multi_turn": 1285.6100989615202, + "russian": 1280.7703717731688, + "spanish": 1272.3016333937603 + }, + "gemma-3-4b-it": { + "overall": 1290.7897280836482, + "coding": 1230.822448768523, + "creative_writing": 1271.2297353652625, + "english": 1299.8828653454273, + "exclude_ties": 1221.230232791308, + "expert": 1222.7991778419553, + "german": 1280.5608602473662, + "hard_prompts": 1252.8158988124724, + "hard_prompts_english": 1253.358700788115, + "industry_business_and_management_and_financial_operations": 1310.8916362089913, + "industry_entertainment_and_sports_and_media": 1233.3923687961199, + "industry_legal_and_government": 1327.5645876359672, + "industry_life_and_physical_and_social_science": 1305.5584347058448, + "industry_mathematical": 1255.649502559546, + "industry_medicine_and_healthcare": 1341.3851905971148, + "industry_software_and_it_services": 1269.4685647259869, + "industry_writing_and_literature_and_language": 1288.1598719764215, + "instruction_following": 1239.3901444819899, + "longer_query": 1273.6034520355056, + "math": 1238.8750736130792, + "multi_turn": 1254.9747706252817, + "russian": 1294.1063233499922 + }, + "glm-4-plus": { + "overall": 1289.7280306082187, + "chinese": 1294.9261386825958, + "coding": 1290.965346956868, + "creative_writing": 1264.110110169016, + "english": 1296.7167268384328, + "exclude_ties": 1221.6512240804002, + "expert": 1251.2939730382354, + "french": 1308.7480897787714, + "german": 1278.2829858661512, + "hard_prompts": 1273.3566709668103, + "hard_prompts_english": 1281.4033308147275, + "industry_business_and_management_and_financial_operations": 1272.9908577036515, + "industry_entertainment_and_sports_and_media": 1250.190588248005, + "industry_legal_and_government": 1309.3137925901033, + "industry_life_and_physical_and_social_science": 1294.9951508598174, + "industry_mathematical": 1282.512060644117, + "industry_medicine_and_healthcare": 1288.5096655814787, + "industry_software_and_it_services": 1292.1000467307222, + "industry_writing_and_literature_and_language": 1278.4737901759352, + "instruction_following": 1264.5503902738246, + "japanese": 1207.7079902998048, + "korean": 1243.2751682789522, + "longer_query": 1286.0860100562033, + "math": 1274.7174369908353, + "multi_turn": 1288.9682892903647, + "russian": 1274.007598135248, + "spanish": 1309.2970066455896 + }, + "hunyuan-large-2025-02-10": { + "overall": 1288.0005352808482, + "chinese": 1345.6907021701536, + "coding": 1307.213519029382, + "creative_writing": 1264.9329539832497, + "english": 1305.066514722254, + "exclude_ties": 1217.3854674317456, + "expert": 1275.1422124542628, + "hard_prompts": 1286.5752669486383, + "hard_prompts_english": 1296.0125542610074, + "industry_business_and_management_and_financial_operations": 1241.2006698292803, + "industry_entertainment_and_sports_and_media": 1262.3669124553815, + "industry_legal_and_government": 1280.6455017695637, + "industry_life_and_physical_and_social_science": 1291.596940044466, + "industry_mathematical": 1304.2180046347826, + "industry_medicine_and_healthcare": 1339.14072387542, + "industry_software_and_it_services": 1307.0723882293478, + "industry_writing_and_literature_and_language": 1270.8682292468643, + "instruction_following": 1277.4055983401358, + "longer_query": 1341.5534098678472, + "math": 1280.575347461318, + "multi_turn": 1284.3398512837007, + "russian": 1264.8025311723145 + }, + "llama-4-maverick-17b-128e-instruct": { + "overall": 1287.7561605644237, + "chinese": 1272.5707303825252, + "coding": 1303.0530359813981, + "creative_writing": 1267.1486879054778, + "english": 1302.741445318437, + "exclude_ties": 1221.665318582513, + "expert": 1260.02470609678, + "french": 1258.6834026134723, + "german": 1293.793005491061, + "hard_prompts": 1281.8051911966754, + "hard_prompts_english": 1296.7287458916526, + "industry_business_and_management_and_financial_operations": 1261.4222912814312, + "industry_entertainment_and_sports_and_media": 1250.5448107789925, + "industry_legal_and_government": 1285.2544957063096, + "industry_life_and_physical_and_social_science": 1288.2613277939038, + "industry_mathematical": 1294.8952140034846, + "industry_medicine_and_healthcare": 1275.6969369248773, + "industry_software_and_it_services": 1300.7360099076168, + "industry_writing_and_literature_and_language": 1277.4018725431465, + "instruction_following": 1268.445817259596, + "japanese": 1202.9994673773876, + "korean": 1203.8044784936912, + "longer_query": 1281.342721867862, + "math": 1299.8058301676126, + "multi_turn": 1287.9211491135277, + "russian": 1285.2816861425072, + "spanish": 1283.92747746319 + }, + "gpt-oss-20b": { + "overall": 1287.6989441238995, + "chinese": 1310.3347015173915, + "coding": 1308.7168367690692, + "creative_writing": 1199.8160874395842, + "english": 1301.9865335234176, + "exclude_ties": 1222.580126306807, + "expert": 1261.231522228783, + "german": 1250.7346538967627, + "hard_prompts": 1274.5722638024151, + "hard_prompts_english": 1292.0525501934555, + "industry_business_and_management_and_financial_operations": 1283.4162459656143, + "industry_entertainment_and_sports_and_media": 1217.5954885999536, + "industry_legal_and_government": 1280.369961454199, + "industry_life_and_physical_and_social_science": 1297.443449151887, + "industry_mathematical": 1304.9228074667976, + "industry_medicine_and_healthcare": 1312.271463872839, + "industry_software_and_it_services": 1312.849602460838, + "industry_writing_and_literature_and_language": 1237.1362029296909, + "instruction_following": 1237.6208331426214, + "japanese": 1249.255614439323, + "korean": 1250.9790841846507, + "longer_query": 1250.6738200573568, + "math": 1317.655287416061, + "multi_turn": 1271.4600119460902, + "russian": 1267.8989162374019, + "spanish": 1267.5315343241118 + }, + "gemini-1.5-flash-002": { + "overall": 1286.7315114687822, + "chinese": 1293.8873694629078, + "coding": 1262.0370277509198, + "creative_writing": 1285.9528963600517, + "english": 1288.6596649042094, + "exclude_ties": 1215.7122952156935, + "expert": 1232.5009985014133, + "french": 1258.99311629648, + "german": 1261.501432651948, + "hard_prompts": 1257.6120635921557, + "hard_prompts_english": 1266.534787576225, + "industry_business_and_management_and_financial_operations": 1264.4035866162749, + "industry_entertainment_and_sports_and_media": 1251.859898900837, + "industry_legal_and_government": 1302.00619764581, + "industry_life_and_physical_and_social_science": 1299.6627801413574, + "industry_mathematical": 1279.213153554807, + "industry_medicine_and_healthcare": 1288.910352493248, + "industry_software_and_it_services": 1279.0551550615653, + "industry_writing_and_literature_and_language": 1293.310017834091, + "instruction_following": 1257.7845251156054, + "japanese": 1252.7235593775129, + "korean": 1220.2809383359859, + "longer_query": 1283.983283404069, + "math": 1268.9507458125458, + "multi_turn": 1252.5185953623059, + "russian": 1286.3498880019433, + "spanish": 1240.7390765222053 + }, + "gpt-4o-mini-2024-07-18": { + "overall": 1286.5014261667675, + "chinese": 1263.0912336381712, + "coding": 1290.2089771669569, + "creative_writing": 1268.4052227036539, + "english": 1300.5253345501321, + "exclude_ties": 1216.4427488687493, + "expert": 1233.7572336288465, + "french": 1297.6389936380438, + "german": 1271.788822009981, + "hard_prompts": 1267.309620996134, + "hard_prompts_english": 1279.747793350754, + "industry_business_and_management_and_financial_operations": 1264.7597000481283, + "industry_entertainment_and_sports_and_media": 1253.6228904516765, + "industry_legal_and_government": 1305.3455260993776, + "industry_life_and_physical_and_social_science": 1286.4272174367488, + "industry_mathematical": 1271.3446146644842, + "industry_medicine_and_healthcare": 1280.4951762531496, + "industry_software_and_it_services": 1291.820533979309, + "industry_writing_and_literature_and_language": 1281.9960279387587, + "instruction_following": 1258.6412065965583, + "japanese": 1216.1048120764042, + "korean": 1195.0352214815553, + "longer_query": 1288.8112727540238, + "math": 1267.1608854554297, + "multi_turn": 1284.9046171926377, + "russian": 1274.0941924288268, + "spanish": 1277.8712839696386 + }, + "gpt-4.1-nano-2025-04-14": { + "overall": 1284.7801319889509, + "chinese": 1270.6376637997712, + "coding": 1306.2560370688502, + "creative_writing": 1260.8383021004806, + "english": 1302.7153609338736, + "exclude_ties": 1210.575629187591, + "expert": 1271.2547038973605, + "german": 1287.900261471526, + "hard_prompts": 1285.8732804501728, + "hard_prompts_english": 1292.612719236699, + "industry_business_and_management_and_financial_operations": 1266.5535644872857, + "industry_entertainment_and_sports_and_media": 1243.2417092272804, + "industry_legal_and_government": 1300.339443738135, + "industry_life_and_physical_and_social_science": 1297.3534954595239, + "industry_mathematical": 1274.1130699483188, + "industry_medicine_and_healthcare": 1275.6752614865395, + "industry_software_and_it_services": 1305.597128759488, + "industry_writing_and_literature_and_language": 1260.9363765492358, + "instruction_following": 1266.9360261781221, + "japanese": 1199.0883604231144, + "longer_query": 1282.7683610752872, + "math": 1274.020025297542, + "multi_turn": 1276.373146543835, + "russian": 1261.0845260481922 + }, + "llama-3.1-405b-instruct-bf16": { + "overall": 1283.7486579845195, + "chinese": 1241.3736022183634, + "coding": 1291.8388099793128, + "creative_writing": 1260.2975041572772, + "english": 1311.7387932172749, + "exclude_ties": 1213.8330724400128, + "expert": 1228.45723107311, + "french": 1279.8366171461894, + "german": 1252.5086357225514, + "hard_prompts": 1269.7033582569147, + "hard_prompts_english": 1292.8233195852627, + "industry_business_and_management_and_financial_operations": 1244.8217921343285, + "industry_entertainment_and_sports_and_media": 1241.3124612235738, + "industry_legal_and_government": 1301.845156156562, + "industry_life_and_physical_and_social_science": 1295.4317563637687, + "industry_mathematical": 1275.4994931752572, + "industry_medicine_and_healthcare": 1285.396607605036, + "industry_software_and_it_services": 1291.4586883485472, + "industry_writing_and_literature_and_language": 1270.4134761509702, + "instruction_following": 1258.9166971833768, + "japanese": 1208.5369430995574, + "korean": 1172.3208501382578, + "longer_query": 1265.8163163630416, + "math": 1278.07176390773, + "multi_turn": 1296.7254392471443, + "russian": 1255.0519345955313, + "spanish": 1257.5651443476827 + }, + "llama-3.1-nemotron-70b-instruct": { + "overall": 1282.726894611593, + "chinese": 1262.046821170445, + "coding": 1272.2929348663724, + "creative_writing": 1269.0703584306254, + "english": 1314.9665331168299, + "exclude_ties": 1211.405519385375, + "expert": 1241.7414806997754, + "hard_prompts": 1266.7231878622408, + "hard_prompts_english": 1284.431853032229, + "industry_business_and_management_and_financial_operations": 1261.569454857798, + "industry_entertainment_and_sports_and_media": 1249.2345833034224, + "industry_legal_and_government": 1303.084721963793, + "industry_life_and_physical_and_social_science": 1293.5227441119878, + "industry_mathematical": 1287.2048692850417, + "industry_medicine_and_healthcare": 1300.0053993785764, + "industry_software_and_it_services": 1284.3158116354616, + "industry_writing_and_literature_and_language": 1252.5003833991123, + "instruction_following": 1252.1249050226127, + "longer_query": 1238.5554471712392, + "math": 1270.872942554864, + "multi_turn": 1275.2453640903632, + "russian": 1225.7037731059554 + }, + "gpt-4o-2024-08-06": { + "overall": 1282.6225833877202, + "chinese": 1253.1834101263203, + "coding": 1284.030711779446, + "creative_writing": 1275.099021809916, + "english": 1297.4990638692666, + "exclude_ties": 1209.4354197136286, + "expert": 1240.0541228409343, + "french": 1263.4779716884555, + "german": 1257.1589599499375, + "hard_prompts": 1264.3677069905716, + "hard_prompts_english": 1278.661099463464, + "industry_business_and_management_and_financial_operations": 1231.614635477597, + "industry_entertainment_and_sports_and_media": 1267.8182479017828, + "industry_legal_and_government": 1275.305550142763, + "industry_life_and_physical_and_social_science": 1274.3580566762284, + "industry_mathematical": 1283.1479295381741, + "industry_medicine_and_healthcare": 1250.7333467373671, + "industry_software_and_it_services": 1278.5841590680789, + "industry_writing_and_literature_and_language": 1289.5483148817757, + "instruction_following": 1266.962777852638, + "japanese": 1234.5202241879465, + "korean": 1218.004048043389, + "longer_query": 1283.5768529578245, + "math": 1285.1941647265157, + "multi_turn": 1278.7164737062965, + "russian": 1270.7263112012643, + "spanish": 1267.6182635660557 + }, + "qwen-max-0919": { + "overall": 1281.9550985191484, + "chinese": 1253.042801671364, + "coding": 1288.5879111409076, + "creative_writing": 1248.60890512647, + "english": 1292.6148472107725, + "exclude_ties": 1206.189058304342, + "expert": 1247.4148811570767, + "french": 1330.683753575309, + "german": 1253.6968407120803, + "hard_prompts": 1269.813434468264, + "hard_prompts_english": 1282.362997784664, + "industry_business_and_management_and_financial_operations": 1255.951053120194, + "industry_entertainment_and_sports_and_media": 1240.8895358085065, + "industry_legal_and_government": 1283.078338673172, + "industry_life_and_physical_and_social_science": 1279.4939184715392, + "industry_mathematical": 1279.3507864745202, + "industry_medicine_and_healthcare": 1257.9625321069257, + "industry_software_and_it_services": 1291.6671867338341, + "industry_writing_and_literature_and_language": 1270.121359254746, + "instruction_following": 1262.1074368553059, + "japanese": 1205.4355761221966, + "korean": 1141.289111907034, + "longer_query": 1288.535851826738, + "math": 1274.869642651075, + "multi_turn": 1276.5917844226087, + "russian": 1272.9156062178376, + "spanish": 1288.6096615039912 + }, + "mercury": { + "overall": 1281.8872209184797, + "coding": 1325.9807253254808, + "creative_writing": 1187.4851187412307, + "english": 1304.523111689402, + "exclude_ties": 1213.7547098313353, + "hard_prompts": 1286.1107283754482, + "hard_prompts_english": 1314.8683209763572, + "industry_business_and_management_and_financial_operations": 1296.2632380729183, + "industry_entertainment_and_sports_and_media": 1205.682583035973, + "industry_life_and_physical_and_social_science": 1297.1862609809104, + "industry_software_and_it_services": 1324.0114553639655, + "industry_writing_and_literature_and_language": 1227.8118559346117, + "instruction_following": 1237.4877850600901, + "longer_query": 1265.0095869979875, + "multi_turn": 1283.7056311357164 + }, + "llama-3.1-405b-instruct-fp8": { + "overall": 1281.8863118105442, + "chinese": 1233.4577757518732, + "coding": 1284.0308831567027, + "creative_writing": 1262.4878360354687, + "english": 1307.9666624626589, + "exclude_ties": 1209.3224449245308, + "expert": 1242.4263059302107, + "french": 1271.6461349969895, + "german": 1250.3749947108427, + "hard_prompts": 1263.8349301355456, + "hard_prompts_english": 1284.9645140311218, + "industry_business_and_management_and_financial_operations": 1247.3793990886084, + "industry_entertainment_and_sports_and_media": 1250.4042658251497, + "industry_legal_and_government": 1303.4019412308821, + "industry_life_and_physical_and_social_science": 1284.6997089175975, + "industry_mathematical": 1275.9187065022475, + "industry_medicine_and_healthcare": 1269.5156408222363, + "industry_software_and_it_services": 1281.8301140732517, + "industry_writing_and_literature_and_language": 1274.082965940438, + "instruction_following": 1258.8877832384358, + "japanese": 1171.7185301915529, + "korean": 1183.6847724335173, + "longer_query": 1259.6526475803335, + "math": 1281.1793986950834, + "multi_turn": 1287.182515301463, + "russian": 1264.1276616405069, + "spanish": 1251.830673794595 + }, + "claude-3-5-sonnet-20240620": { + "overall": 1280.9957975365214, + "chinese": 1263.749482134302, + "coding": 1307.02443220825, + "creative_writing": 1238.6171731761315, + "english": 1287.3748025537727, + "exclude_ties": 1206.2975605664387, + "expert": 1246.3110188393043, + "french": 1294.3394796930697, + "german": 1270.6451673150127, + "hard_prompts": 1275.7740753997073, + "hard_prompts_english": 1284.17749200412, + "industry_business_and_management_and_financial_operations": 1234.7202983885968, + "industry_entertainment_and_sports_and_media": 1238.9504175673053, + "industry_legal_and_government": 1265.7539864599314, + "industry_life_and_physical_and_social_science": 1270.2895238558117, + "industry_mathematical": 1293.4575545436714, + "industry_medicine_and_healthcare": 1243.5674863422128, + "industry_software_and_it_services": 1289.3378040628668, + "industry_writing_and_literature_and_language": 1279.9327538103364, + "instruction_following": 1269.8811290079855, + "japanese": 1231.2082276999658, + "korean": 1200.080890181695, + "longer_query": 1275.095738308723, + "math": 1303.1840339691882, + "multi_turn": 1299.0966363880564, + "russian": 1281.3379932550497, + "spanish": 1288.315307943837 + }, + "llama-4-scout-17b-16e-instruct": { + "overall": 1280.9330092396187, + "chinese": 1253.2710103900017, + "coding": 1286.3510937128112, + "creative_writing": 1250.7412319304624, + "english": 1302.7964713297788, + "exclude_ties": 1210.5999281651402, + "expert": 1238.2345221236008, + "french": 1290.8826653105657, + "german": 1270.77017363987, + "hard_prompts": 1267.825018770773, + "hard_prompts_english": 1291.7030917030402, + "industry_business_and_management_and_financial_operations": 1258.422045826298, + "industry_entertainment_and_sports_and_media": 1244.5466560415466, + "industry_legal_and_government": 1286.220686320444, + "industry_life_and_physical_and_social_science": 1279.7617253668323, + "industry_mathematical": 1284.7415071031155, + "industry_medicine_and_healthcare": 1274.8939681155016, + "industry_software_and_it_services": 1287.3583212837689, + "industry_writing_and_literature_and_language": 1271.3432437156569, + "instruction_following": 1250.6568603674496, + "japanese": 1209.8172736991746, + "korean": 1213.0583329403626, + "longer_query": 1268.5829717534389, + "math": 1286.3183647422616, + "multi_turn": 1282.0482147034404, + "russian": 1264.233892362319, + "spanish": 1266.7988192378668 + }, + "grok-2-mini-2024-08-13": { + "overall": 1280.7769948423545, + "chinese": 1261.4790950824954, + "coding": 1269.084215460308, + "creative_writing": 1242.0797246975226, + "english": 1298.0875370191286, + "exclude_ties": 1208.762114886076, + "expert": 1237.248748039638, + "french": 1286.8922000429682, + "german": 1272.6937951771624, + "hard_prompts": 1255.2933244294518, + "hard_prompts_english": 1269.313774968528, + "industry_business_and_management_and_financial_operations": 1253.6372411181337, + "industry_entertainment_and_sports_and_media": 1233.2921810237692, + "industry_legal_and_government": 1292.433379779915, + "industry_life_and_physical_and_social_science": 1300.0164625084235, + "industry_mathematical": 1272.5554530084628, + "industry_medicine_and_healthcare": 1296.7221890538394, + "industry_software_and_it_services": 1278.4041096606097, + "industry_writing_and_literature_and_language": 1262.1267113705667, + "instruction_following": 1245.0285946216145, + "japanese": 1213.2402580820608, + "korean": 1194.557742590831, + "longer_query": 1266.49334968755, + "math": 1264.9960938035256, + "multi_turn": 1265.2903581666349, + "russian": 1260.0451167746844, + "spanish": 1277.0997353746397 + }, + "gemini-advanced-0514": { + "overall": 1278.6379306766294, + "chinese": 1273.0249647215103, + "coding": 1255.3085538729597, + "creative_writing": 1284.7087498175401, + "english": 1281.6621935544856, + "exclude_ties": 1203.644795477064, + "expert": 1214.4635985704863, + "french": 1307.298097581639, + "german": 1274.6690479393524, + "hard_prompts": 1248.5624482258784, + "hard_prompts_english": 1251.066658809349, + "industry_business_and_management_and_financial_operations": 1246.1482863944102, + "industry_entertainment_and_sports_and_media": 1271.6454665440874, + "industry_legal_and_government": 1304.3243948911345, + "industry_life_and_physical_and_social_science": 1268.1449896993504, + "industry_mathematical": 1275.8451541157388, + "industry_medicine_and_healthcare": 1248.299387734844, + "industry_software_and_it_services": 1267.3869608599548, + "industry_writing_and_literature_and_language": 1286.0925665877583, + "instruction_following": 1253.7415959249183, + "japanese": 1242.135477877896, + "korean": 1231.6806630245903, + "longer_query": 1253.584843807378, + "math": 1271.4833345634092, + "multi_turn": 1268.796738886067, + "russian": 1281.3432607942636, + "spanish": 1240.8678864358674 + }, + "mistral-small-3.1-24b-instruct-2503": { + "overall": 1277.9181890666873, + "chinese": 1246.2252238766948, + "coding": 1310.0621486973214, + "creative_writing": 1253.4664786133212, + "english": 1298.483873716821, + "exclude_ties": 1204.7409988358359, + "expert": 1259.7844219612875, + "french": 1275.4305919198328, + "german": 1262.9121515047452, + "hard_prompts": 1278.1780754003382, + "hard_prompts_english": 1296.9072054207447, + "industry_business_and_management_and_financial_operations": 1258.2980669332865, + "industry_entertainment_and_sports_and_media": 1251.2347293129756, + "industry_legal_and_government": 1271.7234579982533, + "industry_life_and_physical_and_social_science": 1266.0118256498706, + "industry_mathematical": 1272.1750141272348, + "industry_medicine_and_healthcare": 1264.6551301985899, + "industry_software_and_it_services": 1302.5253883076293, + "industry_writing_and_literature_and_language": 1269.7172669377346, + "instruction_following": 1265.1017479263833, + "japanese": 1212.1685429764407, + "korean": 1205.836329053715, + "longer_query": 1299.4216702289132, + "math": 1261.2196381202273, + "multi_turn": 1270.9025372217052, + "russian": 1263.9862315485445, + "spanish": 1280.5108123352097 + }, + "llama-3.3-70b-instruct": { + "overall": 1275.1322992224057, + "chinese": 1215.4664434242354, + "coding": 1268.9064189760625, + "creative_writing": 1252.188698951028, + "english": 1304.056973720813, + "exclude_ties": 1200.8552052621103, + "expert": 1224.4226217473515, + "french": 1282.0951477057674, + "german": 1251.0099929386975, + "hard_prompts": 1258.2851279468719, + "hard_prompts_english": 1279.1852784366106, + "industry_business_and_management_and_financial_operations": 1243.1122413172893, + "industry_entertainment_and_sports_and_media": 1241.9508037611188, + "industry_legal_and_government": 1283.9629984284875, + "industry_life_and_physical_and_social_science": 1284.4569550026895, + "industry_mathematical": 1267.1822421369425, + "industry_medicine_and_healthcare": 1273.920918334477, + "industry_software_and_it_services": 1274.813331102866, + "industry_writing_and_literature_and_language": 1255.5547932615125, + "instruction_following": 1242.7623158703504, + "japanese": 1149.2265486953083, + "korean": 1138.9962183843663, + "longer_query": 1256.4740791187883, + "math": 1267.1354630412247, + "multi_turn": 1280.5262281066434, + "russian": 1249.9787449476169, + "spanish": 1265.895536672222 + }, + "hunyuan-standard-2025-02-10": { + "overall": 1274.045848325818, + "chinese": 1318.940183342735, + "coding": 1270.1812862590027, + "creative_writing": 1242.7905942103225, + "english": 1281.4419748250966, + "exclude_ties": 1200.2446621211157, + "expert": 1247.2582574203618, + "hard_prompts": 1264.5909309399117, + "hard_prompts_english": 1275.4491597065874, + "industry_business_and_management_and_financial_operations": 1254.8322860752494, + "industry_entertainment_and_sports_and_media": 1222.7296115166992, + "industry_legal_and_government": 1289.7700017979894, + "industry_life_and_physical_and_social_science": 1284.7064737206697, + "industry_mathematical": 1288.0357661324133, + "industry_medicine_and_healthcare": 1308.4514920354345, + "industry_software_and_it_services": 1285.5459934459504, + "industry_writing_and_literature_and_language": 1256.9439079496915, + "instruction_following": 1245.1755811353687, + "longer_query": 1300.892181409395, + "math": 1273.978827151746, + "multi_turn": 1274.7219056503586, + "russian": 1257.1600946371423 + }, + "gemini-1.5-pro-001": { + "overall": 1273.5733882422128, + "chinese": 1273.8016480141796, + "coding": 1266.8121175963934, + "creative_writing": 1273.0059998820027, + "english": 1279.3125686184515, + "exclude_ties": 1195.5518413713985, + "expert": 1244.3019360389549, + "french": 1272.7754140293766, + "german": 1249.0055449907436, + "hard_prompts": 1255.250451989953, + "hard_prompts_english": 1260.5443954512923, + "industry_business_and_management_and_financial_operations": 1248.3987404461868, + "industry_entertainment_and_sports_and_media": 1251.8163241911407, + "industry_legal_and_government": 1270.1362841038203, + "industry_life_and_physical_and_social_science": 1267.6569208520677, + "industry_mathematical": 1276.1637999430031, + "industry_medicine_and_healthcare": 1211.0617414313092, + "industry_software_and_it_services": 1273.2629825626518, + "industry_writing_and_literature_and_language": 1288.6509757454323, + "instruction_following": 1253.934097300849, + "japanese": 1239.3759641239676, + "korean": 1224.2736241919295, + "longer_query": 1290.8703660890073, + "math": 1269.3425238231061, + "multi_turn": 1268.0256984498674, + "russian": 1275.1087754313348, + "spanish": 1241.9003610009577 + }, + "gpt-4-turbo-2024-04-09": { + "overall": 1271.7682370733146, + "chinese": 1241.267233413943, + "coding": 1268.8568699283092, + "creative_writing": 1269.014475440346, + "english": 1291.2011291561819, + "exclude_ties": 1190.5611131735293, + "expert": 1222.7210427890304, + "french": 1276.2704524294145, + "german": 1259.283899062333, + "hard_prompts": 1251.2897287372807, + "hard_prompts_english": 1267.7933147269218, + "industry_business_and_management_and_financial_operations": 1222.277769163453, + "industry_entertainment_and_sports_and_media": 1258.5568716274502, + "industry_legal_and_government": 1264.9594190274297, + "industry_life_and_physical_and_social_science": 1256.0461853067745, + "industry_mathematical": 1267.844870961992, + "industry_medicine_and_healthcare": 1238.2249458962385, + "industry_software_and_it_services": 1268.0528085326002, + "industry_writing_and_literature_and_language": 1278.5677311321256, + "instruction_following": 1249.706534297686, + "japanese": 1194.2189897580274, + "korean": 1186.4496130645803, + "longer_query": 1253.9149441582654, + "math": 1271.449817059206, + "multi_turn": 1266.8899614678378, + "russian": 1258.2781723629919, + "spanish": 1258.7643945436837 + }, + "deepseek-v2.5": { + "overall": 1271.326558769553, + "chinese": 1280.349023817662, + "coding": 1301.8364794705099, + "creative_writing": 1237.283461520585, + "english": 1284.3086895055785, + "exclude_ties": 1193.0825918965672, + "expert": 1239.1992649432584, + "french": 1289.7612327429565, + "german": 1226.2643019494426, + "hard_prompts": 1270.692777185896, + "hard_prompts_english": 1281.4421485775804, + "industry_business_and_management_and_financial_operations": 1248.066036388985, + "industry_entertainment_and_sports_and_media": 1230.3222846677993, + "industry_legal_and_government": 1282.237862318794, + "industry_life_and_physical_and_social_science": 1273.5826384621682, + "industry_mathematical": 1273.3880403753415, + "industry_medicine_and_healthcare": 1258.1603234964705, + "industry_software_and_it_services": 1296.0608088350796, + "industry_writing_and_literature_and_language": 1246.7265346609088, + "instruction_following": 1249.4365958910644, + "japanese": 1190.4206286789113, + "korean": 1209.215844624368, + "longer_query": 1273.1412524844354, + "math": 1270.5369751865428, + "multi_turn": 1260.5564796411059, + "russian": 1251.626940771841, + "spanish": 1246.684152881349 + }, + "olmo-3.1-32b-think": { + "overall": 1270.0869437700267, + "chinese": 1252.5357794659167, + "coding": 1288.0549644646524, + "creative_writing": 1224.5822312144164, + "english": 1335.952640657571, + "exclude_ties": 1187.6417238699808, + "expert": 1284.5327403024833, + "french": 1260.9813151776175, + "german": 1247.0406217886289, + "hard_prompts": 1272.527335902779, + "hard_prompts_english": 1321.2934893337838, + "industry_business_and_management_and_financial_operations": 1253.3877615123042, + "industry_entertainment_and_sports_and_media": 1234.9326210620302, + "industry_legal_and_government": 1279.411777226313, + "industry_life_and_physical_and_social_science": 1284.1570978077482, + "industry_mathematical": 1300.6641619674633, + "industry_medicine_and_healthcare": 1259.3717371927114, + "industry_software_and_it_services": 1298.1858575290046, + "industry_writing_and_literature_and_language": 1249.6562739824753, + "instruction_following": 1244.750795014671, + "longer_query": 1265.4999662481673, + "math": 1298.0900613328834, + "multi_turn": 1250.4952073292548, + "russian": 1190.4746849190753, + "spanish": 1289.3636162607386 + }, + "qwen2.5-72b-instruct": { + "overall": 1269.1020080769954, + "chinese": 1271.4516674146294, + "coding": 1292.9588403127634, + "creative_writing": 1221.7988851739372, + "english": 1282.6773828609043, + "exclude_ties": 1189.550038043939, + "expert": 1244.7309306649895, + "french": 1280.8879554781006, + "german": 1233.4104313301148, + "hard_prompts": 1270.9981920794692, + "hard_prompts_english": 1281.19517706413, + "industry_business_and_management_and_financial_operations": 1252.7539832727757, + "industry_entertainment_and_sports_and_media": 1211.3831922565842, + "industry_legal_and_government": 1287.987286029317, + "industry_life_and_physical_and_social_science": 1275.2765170033922, + "industry_mathematical": 1290.373095016483, + "industry_medicine_and_healthcare": 1257.3472776173721, + "industry_software_and_it_services": 1293.1527758833868, + "industry_writing_and_literature_and_language": 1246.8036878069154, + "instruction_following": 1254.3570118129492, + "japanese": 1180.1552110644193, + "korean": 1187.7462842260238, + "longer_query": 1281.785605846425, + "math": 1282.4443480839175, + "multi_turn": 1271.689948191321, + "russian": 1262.778262883333, + "spanish": 1253.7543251457641 + }, + "mistral-large-2407": { + "overall": 1266.2210821244876, + "chinese": 1239.4113388078974, + "coding": 1277.144483614626, + "creative_writing": 1243.3421386553073, + "english": 1289.0679116652395, + "exclude_ties": 1185.8894037464993, + "expert": 1231.5979603189921, + "french": 1273.6477729787696, + "german": 1253.7970406543573, + "hard_prompts": 1256.8599059545027, + "hard_prompts_english": 1270.9089462493416, + "industry_business_and_management_and_financial_operations": 1225.8400959989413, + "industry_entertainment_and_sports_and_media": 1241.9234969185195, + "industry_legal_and_government": 1276.1336876631258, + "industry_life_and_physical_and_social_science": 1266.6159960062055, + "industry_mathematical": 1265.301141008155, + "industry_medicine_and_healthcare": 1244.5156192685995, + "industry_software_and_it_services": 1277.09276991789, + "industry_writing_and_literature_and_language": 1257.760497535555, + "instruction_following": 1248.0775146589795, + "japanese": 1187.8762760164489, + "korean": 1169.771221737269, + "longer_query": 1257.5997771389127, + "math": 1261.3440018036601, + "multi_turn": 1257.1842927230116, + "russian": 1255.732945552131, + "spanish": 1235.043262073933 + }, + "mistral-large-2411": { + "overall": 1265.3544693106196, + "chinese": 1238.602924954232, + "coding": 1275.9588574822485, + "creative_writing": 1242.4134986018066, + "english": 1283.813524398784, + "exclude_ties": 1179.7076354402325, + "expert": 1207.4179995385593, + "french": 1326.2331045729036, + "german": 1241.9654218299806, + "hard_prompts": 1257.0318243304105, + "hard_prompts_english": 1269.1171057837623, + "industry_business_and_management_and_financial_operations": 1237.999640290669, + "industry_entertainment_and_sports_and_media": 1230.8665207046508, + "industry_legal_and_government": 1274.8887961929001, + "industry_life_and_physical_and_social_science": 1260.5657360001642, + "industry_mathematical": 1257.0426625756272, + "industry_medicine_and_healthcare": 1253.6783739407372, + "industry_software_and_it_services": 1275.008135331207, + "industry_writing_and_literature_and_language": 1260.3560803624273, + "instruction_following": 1249.0834105067154, + "japanese": 1170.741316892211, + "korean": 1201.787747007697, + "longer_query": 1261.586203961678, + "math": 1260.9427935670942, + "multi_turn": 1259.4360239169537, + "russian": 1250.4879126481528, + "spanish": 1264.9686990020923 + }, + "athene-70b-0725": { + "overall": 1265.2203919159665, + "chinese": 1236.5849379809151, + "coding": 1267.4720172831294, + "creative_writing": 1238.84339330973, + "english": 1288.0840533654275, + "exclude_ties": 1188.220225518238, + "expert": 1227.1710082801073, + "french": 1287.1427560516252, + "german": 1251.5131407091135, + "hard_prompts": 1243.489902147045, + "hard_prompts_english": 1256.4281183150968, + "industry_business_and_management_and_financial_operations": 1245.443308179688, + "industry_entertainment_and_sports_and_media": 1232.926791096755, + "industry_legal_and_government": 1293.2250981549573, + "industry_life_and_physical_and_social_science": 1294.1256465449137, + "industry_mathematical": 1233.165558892139, + "industry_medicine_and_healthcare": 1271.145465648136, + "industry_software_and_it_services": 1266.188822209578, + "industry_writing_and_literature_and_language": 1240.727892589779, + "instruction_following": 1227.0905974350833, + "japanese": 1138.6672100708952, + "korean": 1150.3976860536748, + "longer_query": 1232.245033962255, + "math": 1231.206778965865, + "multi_turn": 1257.1036801082937, + "russian": 1249.2741267657238, + "spanish": 1258.0978877561388 + }, + "gpt-4-1106-preview": { + "overall": 1263.5151238666338, + "chinese": 1241.4631547806243, + "coding": 1254.518930847893, + "creative_writing": 1244.814971027813, + "english": 1277.1677625344753, + "exclude_ties": 1174.3985771455045, + "expert": 1210.7523121014501, + "french": 1283.4291540256881, + "german": 1250.4279039762669, + "hard_prompts": 1241.5933953225983, + "hard_prompts_english": 1255.5524751173948, + "industry_business_and_management_and_financial_operations": 1210.8735402716966, + "industry_entertainment_and_sports_and_media": 1242.9274598995644, + "industry_legal_and_government": 1249.29545395671, + "industry_life_and_physical_and_social_science": 1255.2754199874403, + "industry_mathematical": 1270.0015529333941, + "industry_medicine_and_healthcare": 1223.1652237125088, + "industry_software_and_it_services": 1251.1194127517151, + "industry_writing_and_literature_and_language": 1264.2701517061992, + "instruction_following": 1240.899833931598, + "japanese": 1209.6572244819013, + "korean": 1183.6733068211088, + "longer_query": 1236.140364941251, + "math": 1269.0589646089934, + "multi_turn": 1256.565094350279, + "russian": 1250.2329964861078, + "spanish": 1259.7521551339169 + }, + "hunyuan-large-vision": { + "overall": 1263.2971265219737, + "chinese": 1274.1597848942574, + "coding": 1306.5086634792103, + "creative_writing": 1253.7086142032276, + "english": 1283.6597972730153, + "exclude_ties": 1187.8875924045906, + "expert": 1252.235911632709, + "hard_prompts": 1257.1881287046758, + "hard_prompts_english": 1278.1409557202994, + "industry_business_and_management_and_financial_operations": 1230.3972424236995, + "industry_entertainment_and_sports_and_media": 1220.6009394784253, + "industry_legal_and_government": 1246.7861392217146, + "industry_life_and_physical_and_social_science": 1257.1702641244615, + "industry_mathematical": 1295.427619478144, + "industry_medicine_and_healthcare": 1227.0222841658524, + "industry_software_and_it_services": 1282.1087503435326, + "industry_writing_and_literature_and_language": 1261.5223794141557, + "instruction_following": 1249.6190105492294, + "longer_query": 1277.9776308482883, + "math": 1267.759130715834, + "multi_turn": 1253.30925093234, + "russian": 1242.4257499239943 + }, + "gpt-4-0125-preview": { + "overall": 1262.2813624573478, + "chinese": 1236.5657719620629, + "coding": 1249.6753380723699, + "creative_writing": 1232.943284075674, + "english": 1276.1749621030058, + "exclude_ties": 1171.9478878525197, + "expert": 1204.782706517608, + "french": 1281.720989175687, + "german": 1245.3588737233677, + "hard_prompts": 1238.9264558356415, + "hard_prompts_english": 1253.002374415038, + "industry_business_and_management_and_financial_operations": 1216.4836241111998, + "industry_entertainment_and_sports_and_media": 1237.3443828413087, + "industry_legal_and_government": 1260.9463077322462, + "industry_life_and_physical_and_social_science": 1250.258216042038, + "industry_mathematical": 1260.8974599267763, + "industry_medicine_and_healthcare": 1230.419934202354, + "industry_software_and_it_services": 1250.518366054268, + "industry_writing_and_literature_and_language": 1257.0922741799227, + "instruction_following": 1235.2897616026735, + "japanese": 1195.5632643478966, + "korean": 1173.3258502114259, + "longer_query": 1243.9267064502533, + "math": 1267.6491910986529, + "multi_turn": 1250.288057396484, + "russian": 1239.8157229693793, + "spanish": 1246.3223027934287 + }, + "claude-3-opus-20240229": { + "overall": 1262.0878528099533, + "chinese": 1246.8112758785985, + "coding": 1264.7036699810074, + "creative_writing": 1235.3421674567635, + "english": 1262.889627004131, + "exclude_ties": 1175.2214543638133, + "expert": 1222.0888602294103, + "french": 1275.8414211307609, + "german": 1258.2426755021643, + "hard_prompts": 1245.8241778053568, + "hard_prompts_english": 1245.0648801801426, + "industry_business_and_management_and_financial_operations": 1221.9670530887183, + "industry_entertainment_and_sports_and_media": 1220.492257872856, + "industry_legal_and_government": 1264.5912291295044, + "industry_life_and_physical_and_social_science": 1254.259604893623, + "industry_mathematical": 1263.706768983415, + "industry_medicine_and_healthcare": 1230.078825202203, + "industry_software_and_it_services": 1259.8094581093778, + "industry_writing_and_literature_and_language": 1268.5054487567636, + "instruction_following": 1248.5719730627602, + "japanese": 1204.3718656968147, + "korean": 1187.1225597431057, + "longer_query": 1259.1892297866352, + "math": 1272.3984382173335, + "multi_turn": 1275.2333225509205, + "russian": 1278.8566930025363, + "spanish": 1245.041087578138 + }, + "llama-3.1-70b-instruct": { + "overall": 1260.9918104277288, + "chinese": 1214.2943598497468, + "coding": 1260.1914789186196, + "creative_writing": 1232.4047718296308, + "english": 1293.7100364152598, + "exclude_ties": 1177.0549665794717, + "expert": 1208.1143306221377, + "french": 1261.9288173699047, + "german": 1221.5301891517497, + "hard_prompts": 1241.600647584643, + "hard_prompts_english": 1263.043710196976, + "industry_business_and_management_and_financial_operations": 1234.9470459157892, + "industry_entertainment_and_sports_and_media": 1230.2133201119377, + "industry_legal_and_government": 1283.5608904768521, + "industry_life_and_physical_and_social_science": 1276.419616271, + "industry_mathematical": 1253.3739941265953, + "industry_medicine_and_healthcare": 1256.061253778064, + "industry_software_and_it_services": 1260.842588728848, + "industry_writing_and_literature_and_language": 1235.6012323659943, + "instruction_following": 1231.5186165907362, + "japanese": 1132.7525744588484, + "korean": 1140.1765599774592, + "longer_query": 1241.051245786627, + "math": 1251.5117849957544, + "multi_turn": 1256.0450905640319, + "russian": 1232.5273612568412, + "spanish": 1251.7720562912523 + }, + "amazon-nova-pro-v1.0": { + "overall": 1258.7374859951742, + "chinese": 1243.1057091463936, + "coding": 1270.4228456319113, + "creative_writing": 1212.9129640867377, + "english": 1275.9138482352666, + "exclude_ties": 1170.1704541388096, + "expert": 1210.1318469158887, + "french": 1272.5177635600467, + "german": 1243.0659867009035, + "hard_prompts": 1245.8130526108525, + "hard_prompts_english": 1259.988396997317, + "industry_business_and_management_and_financial_operations": 1229.056296891497, + "industry_entertainment_and_sports_and_media": 1210.120782218643, + "industry_legal_and_government": 1281.4958175918719, + "industry_life_and_physical_and_social_science": 1255.5221379310588, + "industry_mathematical": 1261.0128616800566, + "industry_medicine_and_healthcare": 1259.3624085830809, + "industry_software_and_it_services": 1271.5775223509609, + "industry_writing_and_literature_and_language": 1246.483430585306, + "instruction_following": 1235.601673760232, + "japanese": 1200.5123030799255, + "korean": 1202.9131561888253, + "longer_query": 1254.9525408582817, + "math": 1251.4060181338255, + "multi_turn": 1245.3563975246643, + "russian": 1238.8834822102015, + "spanish": 1179.4845564855752 + }, + "llama-3.1-tulu-3-70b": { + "overall": 1255.8156748929364, + "chinese": 1248.3196269903237, + "coding": 1235.9172151331331, + "creative_writing": 1231.8382348969524, + "english": 1272.7732680962508, + "exclude_ties": 1170.2053088774155, + "hard_prompts": 1220.4036125377772, + "hard_prompts_english": 1238.3391525191485, + "industry_business_and_management_and_financial_operations": 1225.935311072591, + "industry_entertainment_and_sports_and_media": 1240.4941507508893, + "industry_life_and_physical_and_social_science": 1267.1966395834108, + "industry_mathematical": 1246.156356314803, + "industry_software_and_it_services": 1242.7460278116164, + "industry_writing_and_literature_and_language": 1254.1821923141447, + "instruction_following": 1232.8677380672302, + "longer_query": 1224.4113550095576, + "math": 1241.7804545887711, + "multi_turn": 1251.5159620413406, + "russian": 1244.513234870003 + }, + "claude-3-5-haiku-20241022": { + "overall": 1255.3883577562815, + "chinese": 1224.1168346493391, + "coding": 1286.7557312317783, + "creative_writing": 1232.7084745510901, + "english": 1268.8272505348468, + "exclude_ties": 1171.7857701242315, + "expert": 1207.261185197001, + "french": 1263.7254607865884, + "german": 1239.460502187343, + "hard_prompts": 1251.192820439157, + "hard_prompts_english": 1267.733444612834, + "industry_business_and_management_and_financial_operations": 1222.7257951078186, + "industry_entertainment_and_sports_and_media": 1224.1130442711299, + "industry_legal_and_government": 1258.2853889200999, + "industry_life_and_physical_and_social_science": 1247.1497811896243, + "industry_mathematical": 1249.064124951211, + "industry_medicine_and_healthcare": 1225.636213722657, + "industry_software_and_it_services": 1273.1141952482906, + "industry_writing_and_literature_and_language": 1256.0032351381838, + "instruction_following": 1240.5828870288951, + "japanese": 1175.1682703696977, + "korean": 1172.7767998468032, + "longer_query": 1260.5050546631533, + "math": 1244.3013301077335, + "multi_turn": 1264.6648295633424, + "russian": 1252.815744669575, + "spanish": 1251.6762871229123 + }, + "magistral-medium-2506": { + "overall": 1254.4342533619638, + "chinese": 1224.4232795236335, + "coding": 1319.644846931762, + "creative_writing": 1247.7657410615338, + "english": 1276.5430656969427, + "exclude_ties": 1177.6150462243606, + "expert": 1219.7189776462237, + "french": 1265.7927348854398, + "german": 1244.8407497853968, + "hard_prompts": 1267.8265698675887, + "hard_prompts_english": 1290.8894315851385, + "industry_business_and_management_and_financial_operations": 1223.3535882606946, + "industry_entertainment_and_sports_and_media": 1243.3899034188041, + "industry_legal_and_government": 1234.7022787177493, + "industry_life_and_physical_and_social_science": 1235.605918827801, + "industry_mathematical": 1254.7485135175407, + "industry_medicine_and_healthcare": 1245.0882487017125, + "industry_software_and_it_services": 1282.9734710519813, + "industry_writing_and_literature_and_language": 1257.9241754766263, + "instruction_following": 1255.5390232798582, + "japanese": 1175.2664290244502, + "korean": 1134.2783348771786, + "longer_query": 1294.9154397115785, + "math": 1249.2188456971191, + "multi_turn": 1274.8654519652032, + "russian": 1225.9685962064596, + "spanish": 1273.2385056972862 + }, + "reka-core-20240904": { + "overall": 1248.46013362067, + "chinese": 1240.1017331735784, + "coding": 1230.5409224037498, + "creative_writing": 1236.7699950695203, + "english": 1256.0790294083038, + "exclude_ties": 1156.8460495096697, + "expert": 1212.9796303517337, + "german": 1243.075906222074, + "hard_prompts": 1211.3477865622945, + "hard_prompts_english": 1217.3507445813257, + "industry_business_and_management_and_financial_operations": 1216.9996632079806, + "industry_entertainment_and_sports_and_media": 1207.6050692841827, + "industry_legal_and_government": 1273.5177065899477, + "industry_life_and_physical_and_social_science": 1265.8767770284885, + "industry_mathematical": 1238.944879707795, + "industry_medicine_and_healthcare": 1277.0848318110486, + "industry_software_and_it_services": 1232.6819293119943, + "industry_writing_and_literature_and_language": 1242.29614592232, + "instruction_following": 1212.177556426909, + "longer_query": 1220.4882987446313, + "math": 1221.5395942710588, + "multi_turn": 1226.4725222028837, + "russian": 1248.0917764087658 + }, + "ibm-granite-h-small": { + "overall": 1241.6031141981318, + "chinese": 1246.1168397844835, + "coding": 1249.5113437100083, + "creative_writing": 1211.3974508325218, + "english": 1264.0307496700057, + "exclude_ties": 1156.1216621382414, + "expert": 1241.1120182568222, + "hard_prompts": 1240.079701096113, + "hard_prompts_english": 1267.8927141312502, + "industry_business_and_management_and_financial_operations": 1212.315797345826, + "industry_entertainment_and_sports_and_media": 1213.4734954942169, + "industry_legal_and_government": 1230.8215731571177, + "industry_life_and_physical_and_social_science": 1254.1150543746849, + "industry_mathematical": 1274.742435618412, + "industry_medicine_and_healthcare": 1216.9102401483262, + "industry_software_and_it_services": 1254.195510357098, + "industry_writing_and_literature_and_language": 1228.2186903617794, + "instruction_following": 1221.4082487780536, + "longer_query": 1241.550185547594, + "math": 1251.0846655741652, + "multi_turn": 1247.5051681140703, + "russian": 1183.9368459288705, + "spanish": 1258.5424518861423 + }, + "gemini-1.5-flash-001": { + "overall": 1239.3807282876598, + "chinese": 1233.2643284341727, + "coding": 1236.1723301242773, + "creative_writing": 1222.6698333002018, + "english": 1243.7482600275566, + "exclude_ties": 1142.8391092939482, + "expert": 1198.7310438032587, + "french": 1241.5946357791154, + "german": 1216.2866466810192, + "hard_prompts": 1220.0012432735512, + "hard_prompts_english": 1226.3526178286074, + "industry_business_and_management_and_financial_operations": 1225.3098697886285, + "industry_entertainment_and_sports_and_media": 1207.9170026971615, + "industry_legal_and_government": 1247.569036273426, + "industry_life_and_physical_and_social_science": 1232.232798115508, + "industry_mathematical": 1237.959515146852, + "industry_medicine_and_healthcare": 1185.302876020208, + "industry_software_and_it_services": 1242.5219428392265, + "industry_writing_and_literature_and_language": 1251.7019281657097, + "instruction_following": 1212.216833465865, + "japanese": 1186.691114205012, + "korean": 1200.817403303448, + "longer_query": 1251.761486896497, + "math": 1228.5079351531738, + "multi_turn": 1227.4056058481383, + "russian": 1238.1730959649021, + "spanish": 1223.1447578842917 + }, + "jamba-1.5-large": { + "overall": 1237.1041531700648, + "chinese": 1202.0777391129643, + "coding": 1228.0344774321968, + "creative_writing": 1212.2666331728792, + "english": 1266.7079072026445, + "exclude_ties": 1139.839987213843, + "expert": 1217.0714334918516, + "german": 1199.678203387743, + "hard_prompts": 1218.0366253222844, + "hard_prompts_english": 1232.8393611343945, + "industry_business_and_management_and_financial_operations": 1196.167140828233, + "industry_entertainment_and_sports_and_media": 1218.5879303490697, + "industry_legal_and_government": 1264.2452861905685, + "industry_life_and_physical_and_social_science": 1249.745673687452, + "industry_mathematical": 1227.418180402479, + "industry_medicine_and_healthcare": 1195.3537773838143, + "industry_software_and_it_services": 1237.4772418119232, + "industry_writing_and_literature_and_language": 1227.551171880326, + "instruction_following": 1212.9014412767917, + "japanese": 1118.5682861288049, + "longer_query": 1206.4653657549939, + "math": 1220.8761826035002, + "multi_turn": 1219.7679115629153, + "russian": 1194.556884948057 + }, + "mistral-small-24b-instruct-2501": { + "overall": 1233.5804483839947, + "chinese": 1203.581005471907, + "coding": 1246.228829941966, + "creative_writing": 1195.291729073425, + "english": 1255.8981730838238, + "exclude_ties": 1129.413541330417, + "expert": 1201.3117716400727, + "french": 1204.8960256175383, + "german": 1211.7773981260937, + "hard_prompts": 1233.311607695808, + "hard_prompts_english": 1244.2237893340457, + "industry_business_and_management_and_financial_operations": 1205.3005195785836, + "industry_entertainment_and_sports_and_media": 1184.236727615134, + "industry_legal_and_government": 1218.9317820880747, + "industry_life_and_physical_and_social_science": 1235.1071219794792, + "industry_mathematical": 1242.9645190975732, + "industry_medicine_and_healthcare": 1214.5285551338634, + "industry_software_and_it_services": 1249.0073275165234, + "industry_writing_and_literature_and_language": 1213.829319668628, + "instruction_following": 1214.2612599504791, + "japanese": 1111.7885143252, + "korean": 1188.40752129469, + "longer_query": 1245.9208036216942, + "math": 1239.9967383270393, + "multi_turn": 1216.4265148806162, + "russian": 1215.051182753064 + }, + "gemma-2-27b-it": { + "overall": 1231.450731116251, + "chinese": 1220.0601942378878, + "coding": 1211.4328467383175, + "creative_writing": 1241.1745988588973, + "english": 1241.9469803261923, + "exclude_ties": 1129.0485971756732, + "expert": 1170.9157093836998, + "french": 1248.1600495041434, + "german": 1208.8486090085253, + "hard_prompts": 1198.7192953126146, + "hard_prompts_english": 1207.5326672330964, + "industry_business_and_management_and_financial_operations": 1207.5238366130793, + "industry_entertainment_and_sports_and_media": 1212.8525677216928, + "industry_legal_and_government": 1241.2325334036518, + "industry_life_and_physical_and_social_science": 1228.477589432481, + "industry_mathematical": 1221.3497109737812, + "industry_medicine_and_healthcare": 1196.580243498443, + "industry_software_and_it_services": 1228.8810535165958, + "industry_writing_and_literature_and_language": 1248.6151055680905, + "instruction_following": 1206.294403701385, + "japanese": 1175.8520920165524, + "korean": 1173.5485158114805, + "longer_query": 1230.6635369012147, + "math": 1211.8396972712876, + "multi_turn": 1223.609264958812, + "russian": 1232.7465587749023, + "spanish": 1226.8302422241854 + }, + "qwen2.5-coder-32b-instruct": { + "overall": 1230.129453440933, + "chinese": 1220.892915659597, + "coding": 1276.1492400328646, + "creative_writing": 1174.4153311583761, + "english": 1248.0131551416393, + "exclude_ties": 1126.5929165434436, + "expert": 1220.008315195097, + "hard_prompts": 1251.669710222031, + "hard_prompts_english": 1271.0895600421627, + "industry_business_and_management_and_financial_operations": 1223.766356961923, + "industry_entertainment_and_sports_and_media": 1150.5739650748274, + "industry_legal_and_government": 1256.785038123197, + "industry_life_and_physical_and_social_science": 1221.815586484283, + "industry_mathematical": 1243.1324509757947, + "industry_medicine_and_healthcare": 1185.3873016235736, + "industry_software_and_it_services": 1261.9497489015794, + "industry_writing_and_literature_and_language": 1190.9381425511774, + "instruction_following": 1223.0941774529492, + "longer_query": 1250.9029645403511, + "math": 1250.395301888098, + "multi_turn": 1222.2169268876942, + "russian": 1227.0116084163074 + }, + "command-r-plus-08-2024": { + "overall": 1228.9779390109643, + "chinese": 1225.4569654167117, + "coding": 1187.5954886043921, + "creative_writing": 1235.2081450450275, + "english": 1237.6345671760928, + "exclude_ties": 1125.5360159063512, + "expert": 1172.9603752585676, + "german": 1215.7845259972055, + "hard_prompts": 1185.9308533720373, + "hard_prompts_english": 1196.090234652446, + "industry_business_and_management_and_financial_operations": 1215.0074179421063, + "industry_entertainment_and_sports_and_media": 1223.748168924526, + "industry_legal_and_government": 1268.4441496022346, + "industry_life_and_physical_and_social_science": 1252.2283579098387, + "industry_mathematical": 1197.799193327407, + "industry_medicine_and_healthcare": 1236.733245770012, + "industry_software_and_it_services": 1198.0193066037061, + "industry_writing_and_literature_and_language": 1240.9493868089162, + "instruction_following": 1197.00329938639, + "japanese": 1166.5374333923364, + "longer_query": 1230.4457852769178, + "math": 1187.6747919729628, + "multi_turn": 1212.610091803616, + "russian": 1226.1519905482078 + }, + "amazon-nova-lite-v1.0": { + "overall": 1228.548656736201, + "chinese": 1224.7056219807628, + "coding": 1239.3138924072464, + "creative_writing": 1197.4364867965014, + "english": 1242.5071180512484, + "exclude_ties": 1120.6233076661206, + "expert": 1200.845816709636, + "french": 1238.4406860595209, + "german": 1228.703978657326, + "hard_prompts": 1220.145830737671, + "hard_prompts_english": 1227.2496519239608, + "industry_business_and_management_and_financial_operations": 1218.7901145135936, + "industry_entertainment_and_sports_and_media": 1176.0391007309888, + "industry_legal_and_government": 1238.3348772685001, + "industry_life_and_physical_and_social_science": 1232.6690381615585, + "industry_mathematical": 1234.3759965038216, + "industry_medicine_and_healthcare": 1211.5501359162415, + "industry_software_and_it_services": 1242.1217153864259, + "industry_writing_and_literature_and_language": 1211.0476578160633, + "instruction_following": 1205.330421506204, + "japanese": 1154.0085876327803, + "korean": 1153.2125344927265, + "longer_query": 1233.8702059030434, + "math": 1226.4610155042542, + "multi_turn": 1198.5271117611592, + "russian": 1214.6242788820236, + "spanish": 1222.8954388983088 + }, + "llama-3.1-nemotron-51b-instruct": { + "overall": 1228.0634138596943, + "chinese": 1178.604345212154, + "coding": 1223.449513787656, + "creative_writing": 1213.7009141851597, + "english": 1267.1404145912443, + "exclude_ties": 1122.7778463498737, + "expert": 1166.313872053268, + "hard_prompts": 1203.8704750235856, + "hard_prompts_english": 1206.9842635241343, + "industry_business_and_management_and_financial_operations": 1215.2802978882246, + "industry_entertainment_and_sports_and_media": 1191.2688825167504, + "industry_legal_and_government": 1252.243802794101, + "industry_life_and_physical_and_social_science": 1226.2622011066496, + "industry_mathematical": 1228.3782753299613, + "industry_medicine_and_healthcare": 1185.4875248221442, + "industry_software_and_it_services": 1232.0771736294973, + "industry_writing_and_literature_and_language": 1219.771665154008, + "instruction_following": 1201.0027580720123, + "longer_query": 1205.3597976721428, + "math": 1229.750975773809, + "multi_turn": 1226.7411796720799, + "russian": 1185.254442131722 + }, + "gemma-2-9b-it-simpo": { + "overall": 1227.1441185030342, + "chinese": 1224.1571225116681, + "coding": 1191.4004302731933, + "creative_writing": 1239.320622492361, + "english": 1242.4414079296157, + "exclude_ties": 1122.841797513643, + "expert": 1153.920643507327, + "german": 1218.3011255196116, + "hard_prompts": 1196.4956800641914, + "hard_prompts_english": 1202.841964447706, + "industry_business_and_management_and_financial_operations": 1222.176817139572, + "industry_entertainment_and_sports_and_media": 1215.051208490047, + "industry_legal_and_government": 1268.5142469684442, + "industry_life_and_physical_and_social_science": 1231.3075684769979, + "industry_mathematical": 1195.8255584642125, + "industry_medicine_and_healthcare": 1220.532014583242, + "industry_software_and_it_services": 1212.4219773566201, + "industry_writing_and_literature_and_language": 1225.785097824154, + "instruction_following": 1191.346323667739, + "japanese": 1140.0621024950424, + "longer_query": 1226.3345879930373, + "math": 1172.5770091181046, + "multi_turn": 1219.568525329697, + "russian": 1223.4002476893418 + }, + "glm-4-0520": { + "overall": 1225.9877324468755, + "chinese": 1227.572269154925, + "coding": 1228.140528843338, + "creative_writing": 1198.4741770729115, + "english": 1241.984288987288, + "exclude_ties": 1124.0862760100113, + "expert": 1178.491761541284, + "german": 1183.718158533724, + "hard_prompts": 1212.3516983640948, + "hard_prompts_english": 1220.4006592448575, + "industry_business_and_management_and_financial_operations": 1176.643293413632, + "industry_entertainment_and_sports_and_media": 1185.6488277737812, + "industry_legal_and_government": 1244.1292838131217, + "industry_life_and_physical_and_social_science": 1226.5724819471502, + "industry_mathematical": 1229.5898989632196, + "industry_medicine_and_healthcare": 1202.0146037231343, + "industry_software_and_it_services": 1226.7671857872417, + "industry_writing_and_literature_and_language": 1210.442664180974, + "instruction_following": 1203.041100518194, + "korean": 1041.82768631507, + "longer_query": 1206.8511978549934, + "math": 1217.8207826227442, + "multi_turn": 1217.8525496817633, + "russian": 1212.0556931775563 + }, + "gemini-1.5-flash-8b-001": { + "overall": 1225.9396930258304, + "chinese": 1229.9310440380184, + "coding": 1218.149741878232, + "creative_writing": 1218.0442395188907, + "english": 1231.3639833734615, + "exclude_ties": 1118.2507244328342, + "expert": 1184.541797165876, + "french": 1234.5800326140256, + "german": 1205.8797029282985, + "hard_prompts": 1209.805083736594, + "hard_prompts_english": 1216.3250555223199, + "industry_business_and_management_and_financial_operations": 1210.771585574036, + "industry_entertainment_and_sports_and_media": 1185.4279480927385, + "industry_legal_and_government": 1247.6202301554304, + "industry_life_and_physical_and_social_science": 1232.5705208940876, + "industry_mathematical": 1225.6082761699347, + "industry_medicine_and_healthcare": 1212.657798977542, + "industry_software_and_it_services": 1228.2667069174795, + "industry_writing_and_literature_and_language": 1226.6492693508226, + "instruction_following": 1199.3908152392846, + "japanese": 1150.407154740723, + "korean": 1139.7634099617844, + "longer_query": 1218.7270959316822, + "math": 1206.5072814414084, + "multi_turn": 1185.1528287421766, + "russian": 1234.5975765687494, + "spanish": 1210.6231519881762 + }, + "nemotron-4-340b-instruct": { + "overall": 1224.8929954347655, + "chinese": 1212.8403122683576, + "coding": 1209.763149146358, + "creative_writing": 1203.0698581456104, + "english": 1236.5641221399453, + "exclude_ties": 1122.6150741659435, + "expert": 1170.6806945600422, + "french": 1197.2338631211255, + "german": 1191.776766445781, + "hard_prompts": 1201.9109684887044, + "hard_prompts_english": 1203.3453026777365, + "industry_business_and_management_and_financial_operations": 1184.7303775234177, + "industry_entertainment_and_sports_and_media": 1208.1411442830383, + "industry_legal_and_government": 1245.6704749089695, + "industry_life_and_physical_and_social_science": 1228.7612727405453, + "industry_mathematical": 1215.7731354372659, + "industry_medicine_and_healthcare": 1193.478514004544, + "industry_software_and_it_services": 1216.3013200549494, + "industry_writing_and_literature_and_language": 1223.2536184455175, + "instruction_following": 1203.4038956455001, + "japanese": 1128.4144903460792, + "korean": 1147.5936766398743, + "longer_query": 1224.6581951990452, + "math": 1215.8917102474868, + "multi_turn": 1213.5877028899724, + "russian": 1218.1668892347025, + "spanish": 1193.973337009287 + }, + "c4ai-aya-expanse-32b": { + "overall": 1224.093919844458, + "chinese": 1209.6348720479828, + "coding": 1197.0930748105245, + "creative_writing": 1200.32506538927, + "english": 1228.8877586367498, + "exclude_ties": 1115.2505140807566, + "expert": 1181.4785542928257, + "french": 1248.5775886653541, + "german": 1198.5151260214402, + "hard_prompts": 1193.5120049007564, + "hard_prompts_english": 1197.6298083344846, + "industry_business_and_management_and_financial_operations": 1209.9345862902614, + "industry_entertainment_and_sports_and_media": 1183.8163720516645, + "industry_legal_and_government": 1247.713452799565, + "industry_life_and_physical_and_social_science": 1232.9068428013863, + "industry_mathematical": 1207.9215146750912, + "industry_medicine_and_healthcare": 1221.419307758255, + "industry_software_and_it_services": 1213.3956700345843, + "industry_writing_and_literature_and_language": 1222.0691972824416, + "instruction_following": 1196.3233451667052, + "japanese": 1163.559866181587, + "korean": 1157.6005621033794, + "longer_query": 1227.6709925133764, + "math": 1199.643568998972, + "multi_turn": 1189.908996268733, + "russian": 1225.8781486851558, + "spanish": 1191.3877582982182 + }, + "llama-3-70b-instruct": { + "overall": 1220.8764015644288, + "chinese": 1113.2732447484223, + "coding": 1206.6385029623427, + "creative_writing": 1210.2743060538983, + "english": 1277.4389219570678, + "exclude_ties": 1115.853203051729, + "expert": 1148.502436953816, + "french": 1232.8901020985877, + "german": 1168.9194641095567, + "hard_prompts": 1195.2485771863508, + "hard_prompts_english": 1232.1126238043928, + "industry_business_and_management_and_financial_operations": 1172.2034061767783, + "industry_entertainment_and_sports_and_media": 1195.7818544684342, + "industry_legal_and_government": 1227.90252131455, + "industry_life_and_physical_and_social_science": 1226.1837446534714, + "industry_mathematical": 1213.4125454357882, + "industry_medicine_and_healthcare": 1212.9580297955374, + "industry_software_and_it_services": 1205.3081174845825, + "industry_writing_and_literature_and_language": 1205.9707802592482, + "instruction_following": 1194.4588603988454, + "japanese": 1016.9284972876476, + "korean": 1016.447465633147, + "longer_query": 1174.0965385619143, + "math": 1217.7204813711983, + "multi_turn": 1222.8378222776673, + "russian": 1157.556397965544, + "spanish": 1239.88619653536 + }, + "claude-3-sonnet-20240229": { + "overall": 1218.117751310066, + "chinese": 1187.6634796748028, + "coding": 1223.2376392959509, + "creative_writing": 1186.7098626511636, + "english": 1226.161459315608, + "exclude_ties": 1108.66900938849, + "expert": 1171.8995981975936, + "french": 1229.8002151075086, + "german": 1203.5949514173662, + "hard_prompts": 1197.3815144816035, + "hard_prompts_english": 1204.201075934305, + "industry_business_and_management_and_financial_operations": 1173.8913150714188, + "industry_entertainment_and_sports_and_media": 1171.732051220797, + "industry_legal_and_government": 1213.8673688750062, + "industry_life_and_physical_and_social_science": 1209.5448105735381, + "industry_mathematical": 1212.9496443087821, + "industry_medicine_and_healthcare": 1187.1345704102478, + "industry_software_and_it_services": 1221.8424603710591, + "industry_writing_and_literature_and_language": 1217.6151121270154, + "instruction_following": 1199.6652761619202, + "japanese": 1131.5463617392834, + "korean": 1127.5108822264497, + "longer_query": 1211.147244885295, + "math": 1213.2613958977483, + "multi_turn": 1226.75181170627, + "russian": 1225.4546329663658, + "spanish": 1203.0549860585106 + }, + "reka-flash-20240904": { + "overall": 1217.9992937328666, + "chinese": 1219.789761960267, + "coding": 1190.485433974128, + "creative_writing": 1202.755728441818, + "english": 1222.8664790794778, + "exclude_ties": 1102.9864571663243, + "expert": 1181.3450839067737, + "german": 1178.2000017769637, + "hard_prompts": 1181.8032121996484, + "hard_prompts_english": 1187.9098174729793, + "industry_business_and_management_and_financial_operations": 1191.2551687223513, + "industry_entertainment_and_sports_and_media": 1177.5334944686247, + "industry_legal_and_government": 1217.3385041018237, + "industry_life_and_physical_and_social_science": 1223.3341967953243, + "industry_mathematical": 1201.926664633398, + "industry_medicine_and_healthcare": 1199.0502182091127, + "industry_software_and_it_services": 1199.7414722897042, + "industry_writing_and_literature_and_language": 1216.980467979788, + "instruction_following": 1185.614427117839, + "longer_query": 1191.8255627652115, + "math": 1195.164114556157, + "multi_turn": 1188.685699444227, + "russian": 1210.3595706327483 + }, + "olmo-2-0325-32b-instruct": { + "overall": 1217.9620288540864, + "chinese": 1191.1870795772313, + "coding": 1210.6149410780947, + "creative_writing": 1199.153038636352, + "english": 1254.9823719473102, + "exclude_ties": 1107.1436577922286, + "hard_prompts": 1207.8870723506816, + "hard_prompts_english": 1244.1223840893358, + "industry_business_and_management_and_financial_operations": 1181.0129312812069, + "industry_entertainment_and_sports_and_media": 1224.9028487681405, + "industry_legal_and_government": 1228.3567319214683, + "industry_life_and_physical_and_social_science": 1239.6100713178678, + "industry_mathematical": 1195.075070334155, + "industry_medicine_and_healthcare": 1180.8245195808054, + "industry_software_and_it_services": 1214.7601428040211, + "industry_writing_and_literature_and_language": 1203.592710028142, + "instruction_following": 1186.3159109420387, + "longer_query": 1194.0811827622797, + "math": 1207.258176775466, + "multi_turn": 1220.9351075672016, + "russian": 1186.2505560496472 + }, + "phi-4": { + "overall": 1216.745343879491, + "chinese": 1211.2910293760165, + "coding": 1231.8328739209298, + "creative_writing": 1182.1991035071605, + "english": 1230.839538923798, + "exclude_ties": 1100.5621024232178, + "expert": 1202.2980443015736, + "french": 1225.5090576109, + "german": 1222.7543784408172, + "hard_prompts": 1219.843824853975, + "hard_prompts_english": 1231.1781937996643, + "industry_business_and_management_and_financial_operations": 1198.6880497377902, + "industry_entertainment_and_sports_and_media": 1162.3047019654425, + "industry_legal_and_government": 1241.0444366419733, + "industry_life_and_physical_and_social_science": 1206.7782940550142, + "industry_mathematical": 1253.759687095002, + "industry_medicine_and_healthcare": 1200.6780143387393, + "industry_software_and_it_services": 1228.74169586871, + "industry_writing_and_literature_and_language": 1197.7712016966234, + "instruction_following": 1201.547642075348, + "japanese": 1159.1705449450542, + "korean": 1151.0053607659975, + "longer_query": 1217.375582900172, + "math": 1245.6127178237361, + "multi_turn": 1205.7402354139592, + "russian": 1207.6313402031376, + "spanish": 1230.7183678336621 + }, + "amazon-nova-micro-v1.0": { + "overall": 1208.4842464265766, + "chinese": 1208.7403248754622, + "coding": 1218.2366802481943, + "creative_writing": 1172.316076002584, + "english": 1223.7015902692456, + "exclude_ties": 1084.7449512897165, + "expert": 1182.964631030611, + "french": 1238.7507592194002, + "german": 1192.2417170350066, + "hard_prompts": 1191.40781474518, + "hard_prompts_english": 1204.691908697937, + "industry_business_and_management_and_financial_operations": 1181.8222294235209, + "industry_entertainment_and_sports_and_media": 1163.2883750204019, + "industry_legal_and_government": 1221.1638301350497, + "industry_life_and_physical_and_social_science": 1210.6261553168717, + "industry_mathematical": 1212.0275117427664, + "industry_medicine_and_healthcare": 1206.423869770293, + "industry_software_and_it_services": 1215.9353058060142, + "industry_writing_and_literature_and_language": 1192.163833041444, + "instruction_following": 1174.4576047115204, + "japanese": 1154.8757460653694, + "korean": 1150.059021892073, + "longer_query": 1205.6252708821378, + "math": 1205.9416990695927, + "multi_turn": 1177.7901510899314, + "russian": 1184.100035304627, + "spanish": 1222.182183705143 + }, + "gemma-2-9b-it": { + "overall": 1207.464666543743, + "chinese": 1184.0382549220467, + "coding": 1174.0056190341231, + "creative_writing": 1206.130805508947, + "english": 1219.7653090348092, + "exclude_ties": 1090.4922659076549, + "expert": 1146.671600980408, + "french": 1190.8603570095895, + "german": 1186.0113986418162, + "hard_prompts": 1171.7426666370898, + "hard_prompts_english": 1179.3987803331465, + "industry_business_and_management_and_financial_operations": 1189.7855344569416, + "industry_entertainment_and_sports_and_media": 1185.2819887845542, + "industry_legal_and_government": 1215.177512441468, + "industry_life_and_physical_and_social_science": 1203.9147168499399, + "industry_mathematical": 1193.0982304159181, + "industry_medicine_and_healthcare": 1164.3516912094806, + "industry_software_and_it_services": 1197.5479228252425, + "industry_writing_and_literature_and_language": 1218.884375920663, + "instruction_following": 1178.4779592676755, + "japanese": 1144.664957313506, + "korean": 1136.4430587737993, + "longer_query": 1196.7578075726271, + "math": 1183.0213733251003, + "multi_turn": 1193.2885272965118, + "russian": 1199.2752931165626, + "spanish": 1198.9007397349833 + }, + "gpt-4-0314": { + "overall": 1206.0875195760966, + "chinese": 1182.9739244953244, + "coding": 1209.1966660880676, + "creative_writing": 1190.7644711590765, + "english": 1220.0971343388633, + "exclude_ties": 1086.2219009076537, + "expert": 1148.2681730726022, + "french": 1220.0241657023305, + "german": 1198.1078509653294, + "hard_prompts": 1200.479469824157, + "hard_prompts_english": 1210.4351222481894, + "industry_business_and_management_and_financial_operations": 1144.9908150695442, + "industry_entertainment_and_sports_and_media": 1184.7143250913778, + "industry_legal_and_government": 1191.5972250724276, + "industry_life_and_physical_and_social_science": 1177.2536177395339, + "industry_mathematical": 1227.5433363113207, + "industry_medicine_and_healthcare": 1144.070322109554, + "industry_software_and_it_services": 1195.6958198992554, + "industry_writing_and_literature_and_language": 1218.6056948707392, + "instruction_following": 1201.3199660997307, + "japanese": 1136.8890925011858, + "korean": 1086.8084492055432, + "longer_query": 1190.3670724875062, + "math": 1229.9220233221272, + "multi_turn": 1205.8128993466544, + "russian": 1194.2172664593922, + "spanish": 1195.4433590179874 + }, + "command-r-plus": { + "overall": 1203.910216399082, + "chinese": 1190.0195504223693, + "coding": 1172.3210924361288, + "creative_writing": 1201.4910013953631, + "english": 1213.206193478, + "exclude_ties": 1091.247025175588, + "expert": 1152.8346309021854, + "french": 1209.25823835878, + "german": 1195.335408062121, + "hard_prompts": 1166.8732093674755, + "hard_prompts_english": 1173.0448317726646, + "industry_business_and_management_and_financial_operations": 1190.3637782769802, + "industry_entertainment_and_sports_and_media": 1189.952713626874, + "industry_legal_and_government": 1244.9884515342083, + "industry_life_and_physical_and_social_science": 1216.639110377094, + "industry_mathematical": 1165.07243681389, + "industry_medicine_and_healthcare": 1202.2168065824105, + "industry_software_and_it_services": 1186.7803169077833, + "industry_writing_and_literature_and_language": 1218.5992165198982, + "instruction_following": 1179.9558904505311, + "japanese": 1155.0458663164159, + "korean": 1137.6229241389792, + "longer_query": 1201.6919069188461, + "math": 1164.2612801433404, + "multi_turn": 1192.0592936992812, + "russian": 1204.74969219422, + "spanish": 1187.8850804829901 + }, + "qwen2-72b-instruct": { + "overall": 1203.3519059205507, + "chinese": 1238.955254875461, + "coding": 1196.339845102204, + "creative_writing": 1181.4296647696137, + "english": 1222.1307069184352, + "exclude_ties": 1087.6800099584916, + "expert": 1170.6202362455956, + "french": 1171.0918584286628, + "german": 1150.6528564681505, + "hard_prompts": 1190.973106477503, + "hard_prompts_english": 1207.5402781036112, + "industry_business_and_management_and_financial_operations": 1146.2539006408513, + "industry_entertainment_and_sports_and_media": 1165.9867142872517, + "industry_legal_and_government": 1193.195594003847, + "industry_life_and_physical_and_social_science": 1195.6233769078872, + "industry_mathematical": 1230.571487798842, + "industry_medicine_and_healthcare": 1174.5250152297692, + "industry_software_and_it_services": 1193.9421464323539, + "industry_writing_and_literature_and_language": 1201.5668529571508, + "instruction_following": 1181.4426373877886, + "japanese": 1111.3281428085525, + "korean": 1082.4355874636142, + "longer_query": 1192.4445946591727, + "math": 1234.6529804246106, + "multi_turn": 1195.3920272903606, + "russian": 1167.6228880490225, + "spanish": 1167.6625907785224 + }, + "hunyuan-standard-256k": { + "overall": 1201.9990040885573, + "chinese": 1240.5753401607847, + "coding": 1231.230956062375, + "creative_writing": 1165.520711506692, + "english": 1215.0068223570688, + "exclude_ties": 1079.7635424858481, + "hard_prompts": 1218.533629401457, + "hard_prompts_english": 1229.9035726299667, + "industry_business_and_management_and_financial_operations": 1177.609576507145, + "industry_entertainment_and_sports_and_media": 1153.2142894449435, + "industry_life_and_physical_and_social_science": 1201.0139600323866, + "industry_mathematical": 1243.1738240254383, + "industry_software_and_it_services": 1223.295262253122, + "industry_writing_and_literature_and_language": 1173.2395139975883, + "instruction_following": 1202.1006585563339, + "longer_query": 1224.6388808369366, + "math": 1234.5069113953766, + "multi_turn": 1174.543427824922, + "russian": 1157.500254104905 + }, + "claude-3-haiku-20240307": { + "overall": 1194.7575707782157, + "chinese": 1153.76101767434, + "coding": 1199.493532389283, + "creative_writing": 1157.8584214631096, + "english": 1206.1445682499866, + "exclude_ties": 1075.5844255982838, + "expert": 1147.2620801554235, + "french": 1195.4525978008176, + "german": 1173.5139830458334, + "hard_prompts": 1174.331051947714, + "hard_prompts_english": 1183.146531476807, + "industry_business_and_management_and_financial_operations": 1157.4627600032313, + "industry_entertainment_and_sports_and_media": 1152.2543884029542, + "industry_legal_and_government": 1191.30983079951, + "industry_life_and_physical_and_social_science": 1178.7690082569939, + "industry_mathematical": 1187.2620756538504, + "industry_medicine_and_healthcare": 1164.0451631197216, + "industry_software_and_it_services": 1194.945899185239, + "industry_writing_and_literature_and_language": 1193.8029369194853, + "instruction_following": 1173.6234067289872, + "japanese": 1102.7553243828559, + "korean": 1108.2698759540867, + "longer_query": 1190.4842731510803, + "math": 1187.9133629604746, + "multi_turn": 1189.817544468528, + "russian": 1203.031718347805, + "spanish": 1164.345394890418 + }, + "llama-3.1-tulu-3-8b": { + "overall": 1193.3982307631145, + "chinese": 1174.714148392091, + "coding": 1183.9498303477071, + "creative_writing": 1182.051840912842, + "english": 1214.958047875984, + "exclude_ties": 1060.4075025168843, + "hard_prompts": 1174.6439639339774, + "hard_prompts_english": 1179.6254507951578, + "industry_business_and_management_and_financial_operations": 1196.3693079261234, + "industry_entertainment_and_sports_and_media": 1161.2327399271107, + "industry_life_and_physical_and_social_science": 1193.7474656038603, + "industry_mathematical": 1192.5525250193175, + "industry_software_and_it_services": 1192.5053865501327, + "industry_writing_and_literature_and_language": 1177.491779778506, + "instruction_following": 1174.494799451399, + "longer_query": 1180.8659857169387, + "math": 1194.629438711453, + "multi_turn": 1153.5198272773466, + "russian": 1192.0179838921454 + }, + "deepseek-coder-v2": { + "overall": 1191.2702997379824, + "chinese": 1199.899637178747, + "coding": 1251.4839713348838, + "creative_writing": 1120.790918135169, + "english": 1194.5176751908616, + "exclude_ties": 1070.6534079283267, + "expert": 1180.809589693684, + "french": 1185.4608863956537, + "german": 1164.1768214963652, + "hard_prompts": 1207.1799926617005, + "hard_prompts_english": 1219.0749885762864, + "industry_business_and_management_and_financial_operations": 1155.038374659723, + "industry_entertainment_and_sports_and_media": 1122.5435701628594, + "industry_legal_and_government": 1163.1637895097394, + "industry_life_and_physical_and_social_science": 1163.0819735169523, + "industry_mathematical": 1232.6553347278941, + "industry_medicine_and_healthcare": 1132.1869241898899, + "industry_software_and_it_services": 1239.7673582839086, + "industry_writing_and_literature_and_language": 1170.1788025233236, + "instruction_following": 1180.6093433783167, + "japanese": 1126.7204156669175, + "korean": 1103.7730085057674, + "longer_query": 1219.1322536321402, + "math": 1241.1492805472494, + "multi_turn": 1176.4209386583984, + "russian": 1186.6330872855883, + "spanish": 1151.9624717673112 + }, + "ministral-8b-2410": { + "overall": 1190.9898726526967, + "chinese": 1192.4144501091807, + "coding": 1202.4042018012196, + "creative_writing": 1175.23975095592, + "english": 1212.5086037940591, + "exclude_ties": 1060.8289342098187, + "expert": 1169.2365607958764, + "hard_prompts": 1191.581360934498, + "hard_prompts_english": 1207.9903048363544, + "industry_business_and_management_and_financial_operations": 1172.585289057457, + "industry_entertainment_and_sports_and_media": 1140.421486171359, + "industry_legal_and_government": 1211.1778682440129, + "industry_life_and_physical_and_social_science": 1189.4439581146094, + "industry_mathematical": 1183.9478762586355, + "industry_medicine_and_healthcare": 1179.0392418671786, + "industry_software_and_it_services": 1214.2205773177643, + "industry_writing_and_literature_and_language": 1180.4647961595165, + "instruction_following": 1160.8799247502384, + "longer_query": 1212.0536916080268, + "math": 1188.0693410419171, + "multi_turn": 1166.2255249878044, + "russian": 1194.0288415499822 + }, + "command-r-08-2024": { + "overall": 1187.4383323178085, + "chinese": 1181.2746059049803, + "coding": 1169.9475652173649, + "creative_writing": 1170.4372957884414, + "english": 1197.8092696055774, + "exclude_ties": 1056.742924568918, + "expert": 1137.072274520713, + "german": 1176.0718779169315, + "hard_prompts": 1164.5838150919758, + "hard_prompts_english": 1172.5330875496747, + "industry_business_and_management_and_financial_operations": 1157.0036917788523, + "industry_entertainment_and_sports_and_media": 1162.3475383013538, + "industry_legal_and_government": 1229.6852432618982, + "industry_life_and_physical_and_social_science": 1218.6760425943203, + "industry_mathematical": 1165.427889576435, + "industry_medicine_and_healthcare": 1207.2183328299889, + "industry_software_and_it_services": 1174.5824128713134, + "industry_writing_and_literature_and_language": 1192.9616322371467, + "instruction_following": 1167.3543669331889, + "japanese": 1143.5692366979881, + "korean": 1162.2556154412941, + "longer_query": 1197.7487780955894, + "math": 1154.6949302721846, + "multi_turn": 1162.9252209480705, + "russian": 1172.733475806188 + }, + "jamba-1.5-mini": { + "overall": 1186.7556842825074, + "chinese": 1134.64453291743, + "coding": 1178.8008441400368, + "creative_writing": 1164.8978722617378, + "english": 1224.029802679091, + "exclude_ties": 1057.3021338635313, + "expert": 1135.0290988554025, + "german": 1161.8229383675753, + "hard_prompts": 1167.1664204222711, + "hard_prompts_english": 1183.5655841182484, + "industry_business_and_management_and_financial_operations": 1157.3300422039108, + "industry_entertainment_and_sports_and_media": 1158.495546631269, + "industry_legal_and_government": 1223.7003236099627, + "industry_life_and_physical_and_social_science": 1190.6313817571402, + "industry_mathematical": 1160.9159127697185, + "industry_medicine_and_healthcare": 1179.857409737974, + "industry_software_and_it_services": 1189.7950585166186, + "industry_writing_and_literature_and_language": 1165.2783316068956, + "instruction_following": 1148.3047545083418, + "japanese": 1019.5193841906383, + "longer_query": 1156.7859119150721, + "math": 1160.0779188661024, + "multi_turn": 1156.7870850464597, + "russian": 1155.148001899419 + }, + "llama-3.1-8b-instruct": { + "overall": 1186.6599419952554, + "chinese": 1149.8238686968934, + "coding": 1195.3150042071118, + "creative_writing": 1154.8366484502226, + "english": 1217.5921483909478, + "exclude_ties": 1055.449670595683, + "expert": 1143.4273457302681, + "french": 1178.1275813572174, + "german": 1144.1360355461466, + "hard_prompts": 1175.0734218581356, + "hard_prompts_english": 1197.3712258495268, + "industry_business_and_management_and_financial_operations": 1165.8424047644608, + "industry_entertainment_and_sports_and_media": 1144.9988342380377, + "industry_legal_and_government": 1206.3599741393896, + "industry_life_and_physical_and_social_science": 1193.678001241338, + "industry_mathematical": 1180.0941269357065, + "industry_medicine_and_healthcare": 1180.5061191291904, + "industry_software_and_it_services": 1195.9461308315383, + "industry_writing_and_literature_and_language": 1161.9596815984055, + "instruction_following": 1159.2496087088389, + "japanese": 1061.3171683200524, + "korean": 1052.5042001720221, + "longer_query": 1181.6963563473164, + "math": 1179.0727648700613, + "multi_turn": 1171.8824704212577, + "russian": 1157.1873989668168, + "spanish": 1167.9630391911642 + }, + "gpt-4-0613": { + "overall": 1186.085395474718, + "chinese": 1135.1973941386336, + "coding": 1187.908534021269, + "creative_writing": 1191.97302150114, + "english": 1205.3344354346227, + "exclude_ties": 1058.7881053578835, + "expert": 1127.341686985661, + "french": 1170.7952702880175, + "german": 1161.4304931000004, + "hard_prompts": 1174.742495749355, + "hard_prompts_english": 1189.9083344774156, + "industry_business_and_management_and_financial_operations": 1121.6158716699128, + "industry_entertainment_and_sports_and_media": 1180.3349298330147, + "industry_legal_and_government": 1154.6358606225408, + "industry_life_and_physical_and_social_science": 1153.7595214803578, + "industry_mathematical": 1200.9620878837477, + "industry_medicine_and_healthcare": 1122.1924662778679, + "industry_software_and_it_services": 1169.68944235476, + "industry_writing_and_literature_and_language": 1213.8678363163278, + "instruction_following": 1188.8628574444078, + "japanese": 1114.5061506091383, + "korean": 1059.562779795483, + "longer_query": 1187.9604794860568, + "math": 1216.8227253858793, + "multi_turn": 1184.5798791981545, + "russian": 1171.9532384191211, + "spanish": 1166.7094475337447 + }, + "c4ai-aya-expanse-8b": { + "overall": 1185.1982637155252, + "chinese": 1180.047024314373, + "coding": 1161.0326303782429, + "creative_writing": 1168.5933515262514, + "english": 1190.8678397097144, + "exclude_ties": 1053.0768900819298, + "expert": 1152.0537601709862, + "german": 1187.5643176918666, + "hard_prompts": 1156.3130107489144, + "hard_prompts_english": 1155.6398357454345, + "industry_business_and_management_and_financial_operations": 1176.1888672261384, + "industry_entertainment_and_sports_and_media": 1150.2427769952706, + "industry_legal_and_government": 1224.9244832311338, + "industry_life_and_physical_and_social_science": 1204.063212166173, + "industry_mathematical": 1176.3451856901436, + "industry_medicine_and_healthcare": 1201.950142683249, + "industry_software_and_it_services": 1177.7839008916271, + "industry_writing_and_literature_and_language": 1185.3867682494704, + "instruction_following": 1155.1467108266252, + "japanese": 1120.6699523228372, + "longer_query": 1191.5469906152603, + "math": 1167.9674295851828, + "multi_turn": 1159.5222908186147, + "russian": 1196.1688243473345 + }, + "mistral-large-2402": { + "overall": 1176.5561410249732, + "chinese": 1119.3463852640239, + "coding": 1183.4522584002934, + "creative_writing": 1161.6086285809745, + "english": 1199.950217607728, + "exclude_ties": 1047.18647102525, + "expert": 1122.5349648552065, + "french": 1210.4325620942232, + "german": 1177.7724109482645, + "hard_prompts": 1167.4895178499085, + "hard_prompts_english": 1179.957770208308, + "industry_business_and_management_and_financial_operations": 1130.0816727979777, + "industry_entertainment_and_sports_and_media": 1151.0926944260764, + "industry_legal_and_government": 1180.9168935917526, + "industry_life_and_physical_and_social_science": 1154.2187564678984, + "industry_mathematical": 1189.771791096956, + "industry_medicine_and_healthcare": 1132.121584065867, + "industry_software_and_it_services": 1171.6251106406787, + "industry_writing_and_literature_and_language": 1179.6492488062681, + "instruction_following": 1169.4944453426754, + "japanese": 1008.5676143573917, + "korean": 1016.8530132718247, + "longer_query": 1173.2534003364183, + "math": 1199.681265086468, + "multi_turn": 1171.8957339584658, + "russian": 1178.3592665129847, + "spanish": 1193.4593159850365 + }, + "qwen1.5-110b-chat": { + "overall": 1174.6497260264787, + "chinese": 1204.8271106691761, + "coding": 1184.4238832206952, + "creative_writing": 1148.4372459559386, + "english": 1198.046391286768, + "exclude_ties": 1044.8577217742923, + "expert": 1143.8127834079614, + "french": 1151.908491805736, + "german": 1122.50834447584, + "hard_prompts": 1168.306547129292, + "hard_prompts_english": 1186.643380842386, + "industry_business_and_management_and_financial_operations": 1140.0742541625532, + "industry_entertainment_and_sports_and_media": 1138.8946529319526, + "industry_legal_and_government": 1188.300211512404, + "industry_life_and_physical_and_social_science": 1166.3629867093875, + "industry_mathematical": 1190.0226169856242, + "industry_medicine_and_healthcare": 1144.8208581773902, + "industry_software_and_it_services": 1175.3499356267635, + "industry_writing_and_literature_and_language": 1170.3959575039294, + "instruction_following": 1158.1301192274077, + "japanese": 1074.51063512773, + "korean": 1043.3934687587728, + "longer_query": 1157.1605721306125, + "math": 1185.2608671153719, + "multi_turn": 1159.6501702824903, + "russian": 1117.2126457842596, + "spanish": 1140.703637334983 + }, + "yi-1.5-34b-chat": { + "overall": 1173.0370466804861, + "chinese": 1212.0630863386716, + "coding": 1169.7914263710497, + "creative_writing": 1135.6797499677507, + "english": 1214.80714921023, + "exclude_ties": 1043.1800705938153, + "expert": 1143.4356685285106, + "french": 1157.1128308679663, + "german": 1110.7088948549958, + "hard_prompts": 1160.1940209840236, + "hard_prompts_english": 1181.356038453742, + "industry_business_and_management_and_financial_operations": 1134.6267702127475, + "industry_entertainment_and_sports_and_media": 1121.03791083457, + "industry_legal_and_government": 1196.4112734283135, + "industry_life_and_physical_and_social_science": 1173.5295929505216, + "industry_mathematical": 1185.3006487985672, + "industry_medicine_and_healthcare": 1158.2826042718011, + "industry_software_and_it_services": 1172.9350442695018, + "industry_writing_and_literature_and_language": 1144.0173199437722, + "instruction_following": 1139.505296782584, + "japanese": 1021.7432144036252, + "korean": 1005.0131898710167, + "longer_query": 1143.5944524275064, + "math": 1181.8605738394178, + "multi_turn": 1152.9590931897844, + "russian": 1089.82091641018, + "spanish": 1119.4584074449954 + }, + "reka-flash-21b-20240226-online": { + "overall": 1170.5114083544643, + "chinese": 1142.2513977778767, + "coding": 1165.0131856615978, + "creative_writing": 1131.0734377262459, + "english": 1180.802381291418, + "exclude_ties": 1040.7068139495088, + "expert": 1115.8528301729039, + "french": 1163.4263984118495, + "german": 1150.7440888511876, + "hard_prompts": 1137.270978956061, + "hard_prompts_english": 1148.5641279901472, + "industry_business_and_management_and_financial_operations": 1135.295814881332, + "industry_entertainment_and_sports_and_media": 1134.225166347242, + "industry_legal_and_government": 1194.3401274047683, + "industry_life_and_physical_and_social_science": 1168.5958662109235, + "industry_mathematical": 1165.6984184584057, + "industry_medicine_and_healthcare": 1141.6633907953908, + "industry_software_and_it_services": 1165.7473078598791, + "industry_writing_and_literature_and_language": 1160.1937284679414, + "instruction_following": 1129.7254638496781, + "japanese": 1075.4109961281126, + "korean": 1103.7797218183289, + "longer_query": 1139.4090423881144, + "math": 1167.8560280706845, + "multi_turn": 1150.9398969328863, + "russian": 1162.2224429501093, + "spanish": 1147.8023503327654 + }, + "qwen1.5-72b-chat": { + "overall": 1166.2309127905257, + "chinese": 1185.2139444403301, + "coding": 1165.9437379569126, + "creative_writing": 1137.596200283484, + "english": 1186.5775044564548, + "exclude_ties": 1032.7196516656813, + "expert": 1135.4369260879894, + "french": 1159.6019946670149, + "german": 1083.5695185307031, + "hard_prompts": 1148.2552872975402, + "hard_prompts_english": 1164.0040608749325, + "industry_business_and_management_and_financial_operations": 1116.8594985567747, + "industry_entertainment_and_sports_and_media": 1125.795044845788, + "industry_legal_and_government": 1173.504296267449, + "industry_life_and_physical_and_social_science": 1158.1666653628017, + "industry_mathematical": 1171.7733367691621, + "industry_medicine_and_healthcare": 1135.2566533707063, + "industry_software_and_it_services": 1161.8089906290863, + "industry_writing_and_literature_and_language": 1159.0279940154937, + "instruction_following": 1141.6455972520903, + "japanese": 1061.658459594611, + "korean": 1049.2423033238124, + "longer_query": 1157.103723547595, + "math": 1163.5546319888686, + "multi_turn": 1159.8165202761288, + "russian": 1102.9581104722176, + "spanish": 1109.0058968418473 + }, + "llama-3-8b-instruct": { + "overall": 1165.6826927481038, + "chinese": 1075.0994080605649, + "coding": 1152.1130135261283, + "creative_writing": 1150.3404115360586, + "english": 1213.2837522131395, + "exclude_ties": 1030.4915561292903, + "expert": 1112.6898863982533, + "french": 1159.4987496293697, + "german": 1104.2896897981086, + "hard_prompts": 1133.6355048690798, + "hard_prompts_english": 1165.4215420839284, + "industry_business_and_management_and_financial_operations": 1127.0138054543718, + "industry_entertainment_and_sports_and_media": 1132.3794871435632, + "industry_legal_and_government": 1167.7881701518459, + "industry_life_and_physical_and_social_science": 1170.4706944746404, + "industry_mathematical": 1153.2232909215531, + "industry_medicine_and_healthcare": 1146.9376193123985, + "industry_software_and_it_services": 1157.0536492125777, + "industry_writing_and_literature_and_language": 1147.69403752368, + "instruction_following": 1127.4750634096242, + "japanese": 967.8433775334838, + "korean": 1003.31401615971, + "longer_query": 1127.7955069818413, + "math": 1151.0644406881981, + "multi_turn": 1152.3542887002923, + "russian": 1108.1220395321138, + "spanish": 1171.6171534315909 + }, + "mistral-medium": { + "overall": 1165.2168265840091, + "chinese": 1108.124073872048, + "coding": 1162.5678445828853, + "creative_writing": 1158.2317975619585, + "english": 1190.335291552913, + "exclude_ties": 1030.925160034741, + "expert": 1124.7364057349143, + "french": 1180.4475147823298, + "german": 1154.5509946712539, + "hard_prompts": 1149.6420678164652, + "hard_prompts_english": 1159.5521172063527, + "industry_business_and_management_and_financial_operations": 1119.4029048016762, + "industry_entertainment_and_sports_and_media": 1138.1516664847643, + "industry_legal_and_government": 1169.9863603672811, + "industry_life_and_physical_and_social_science": 1158.7089767815773, + "industry_mathematical": 1177.8023487531514, + "industry_medicine_and_healthcare": 1131.1103507141936, + "industry_software_and_it_services": 1154.6934167261356, + "industry_writing_and_literature_and_language": 1171.9312695345025, + "instruction_following": 1152.0716582046211, + "japanese": 1012.8969243697554, + "korean": 1031.833848033863, + "longer_query": 1153.5649983944995, + "math": 1179.6646177753046, + "multi_turn": 1147.1539281577182, + "russian": 1166.5075094657016, + "spanish": 1142.1747844311476 + }, + "reka-flash-21b-20240226": { + "overall": 1165.0670209025395, + "chinese": 1134.379046175888, + "coding": 1154.172076737961, + "creative_writing": 1124.3744931411122, + "english": 1179.1274929285191, + "exclude_ties": 1032.4404936004103, + "expert": 1106.6391664793177, + "french": 1175.0691276788125, + "german": 1152.5225589705801, + "hard_prompts": 1136.9118201118517, + "hard_prompts_english": 1145.9391647488337, + "industry_business_and_management_and_financial_operations": 1126.8149430405815, + "industry_entertainment_and_sports_and_media": 1100.943362817633, + "industry_legal_and_government": 1180.6044230804425, + "industry_life_and_physical_and_social_science": 1160.0567307268548, + "industry_mathematical": 1158.5745085206609, + "industry_medicine_and_healthcare": 1146.1458265017418, + "industry_software_and_it_services": 1156.4192336393166, + "industry_writing_and_literature_and_language": 1152.1156787682273, + "instruction_following": 1121.7274502846549, + "japanese": 1049.283612283459, + "korean": 1068.0927374742514, + "longer_query": 1140.899786511168, + "math": 1155.396361581296, + "multi_turn": 1141.4152056392409, + "russian": 1158.7378826194963, + "spanish": 1130.7506550791184 + }, + "command-r": { + "overall": 1163.2133120644057, + "chinese": 1145.4823632973576, + "coding": 1128.1942600343, + "creative_writing": 1154.8276588710803, + "english": 1174.2826242084209, + "exclude_ties": 1028.0581451339033, + "expert": 1106.9971371004056, + "french": 1163.1122713557515, + "german": 1127.4845015793167, + "hard_prompts": 1115.0781698053188, + "hard_prompts_english": 1124.3214112527953, + "industry_business_and_management_and_financial_operations": 1141.1551460915027, + "industry_entertainment_and_sports_and_media": 1142.6980916684206, + "industry_legal_and_government": 1206.735434110397, + "industry_life_and_physical_and_social_science": 1170.9966661900226, + "industry_mathematical": 1124.7389172135938, + "industry_medicine_and_healthcare": 1151.1767711106459, + "industry_software_and_it_services": 1143.5929080329092, + "industry_writing_and_literature_and_language": 1161.5716111999627, + "instruction_following": 1124.9443823559798, + "japanese": 1094.2974288976784, + "korean": 1091.4316928416877, + "longer_query": 1157.6842005337576, + "math": 1119.8555921348575, + "multi_turn": 1147.3743460785506, + "russian": 1156.4825315387345, + "spanish": 1149.5468961847712 + }, + "mixtral-8x22b-instruct-v0.1": { + "overall": 1162.0656681581058, + "chinese": 1115.4036992712533, + "coding": 1166.1153287502993, + "creative_writing": 1141.8915885231809, + "english": 1187.2005822601127, + "exclude_ties": 1024.2208043156243, + "expert": 1112.6081984087637, + "french": 1166.7331821148794, + "german": 1141.0910214868718, + "hard_prompts": 1150.1069911866673, + "hard_prompts_english": 1164.7361833883724, + "industry_business_and_management_and_financial_operations": 1118.5043030120773, + "industry_entertainment_and_sports_and_media": 1139.0229540208993, + "industry_legal_and_government": 1165.1380689003365, + "industry_life_and_physical_and_social_science": 1151.3892928693424, + "industry_mathematical": 1178.9352508454454, + "industry_medicine_and_healthcare": 1118.7317713832667, + "industry_software_and_it_services": 1155.116774085558, + "industry_writing_and_literature_and_language": 1160.938492383495, + "instruction_following": 1147.4245463927637, + "japanese": 1037.0567858237266, + "korean": 1056.3790034633823, + "longer_query": 1143.8545437387893, + "math": 1184.138716357128, + "multi_turn": 1130.0535641965569, + "russian": 1156.6041437463523, + "spanish": 1149.4577563967996 + }, + "qwq-32b-preview": { + "overall": 1161.8800682591163, + "chinese": 1216.9308007503507, + "coding": 1155.7362040439048, + "creative_writing": 1134.255770530875, + "english": 1184.3590195920387, + "exclude_ties": 1022.9356412487948, + "expert": 1133.9485710264876, + "hard_prompts": 1162.9494686982905, + "hard_prompts_english": 1172.7459400786015, + "industry_business_and_management_and_financial_operations": 1098.2247609186243, + "industry_entertainment_and_sports_and_media": 1114.909502597613, + "industry_legal_and_government": 1143.1352750566725, + "industry_life_and_physical_and_social_science": 1154.3391448224706, + "industry_mathematical": 1221.3585639587895, + "industry_software_and_it_services": 1155.1726601769037, + "industry_writing_and_literature_and_language": 1139.3160559846247, + "instruction_following": 1156.2145317128616, + "longer_query": 1166.89286874066, + "math": 1213.333065090664, + "multi_turn": 1140.5790725181819, + "russian": 1113.3939463394481 + }, + "internlm2_5-20b-chat": { + "overall": 1158.5235952228168, + "chinese": 1198.509546108844, + "coding": 1159.527168249704, + "creative_writing": 1123.2790571415492, + "english": 1202.9855025388943, + "exclude_ties": 1010.5673698472076, + "expert": 1150.0101389321658, + "german": 1062.5714568779317, + "hard_prompts": 1158.41584652443, + "hard_prompts_english": 1190.5517236249393, + "industry_business_and_management_and_financial_operations": 1115.0701749865643, + "industry_entertainment_and_sports_and_media": 1103.4477858581135, + "industry_legal_and_government": 1151.4816547659323, + "industry_life_and_physical_and_social_science": 1165.3256287190366, + "industry_mathematical": 1180.8665913916543, + "industry_medicine_and_healthcare": 1150.6079200278227, + "industry_software_and_it_services": 1164.6377404328157, + "industry_writing_and_literature_and_language": 1135.8458504970797, + "instruction_following": 1135.3797381264658, + "longer_query": 1149.328422750023, + "math": 1179.6101720815946, + "multi_turn": 1135.4429161335706, + "russian": 1105.8329669977613 + }, + "gemma-2-2b-it": { + "overall": 1155.8471997670497, + "chinese": 1131.2281731546873, + "coding": 1112.7793971760475, + "creative_writing": 1147.3051542979756, + "english": 1181.5097336474532, + "exclude_ties": 1004.7303713225748, + "expert": 1095.6023821102526, + "french": 1157.9326677879267, + "german": 1114.0555874916472, + "hard_prompts": 1112.887889914904, + "hard_prompts_english": 1128.9823161053107, + "industry_business_and_management_and_financial_operations": 1125.57146132343, + "industry_entertainment_and_sports_and_media": 1127.2686770012303, + "industry_legal_and_government": 1169.6074707441821, + "industry_life_and_physical_and_social_science": 1159.317667070379, + "industry_mathematical": 1145.4966714879893, + "industry_medicine_and_healthcare": 1142.6032713733812, + "industry_software_and_it_services": 1133.1130506987092, + "industry_writing_and_literature_and_language": 1157.0070895870294, + "instruction_following": 1117.9203852184598, + "japanese": 1083.5347470424215, + "korean": 1054.558030507404, + "longer_query": 1130.1017286917959, + "math": 1134.3814764389872, + "multi_turn": 1117.8599051163578, + "russian": 1117.0158812259908, + "spanish": 1138.5900913150788 + }, + "granite-3.1-8b-instruct": { + "overall": 1149.6170287061188, + "chinese": 1143.9688829183565, + "coding": 1186.8572031188944, + "creative_writing": 1129.4194822120023, + "english": 1189.3020160454428, + "exclude_ties": 992.712426521211, + "expert": 1141.0308829700034, + "hard_prompts": 1145.8107225011927, + "hard_prompts_english": 1165.3842930267906, + "industry_business_and_management_and_financial_operations": 1141.9125882827761, + "industry_entertainment_and_sports_and_media": 1111.5585150160132, + "industry_legal_and_government": 1170.5699957451588, + "industry_life_and_physical_and_social_science": 1136.6560312578192, + "industry_mathematical": 1180.3814153579742, + "industry_software_and_it_services": 1162.9196710754004, + "industry_writing_and_literature_and_language": 1138.4206314806656, + "instruction_following": 1131.0495402115807, + "longer_query": 1161.9756575131423, + "math": 1151.426244286996, + "multi_turn": 1108.0018290568266, + "russian": 1090.758821087407 + }, + "gemini-pro-dev-api": { + "overall": 1148.9265085403435, + "chinese": 1123.0931352815678, + "coding": 1104.5003348561415, + "creative_writing": 1130.98360046009, + "english": 1161.9016894009005, + "exclude_ties": 1008.4078306596635, + "expert": 1058.4731548248299, + "french": 1145.760996013565, + "german": 1124.3881463562375, + "hard_prompts": 1109.6893156065853, + "hard_prompts_english": 1118.6093877504, + "industry_business_and_management_and_financial_operations": 1130.594036294934, + "industry_entertainment_and_sports_and_media": 1100.0343679372866, + "industry_legal_and_government": 1166.3401448999446, + "industry_life_and_physical_and_social_science": 1144.2768581167848, + "industry_mathematical": 1123.6702971130676, + "industry_medicine_and_healthcare": 1133.6283400150442, + "industry_software_and_it_services": 1132.1112839052366, + "industry_writing_and_literature_and_language": 1143.8577017886569, + "instruction_following": 1114.4276364854086, + "japanese": 1022.9911693010497, + "longer_query": 1131.7672432491745, + "math": 1131.9048278891248, + "multi_turn": 1124.132444016379, + "russian": 1184.8624455922761, + "spanish": 1117.721474201568 + }, + "zephyr-orpo-141b-A35b-v0.1": { + "overall": 1143.770818913676, + "chinese": 1081.037352183409, + "coding": 1129.7301584714644, + "creative_writing": 1113.6838235278096, + "english": 1174.4743531590943, + "exclude_ties": 998.8107595862614, + "expert": 1061.234166492039, + "hard_prompts": 1116.113561002931, + "hard_prompts_english": 1134.8021188891462, + "industry_business_and_management_and_financial_operations": 1088.8747578230782, + "industry_entertainment_and_sports_and_media": 1128.5183771789302, + "industry_legal_and_government": 1135.477302044265, + "industry_life_and_physical_and_social_science": 1142.5546999689109, + "industry_mathematical": 1144.5656619717415, + "industry_medicine_and_healthcare": 1105.2509493762138, + "industry_software_and_it_services": 1134.6105317229508, + "industry_writing_and_literature_and_language": 1139.02775367279, + "instruction_following": 1118.0014526980524, + "longer_query": 1104.9645705629337, + "math": 1147.6636808672733, + "multi_turn": 1099.4726664803316, + "russian": 1114.7299304565431 + }, + "phi-3-medium-4k-instruct": { + "overall": 1137.6288650409356, + "chinese": 1107.1594767467413, + "coding": 1130.6394190372687, + "creative_writing": 1107.00289529006, + "english": 1169.3238911843061, + "exclude_ties": 978.9619471383016, + "expert": 1107.5707074390616, + "french": 1111.1512322489075, + "german": 1100.6281538201674, + "hard_prompts": 1126.9505789769742, + "hard_prompts_english": 1144.2046355503714, + "industry_business_and_management_and_financial_operations": 1098.625436095518, + "industry_entertainment_and_sports_and_media": 1081.5540439768524, + "industry_legal_and_government": 1130.8336457400237, + "industry_life_and_physical_and_social_science": 1138.314110957879, + "industry_mathematical": 1175.708606836471, + "industry_medicine_and_healthcare": 1117.6821244594028, + "industry_software_and_it_services": 1134.3646020632784, + "industry_writing_and_literature_and_language": 1126.2830608408808, + "instruction_following": 1113.7249964182279, + "japanese": 1042.8024657576511, + "korean": 953.7689078719043, + "longer_query": 1120.2182943240568, + "math": 1173.2572445593069, + "multi_turn": 1087.8734236610553, + "russian": 1143.8110086379775, + "spanish": 1093.8610249673975 + }, + "qwen1.5-32b-chat": { + "overall": 1137.280938320662, + "chinese": 1175.5596521090845, + "coding": 1155.153881093347, + "creative_writing": 1083.4540537693601, + "english": 1159.885187973348, + "exclude_ties": 989.2041719160413, + "expert": 1125.7521162177372, + "french": 1101.4481442329, + "german": 1058.1481212368376, + "hard_prompts": 1130.8569477736494, + "hard_prompts_english": 1148.337086850072, + "industry_business_and_management_and_financial_operations": 1094.1311083132418, + "industry_entertainment_and_sports_and_media": 1080.8865990959057, + "industry_legal_and_government": 1140.5594824448258, + "industry_life_and_physical_and_social_science": 1132.592437910056, + "industry_mathematical": 1162.4912912173295, + "industry_medicine_and_healthcare": 1109.823862857264, + "industry_software_and_it_services": 1147.5526100213322, + "industry_writing_and_literature_and_language": 1118.0598240638865, + "instruction_following": 1115.8572150310954, + "japanese": 1027.5703154745552, + "korean": 1007.3749748111516, + "longer_query": 1146.0911316080303, + "math": 1154.935844450442, + "multi_turn": 1139.6054775826353, + "russian": 1071.694047206556, + "spanish": 1087.7658125087532 + }, + "starling-lm-7b-beta": { + "overall": 1132.1213450936666, + "chinese": 1113.0074608471414, + "coding": 1143.1757895319251, + "creative_writing": 1096.1897335215538, + "english": 1161.9746483149588, + "exclude_ties": 986.1186670319614, + "expert": 1083.388930331715, + "french": 1116.893339699383, + "german": 1080.9993035021566, + "hard_prompts": 1118.8445252548572, + "hard_prompts_english": 1132.6976015678072, + "industry_business_and_management_and_financial_operations": 1103.7646439960524, + "industry_entertainment_and_sports_and_media": 1085.2763378936047, + "industry_legal_and_government": 1151.2286899568462, + "industry_life_and_physical_and_social_science": 1142.2700284001228, + "industry_mathematical": 1129.2252822676512, + "industry_medicine_and_healthcare": 1128.2250013228058, + "industry_software_and_it_services": 1142.5191389602323, + "industry_writing_and_literature_and_language": 1111.0619800839117, + "instruction_following": 1096.0543987385286, + "japanese": 1000.9702912859998, + "longer_query": 1104.2417429550733, + "math": 1124.2967537508116, + "multi_turn": 1111.0809999625599, + "russian": 1101.4305919481367 + }, + "mixtral-8x7b-instruct-v0.1": { + "overall": 1131.6366938222775, + "chinese": 1054.4190227071558, + "coding": 1126.89285019774, + "creative_writing": 1109.3126841465253, + "english": 1167.613093810626, + "exclude_ties": 978.8464348544642, + "expert": 1087.1310483155137, + "french": 1166.879659594482, + "german": 1113.8440841837266, + "hard_prompts": 1115.0727841905682, + "hard_prompts_english": 1136.662231296448, + "industry_business_and_management_and_financial_operations": 1083.490448916274, + "industry_entertainment_and_sports_and_media": 1105.5284947758173, + "industry_legal_and_government": 1132.1145707121668, + "industry_life_and_physical_and_social_science": 1117.5204186717765, + "industry_mathematical": 1142.8015436784383, + "industry_medicine_and_healthcare": 1092.4413608181526, + "industry_software_and_it_services": 1125.9380038675436, + "industry_writing_and_literature_and_language": 1122.7627067722874, + "instruction_following": 1109.6751379946902, + "japanese": 931.5970376096602, + "korean": 967.5854243391213, + "longer_query": 1102.9337483056281, + "math": 1146.9807503832558, + "multi_turn": 1114.4960996254022, + "russian": 1089.246472356544, + "spanish": 1109.726428945319 + }, + "gemini-pro": { + "overall": 1130.6163392835974, + "chinese": 1095.4515766317868, + "coding": 1108.4070993278733, + "creative_writing": 1104.3894224264989, + "english": 1153.7753614035455, + "exclude_ties": 978.0191700543078, + "hard_prompts": 1094.1982770478514, + "hard_prompts_english": 1112.411553778765, + "industry_business_and_management_and_financial_operations": 1082.9735423727277, + "industry_entertainment_and_sports_and_media": 1085.381656508629, + "industry_legal_and_government": 1107.891257029525, + "industry_life_and_physical_and_social_science": 1118.8561746034702, + "industry_mathematical": 1135.8855032912895, + "industry_medicine_and_healthcare": 1115.33964604437, + "industry_software_and_it_services": 1111.991008051241, + "industry_writing_and_literature_and_language": 1126.7110954662958, + "instruction_following": 1110.2741451760348, + "math": 1128.775203213111, + "multi_turn": 1139.3476401795403 + }, + "yi-34b-chat": { + "overall": 1128.5483464232634, + "chinese": 1174.6410791963565, + "coding": 1112.2738319388736, + "creative_writing": 1108.5733115792355, + "english": 1159.1528331896693, + "exclude_ties": 976.7164451580841, + "expert": 1060.7902571484199, + "french": 1082.0899447577908, + "german": 1041.9055683558447, + "hard_prompts": 1103.9326194256623, + "hard_prompts_english": 1121.852207711905, + "industry_business_and_management_and_financial_operations": 1088.5168015070499, + "industry_entertainment_and_sports_and_media": 1107.040746312115, + "industry_legal_and_government": 1137.7177071276135, + "industry_life_and_physical_and_social_science": 1134.2518911023776, + "industry_mathematical": 1115.735968400762, + "industry_medicine_and_healthcare": 1101.4195561600548, + "industry_software_and_it_services": 1122.8665444181854, + "industry_writing_and_literature_and_language": 1116.5038990405442, + "instruction_following": 1091.1755400088857, + "japanese": 993.4551179557272, + "korean": 959.1650223397257, + "longer_query": 1093.827327461079, + "math": 1113.488673649423, + "multi_turn": 1112.656568946993, + "russian": 1048.347542577103, + "spanish": 1068.859075443921 + }, + "qwen1.5-14b-chat": { + "overall": 1127.7717310008918, + "chinese": 1146.1081765617996, + "coding": 1138.2674197203523, + "creative_writing": 1091.0476153463692, + "english": 1148.019330866527, + "exclude_ties": 976.3085619363254, + "expert": 1093.478346624651, + "french": 1116.6635302028985, + "german": 1042.3803444213572, + "hard_prompts": 1113.6071532421129, + "hard_prompts_english": 1125.333198663471, + "industry_business_and_management_and_financial_operations": 1082.7439361943188, + "industry_entertainment_and_sports_and_media": 1068.7590002489499, + "industry_legal_and_government": 1119.273794967448, + "industry_life_and_physical_and_social_science": 1102.906729554465, + "industry_mathematical": 1135.5524001044282, + "industry_medicine_and_healthcare": 1103.7464033476779, + "industry_software_and_it_services": 1141.410078597613, + "industry_writing_and_literature_and_language": 1112.243669571078, + "instruction_following": 1101.984166968209, + "japanese": 1019.6692575658401, + "longer_query": 1112.8909700092509, + "math": 1125.251482144727, + "multi_turn": 1109.6685035010764, + "russian": 1044.7090159512038, + "spanish": 1084.0740810228508 + }, + "granite-3.1-2b-instruct": { + "overall": 1127.4989511849708, + "chinese": 1137.4438574502153, + "coding": 1149.7901215051609, + "creative_writing": 1116.5137472300075, + "english": 1172.6294225843885, + "exclude_ties": 946.4160079003408, + "expert": 1130.577877400153, + "hard_prompts": 1138.4560841295024, + "hard_prompts_english": 1158.2658205137486, + "industry_business_and_management_and_financial_operations": 1139.1243630059384, + "industry_entertainment_and_sports_and_media": 1105.0929156614031, + "industry_life_and_physical_and_social_science": 1108.4540134522795, + "industry_mathematical": 1186.0335239104122, + "industry_software_and_it_services": 1140.511636774425, + "industry_writing_and_literature_and_language": 1091.9657068318274, + "instruction_following": 1116.2870563879192, + "longer_query": 1155.1448087802887, + "math": 1159.1404151263414, + "multi_turn": 1099.312841461066, + "russian": 1062.2752347254311 + }, + "gpt-3.5-turbo-0125": { + "overall": 1124.9988450832168, + "chinese": 1074.366857026728, + "coding": 1136.9360729729422, + "creative_writing": 1092.917107248418, + "english": 1139.253964837572, + "exclude_ties": 967.7325966872149, + "expert": 1064.920255880439, + "french": 1118.5102690702092, + "german": 1090.2892986496804, + "hard_prompts": 1108.2760883058054, + "hard_prompts_english": 1123.6100536769527, + "industry_business_and_management_and_financial_operations": 1076.0423201651363, + "industry_entertainment_and_sports_and_media": 1088.9081076825905, + "industry_legal_and_government": 1118.0051123923154, + "industry_life_and_physical_and_social_science": 1087.3828947180207, + "industry_mathematical": 1126.7317547418918, + "industry_medicine_and_healthcare": 1073.1068334464762, + "industry_software_and_it_services": 1116.2799157242557, + "industry_writing_and_literature_and_language": 1138.9001699587247, + "instruction_following": 1118.9021790154263, + "japanese": 1043.7249755902205, + "korean": 1018.5074956734547, + "longer_query": 1120.9437743127373, + "math": 1141.5984736030427, + "multi_turn": 1116.7067320336168, + "russian": 1122.098360969987, + "spanish": 1119.2683982528679 + }, + "tulu-2-dpo-70b": { + "overall": 1121.112732568856, + "chinese": 1012.5287532544303, + "coding": 1116.6899892146196, + "creative_writing": 1105.504933542735, + "english": 1150.0082214110098, + "exclude_ties": 963.2270761427557, + "hard_prompts": 1104.7825466232534, + "hard_prompts_english": 1125.9797106174738, + "industry_business_and_management_and_financial_operations": 1077.2714247245087, + "industry_entertainment_and_sports_and_media": 1115.9519074161333, + "industry_legal_and_government": 1135.5858589170375, + "industry_life_and_physical_and_social_science": 1105.312186731859, + "industry_mathematical": 1098.8127338607965, + "industry_medicine_and_healthcare": 1058.7936220087431, + "industry_software_and_it_services": 1111.0879958065962, + "industry_writing_and_literature_and_language": 1123.2321656361805, + "instruction_following": 1110.6376139361755, + "longer_query": 1106.0258381576928, + "math": 1107.1062156972034, + "multi_turn": 1085.341840772172 + }, + "wizardlm-70b": { + "overall": 1119.5983481500375, + "chinese": 1051.4585046590687, + "coding": 1081.6657340664392, + "creative_writing": 1149.7190642269468, + "english": 1150.4291661289117, + "exclude_ties": 966.8264590254165, + "german": 1082.5099872326346, + "hard_prompts": 1079.5335374528609, + "hard_prompts_english": 1097.872549922231, + "industry_business_and_management_and_financial_operations": 1078.3506600097326, + "industry_entertainment_and_sports_and_media": 1112.6728594766982, + "industry_legal_and_government": 1129.2940033222867, + "industry_life_and_physical_and_social_science": 1109.7668193310099, + "industry_mathematical": 1107.2906979054046, + "industry_medicine_and_healthcare": 1093.5885599490202, + "industry_software_and_it_services": 1089.4344173539348, + "industry_writing_and_literature_and_language": 1144.4683004282197, + "instruction_following": 1093.348113516515, + "longer_query": 1097.1725675501493, + "math": 1115.848349306136, + "multi_turn": 1108.2237935113124, + "russian": 1154.1751341340887 + }, + "dbrx-instruct-preview": { + "overall": 1118.9467009176078, + "chinese": 1066.8638914046867, + "coding": 1132.2674636352447, + "creative_writing": 1104.7725576534863, + "english": 1155.1308390479599, + "exclude_ties": 959.2106274909816, + "expert": 1075.548844100024, + "french": 1096.739999486555, + "german": 1057.2213459586446, + "hard_prompts": 1113.3591394856983, + "hard_prompts_english": 1139.4085796540849, + "industry_business_and_management_and_financial_operations": 1071.9908635176084, + "industry_entertainment_and_sports_and_media": 1099.1599977191559, + "industry_legal_and_government": 1104.5656162801156, + "industry_life_and_physical_and_social_science": 1100.33156877453, + "industry_mathematical": 1130.923948110299, + "industry_medicine_and_healthcare": 1066.795494055191, + "industry_software_and_it_services": 1117.4920365426733, + "industry_writing_and_literature_and_language": 1121.389954533117, + "instruction_following": 1111.8734271889896, + "japanese": 989.9516973075322, + "korean": 992.5795792736058, + "longer_query": 1111.7024787265634, + "math": 1144.8863265918085, + "multi_turn": 1110.9037840661692, + "russian": 1076.8189627228367, + "spanish": 1062.7933475575103 + }, + "llama-2-70b-chat": { + "overall": 1115.3056283689225, + "chinese": 993.6844421395024, + "coding": 1079.2933947171143, + "creative_writing": 1075.4161284770116, + "english": 1156.0411253773987, + "exclude_ties": 955.6091061003037, + "expert": 1038.711993150865, + "french": 1090.742288944267, + "german": 1041.1297891415948, + "hard_prompts": 1073.4034296538816, + "hard_prompts_english": 1093.0571094891086, + "industry_business_and_management_and_financial_operations": 1082.6980653926448, + "industry_entertainment_and_sports_and_media": 1064.8135650516022, + "industry_legal_and_government": 1114.0689142987662, + "industry_life_and_physical_and_social_science": 1120.1825449723956, + "industry_mathematical": 1089.360176430317, + "industry_medicine_and_healthcare": 1092.1099352399976, + "industry_software_and_it_services": 1099.950908076864, + "industry_writing_and_literature_and_language": 1092.591211180052, + "instruction_following": 1071.1431670935904, + "japanese": 927.1534566895618, + "korean": 963.7621567876839, + "longer_query": 1062.1517281534025, + "math": 1091.089339228038, + "multi_turn": 1087.9323947877879, + "russian": 1081.7097506436692, + "spanish": 1141.2873755786366 + }, + "nous-hermes-2-mixtral-8x7b-dpo": { + "overall": 1112.1165054092385, + "coding": 1080.2143593486546, + "creative_writing": 1102.1989644714208, + "english": 1139.6092134818346, + "exclude_ties": 951.5006993681483, + "hard_prompts": 1053.8032159436991, + "hard_prompts_english": 1068.154564188042, + "industry_business_and_management_and_financial_operations": 1071.0834438142715, + "industry_entertainment_and_sports_and_media": 1067.0978459688913, + "industry_life_and_physical_and_social_science": 1104.1880569322489, + "industry_mathematical": 1049.8319762548026, + "industry_medicine_and_healthcare": 1055.11984509256, + "industry_software_and_it_services": 1095.7052562602084, + "industry_writing_and_literature_and_language": 1104.9427732327654, + "instruction_following": 1056.2615268898526, + "longer_query": 1045.8842297700423, + "math": 1060.4238083560303, + "multi_turn": 1080.9991581101983 + }, + "phi-3-small-8k-instruct": { + "overall": 1109.9009278114286, + "chinese": 1059.8089022413876, + "coding": 1101.9266714443065, + "creative_writing": 1083.3360270023104, + "english": 1150.604414947451, + "exclude_ties": 936.4870315647981, + "expert": 1066.5382029792595, + "french": 1135.4773341886303, + "german": 1079.5355750793206, + "hard_prompts": 1100.6866727746137, + "hard_prompts_english": 1127.0072503758743, + "industry_business_and_management_and_financial_operations": 1077.0270137354253, + "industry_entertainment_and_sports_and_media": 1055.8600812560467, + "industry_legal_and_government": 1102.6224884841845, + "industry_life_and_physical_and_social_science": 1107.4304476597304, + "industry_mathematical": 1152.886775002515, + "industry_medicine_and_healthcare": 1096.064477025437, + "industry_software_and_it_services": 1109.4246262908282, + "industry_writing_and_literature_and_language": 1093.61002907389, + "instruction_following": 1086.9217196510438, + "japanese": 966.0806454717283, + "korean": 893.5906847830361, + "longer_query": 1087.7528688732893, + "math": 1150.6073315220476, + "multi_turn": 1068.2982560940673, + "russian": 1109.352796674033, + "spanish": 1109.9267664987465 + }, + "llama-3.2-3b-instruct": { + "overall": 1109.6123660196954, + "chinese": 1016.3221817326748, + "coding": 1098.2266153343987, + "creative_writing": 1094.7057603868832, + "english": 1179.666277645839, + "exclude_ties": 933.6590496099645, + "expert": 1089.5699255709455, + "german": 1056.006891183764, + "hard_prompts": 1095.016237272217, + "hard_prompts_english": 1142.5831383975058, + "industry_business_and_management_and_financial_operations": 1073.5414563493193, + "industry_entertainment_and_sports_and_media": 1083.7816860405778, + "industry_legal_and_government": 1115.3168264019582, + "industry_life_and_physical_and_social_science": 1106.2434905880586, + "industry_mathematical": 1128.3620117499345, + "industry_medicine_and_healthcare": 1103.2003605726761, + "industry_software_and_it_services": 1104.2572395746652, + "industry_writing_and_literature_and_language": 1095.465758606638, + "instruction_following": 1089.1840994880886, + "longer_query": 1100.5615299053245, + "math": 1125.7516523767447, + "multi_turn": 1104.9882662190585, + "russian": 947.7724078639333 + }, + "starling-lm-7b-alpha": { + "overall": 1107.7385523145826, + "chinese": 1044.2647079593983, + "coding": 1098.019670316551, + "creative_writing": 1099.7131873615804, + "english": 1138.724875614976, + "exclude_ties": 945.3778510158811, + "expert": 1032.6972177994041, + "french": 1085.7032407971515, + "german": 1025.1522554315275, + "hard_prompts": 1075.7417496688854, + "hard_prompts_english": 1096.4452438868334, + "industry_business_and_management_and_financial_operations": 1072.5033749020536, + "industry_entertainment_and_sports_and_media": 1082.772724019333, + "industry_legal_and_government": 1122.2554835638425, + "industry_life_and_physical_and_social_science": 1110.1315042925678, + "industry_mathematical": 1082.0571438454986, + "industry_medicine_and_healthcare": 1110.3722082918252, + "industry_software_and_it_services": 1092.3652944396945, + "industry_writing_and_literature_and_language": 1098.2063908258842, + "instruction_following": 1070.2224632519337, + "longer_query": 1081.1968194086555, + "math": 1081.325800372922, + "multi_turn": 1084.9361427582041, + "russian": 1094.3600991741976 + }, + "openchat-3.5-0106": { + "overall": 1106.686082839829, + "chinese": 1082.7775198303339, + "coding": 1113.4189999703435, + "creative_writing": 1093.0001774032348, + "english": 1131.2201739058314, + "exclude_ties": 944.3734085563319, + "expert": 1051.8807416120462, + "french": 1092.9115714161412, + "german": 1059.725923101728, + "hard_prompts": 1085.328042822534, + "hard_prompts_english": 1096.6350758118092, + "industry_business_and_management_and_financial_operations": 1073.9892864376136, + "industry_entertainment_and_sports_and_media": 1074.6457739377736, + "industry_legal_and_government": 1105.3254666919115, + "industry_life_and_physical_and_social_science": 1079.054317858213, + "industry_mathematical": 1094.1925602648193, + "industry_medicine_and_healthcare": 1072.2017385665097, + "industry_software_and_it_services": 1105.7995168784544, + "industry_writing_and_literature_and_language": 1105.079819754253, + "instruction_following": 1076.7322044609443, + "longer_query": 1085.7635289856535, + "math": 1106.50158450807, + "multi_turn": 1090.7378202336045, + "russian": 1083.5717197963522 + }, + "vicuna-33b": { + "overall": 1105.3374888487729, + "chinese": 1034.9058619246462, + "coding": 1078.7573207181213, + "creative_writing": 1125.2093814073176, + "english": 1133.555964315036, + "exclude_ties": 945.1612132707996, + "expert": 1028.6597741074818, + "french": 1086.4328734658577, + "german": 1058.9877251822602, + "hard_prompts": 1067.2512913148475, + "hard_prompts_english": 1081.3569542422765, + "industry_business_and_management_and_financial_operations": 1054.280787234598, + "industry_entertainment_and_sports_and_media": 1103.2718604063132, + "industry_legal_and_government": 1098.876935077325, + "industry_life_and_physical_and_social_science": 1100.1265324290819, + "industry_mathematical": 1076.6000625979082, + "industry_medicine_and_healthcare": 1059.1454267970878, + "industry_software_and_it_services": 1086.831845175807, + "industry_writing_and_literature_and_language": 1118.695704703838, + "instruction_following": 1065.2621735773796, + "japanese": 958.9755761941201, + "longer_query": 1028.7458348708096, + "math": 1070.4974959142562, + "multi_turn": 1080.3591915003167, + "russian": 1081.5185635530156, + "spanish": 1078.684228063447 + }, + "deepseek-llm-67b-chat": { + "overall": 1105.106932998483, + "chinese": 1130.8536983630333, + "coding": 1096.0816891561526, + "creative_writing": 1067.9490582979915, + "english": 1127.1287421208067, + "exclude_ties": 942.8760510397876, + "hard_prompts": 1070.1911079659585, + "hard_prompts_english": 1087.0986204652356, + "industry_business_and_management_and_financial_operations": 1047.517269719864, + "industry_entertainment_and_sports_and_media": 1062.0511827772848, + "industry_legal_and_government": 1089.3237924004243, + "industry_life_and_physical_and_social_science": 1066.2195245974242, + "industry_mathematical": 1092.973936100328, + "industry_medicine_and_healthcare": 1060.5538287054715, + "industry_software_and_it_services": 1095.925153943328, + "industry_writing_and_literature_and_language": 1110.557008525529, + "instruction_following": 1079.3715683758244, + "longer_query": 1092.0955042070227, + "math": 1107.4988675388884, + "multi_turn": 1081.8472241553475 + }, + "snowflake-arctic-instruct": { + "overall": 1101.0530024577188, + "chinese": 1066.127016595471, + "coding": 1089.9255139627635, + "creative_writing": 1081.457225200233, + "english": 1121.6544349719916, + "exclude_ties": 927.6925621792445, + "expert": 1027.723308577089, + "french": 1132.474940783119, + "german": 1073.4768241731676, + "hard_prompts": 1071.8886861446936, + "hard_prompts_english": 1089.8223057684233, + "industry_business_and_management_and_financial_operations": 1031.3606056109938, + "industry_entertainment_and_sports_and_media": 1069.2031432968602, + "industry_legal_and_government": 1091.3489555415276, + "industry_life_and_physical_and_social_science": 1082.2121870643737, + "industry_mathematical": 1097.2541854852984, + "industry_medicine_and_healthcare": 1062.202361593351, + "industry_software_and_it_services": 1088.3994867224592, + "industry_writing_and_literature_and_language": 1100.7731027321424, + "instruction_following": 1066.3932809388161, + "japanese": 969.8611193735348, + "korean": 939.3260242568324, + "longer_query": 999.3865417035855, + "math": 1108.5206117216462, + "multi_turn": 1072.1064606573748, + "russian": 1102.2685528178208, + "spanish": 1086.9112017708508 + }, + "llama2-70b-steerlm-chat": { + "overall": 1098.1678284901598, + "coding": 1025.2782969309017, + "creative_writing": 1091.047238994194, + "english": 1119.8536805042113, + "exclude_ties": 930.8466788016968, + "hard_prompts": 1047.0681472783708, + "hard_prompts_english": 1058.5123413800438, + "industry_business_and_management_and_financial_operations": 1078.095823235454, + "industry_entertainment_and_sports_and_media": 1069.5182711481432, + "industry_life_and_physical_and_social_science": 1094.6841415017548, + "industry_mathematical": 1064.9314783027557, + "industry_software_and_it_services": 1056.72699542208, + "industry_writing_and_literature_and_language": 1094.2761232727785, + "instruction_following": 1060.7301751954674, + "longer_query": 997.889027530087, + "math": 1071.6806704612663, + "multi_turn": 1057.827482787997 + }, + "openchat-3.5": { + "overall": 1097.1094684536292, + "chinese": 1069.0482777121065, + "coding": 1074.7238168964832, + "creative_writing": 1126.0990526301148, + "english": 1117.7047845024408, + "exclude_ties": 929.7590841896142, + "expert": 1041.4015169982858, + "hard_prompts": 1065.8565551925608, + "hard_prompts_english": 1072.879314491555, + "industry_business_and_management_and_financial_operations": 1035.9818514364179, + "industry_entertainment_and_sports_and_media": 1096.7707681433321, + "industry_legal_and_government": 1113.0829565839313, + "industry_life_and_physical_and_social_science": 1067.7176680746554, + "industry_mathematical": 1069.121401168096, + "industry_medicine_and_healthcare": 1064.738940782262, + "industry_software_and_it_services": 1084.3646103067656, + "industry_writing_and_literature_and_language": 1116.7151769642915, + "instruction_following": 1070.4607170668892, + "longer_query": 1093.0326705755324, + "math": 1070.5568850276447, + "multi_turn": 1085.1984362270268, + "russian": 1105.520559526627 + }, + "granite-3.0-8b-instruct": { + "overall": 1096.515838435113, + "chinese": 1062.1441914277218, + "coding": 1113.002916105377, + "creative_writing": 1071.1580550906706, + "english": 1149.3688352781046, + "exclude_ties": 910.3885878404047, + "expert": 1086.859828689544, + "hard_prompts": 1092.0928266328829, + "hard_prompts_english": 1123.3123957138441, + "industry_business_and_management_and_financial_operations": 1040.4463213878778, + "industry_entertainment_and_sports_and_media": 1061.391148092258, + "industry_legal_and_government": 1084.0681413330963, + "industry_life_and_physical_and_social_science": 1099.4440052407824, + "industry_mathematical": 1152.165632133746, + "industry_medicine_and_healthcare": 1057.0177458465678, + "industry_software_and_it_services": 1101.020613317638, + "industry_writing_and_literature_and_language": 1078.2870985704192, + "instruction_following": 1088.3260674569726, + "longer_query": 1122.125120874256, + "math": 1143.2094452739602, + "multi_turn": 1062.6791755373004, + "russian": 1059.1417623489538 + }, + "gpt-3.5-turbo-1106": { + "overall": 1094.198279837674, + "chinese": 1011.1763865338409, + "coding": 1116.303808496236, + "creative_writing": 1035.5992110076613, + "english": 1113.1177698648212, + "exclude_ties": 923.4208425977488, + "expert": 1069.0551434263914, + "french": 1067.0387401017306, + "german": 1055.8242640148437, + "hard_prompts": 1098.945127703504, + "hard_prompts_english": 1115.9546538067839, + "industry_business_and_management_and_financial_operations": 1019.5217129582943, + "industry_entertainment_and_sports_and_media": 1032.9533033814296, + "industry_legal_and_government": 1056.3750925058998, + "industry_life_and_physical_and_social_science": 1051.599377614069, + "industry_mathematical": 1128.2114017216459, + "industry_medicine_and_healthcare": 1004.8922484222555, + "industry_software_and_it_services": 1092.2533480401653, + "industry_writing_and_literature_and_language": 1090.8690299698999, + "instruction_following": 1092.9373668045475, + "longer_query": 1069.7991454367127, + "math": 1141.0865250822308, + "multi_turn": 1077.7591514973046, + "russian": 1076.1549952562157, + "spanish": 1091.4801837268133 + }, + "gemma-1.1-7b-it": { + "overall": 1094.050008332462, + "chinese": 1059.6125534280072, + "coding": 1084.2705449212303, + "creative_writing": 1059.9603785681538, + "english": 1124.104907087397, + "exclude_ties": 918.4647960910461, + "expert": 1038.8377349049997, + "french": 1066.187693361992, + "german": 1053.5431602874592, + "hard_prompts": 1071.0324887573104, + "hard_prompts_english": 1092.4532341543138, + "industry_business_and_management_and_financial_operations": 1062.8123564717212, + "industry_entertainment_and_sports_and_media": 1050.004369325983, + "industry_legal_and_government": 1094.1879567643543, + "industry_life_and_physical_and_social_science": 1078.228985238907, + "industry_mathematical": 1097.7643417395575, + "industry_medicine_and_healthcare": 1071.2662224065748, + "industry_software_and_it_services": 1096.1652179713078, + "industry_writing_and_literature_and_language": 1073.012122210017, + "instruction_following": 1057.1462887953376, + "japanese": 971.0984624628472, + "korean": 987.6258395275293, + "longer_query": 1055.761276570562, + "math": 1106.6858341564703, + "multi_turn": 1039.6801893238753, + "russian": 1044.7174512112556, + "spanish": 1048.021436132006 + }, + "openhermes-2.5-mistral-7b": { + "overall": 1092.5764020453375, + "chinese": 1013.3642577582755, + "coding": 1064.8700685913977, + "creative_writing": 1097.7115399233605, + "english": 1125.6420686563147, + "exclude_ties": 923.8183993662931, + "hard_prompts": 1071.9614199850876, + "hard_prompts_english": 1088.3974952378746, + "industry_business_and_management_and_financial_operations": 1024.245861384951, + "industry_entertainment_and_sports_and_media": 1074.3395786566718, + "industry_legal_and_government": 1099.6902832281687, + "industry_life_and_physical_and_social_science": 1065.6739189902266, + "industry_mathematical": 1096.8961224150612, + "industry_medicine_and_healthcare": 1050.771967281887, + "industry_software_and_it_services": 1077.7778510389132, + "industry_writing_and_literature_and_language": 1095.8593456477922, + "instruction_following": 1070.5299533603538, + "longer_query": 1054.3272188459075, + "math": 1097.4540974562592, + "multi_turn": 1064.324091585232 + }, + "mistral-7b-instruct-v0.2": { + "overall": 1089.6422160882228, + "chinese": 1008.3684412017383, + "coding": 1082.224771402296, + "creative_writing": 1068.2115066974075, + "english": 1133.5456415492022, + "exclude_ties": 915.3885543280276, + "expert": 1035.3713976510003, + "french": 1037.6101856374105, + "german": 987.1842564964254, + "hard_prompts": 1067.5475639548986, + "hard_prompts_english": 1091.2804383458406, + "industry_business_and_management_and_financial_operations": 1038.0527530172083, + "industry_entertainment_and_sports_and_media": 1058.2199849465987, + "industry_legal_and_government": 1098.669066355482, + "industry_life_and_physical_and_social_science": 1078.7715398200073, + "industry_mathematical": 1085.3741547057245, + "industry_medicine_and_healthcare": 1066.8429962422213, + "industry_software_and_it_services": 1083.8569833187537, + "industry_writing_and_literature_and_language": 1072.9265960599455, + "instruction_following": 1060.2923787716754, + "japanese": 878.4854256023141, + "longer_query": 1060.3599166522267, + "math": 1085.2196043478334, + "multi_turn": 1062.3164836275496, + "russian": 1017.1257378852893, + "spanish": 1025.1106422917894 + }, + "llama-2-13b-chat": { + "overall": 1084.534287763448, + "chinese": 1000.063104617057, + "coding": 1062.836463993498, + "creative_writing": 1047.4028222465752, + "english": 1119.9407947164386, + "exclude_ties": 908.9036624337416, + "expert": 1029.418949699735, + "french": 1044.8258455398543, + "german": 1008.9091989960154, + "hard_prompts": 1051.3074343456608, + "hard_prompts_english": 1074.4446870986515, + "industry_business_and_management_and_financial_operations": 1055.5114993433897, + "industry_entertainment_and_sports_and_media": 1047.8294947751012, + "industry_legal_and_government": 1079.9218326632288, + "industry_life_and_physical_and_social_science": 1082.2271246675286, + "industry_mathematical": 1068.7670981433776, + "industry_medicine_and_healthcare": 1070.5465493005277, + "industry_software_and_it_services": 1073.684873798875, + "industry_writing_and_literature_and_language": 1061.3989064903085, + "instruction_following": 1045.620589230988, + "japanese": 894.3184563653315, + "korean": 953.0834578710701, + "longer_query": 1064.3024772625295, + "math": 1064.7404493137647, + "multi_turn": 1050.02054200459, + "russian": 1053.3519672784873, + "spanish": 1085.3009765592062 + }, + "qwen1.5-7b-chat": { + "overall": 1083.5370077471587, + "chinese": 1139.5873918621269, + "coding": 1107.8132152385767, + "creative_writing": 1035.0675433821523, + "english": 1105.3156705760061, + "exclude_ties": 907.8114402640263, + "expert": 1054.4084894530847, + "hard_prompts": 1065.7693312396839, + "hard_prompts_english": 1076.9261867456444, + "industry_business_and_management_and_financial_operations": 1063.5386773284895, + "industry_entertainment_and_sports_and_media": 1030.2021506965984, + "industry_legal_and_government": 1103.1352631424304, + "industry_life_and_physical_and_social_science": 1076.6277937222667, + "industry_mathematical": 1107.323950872456, + "industry_medicine_and_healthcare": 1054.9471434814775, + "industry_software_and_it_services": 1091.1470953659327, + "industry_writing_and_literature_and_language": 1064.2657617386553, + "instruction_following": 1058.555841003934, + "longer_query": 1090.2824544680307, + "math": 1079.5726490898387, + "multi_turn": 1061.4362045552805, + "russian": 1004.6383505197655 + }, + "solar-10.7b-instruct-v1.0": { + "overall": 1083.1643178661618, + "coding": 1067.299984898582, + "creative_writing": 1089.8930853787665, + "english": 1117.3063387472548, + "exclude_ties": 908.3493577737042, + "hard_prompts": 1062.425532707174, + "hard_prompts_english": 1076.9384454594974, + "industry_business_and_management_and_financial_operations": 1019.8166156787578, + "industry_entertainment_and_sports_and_media": 1058.7988473104638, + "industry_legal_and_government": 1082.6909088946131, + "industry_life_and_physical_and_social_science": 1072.514043996448, + "industry_mathematical": 1056.4203150128178, + "industry_medicine_and_healthcare": 1046.1329651167898, + "industry_software_and_it_services": 1068.4984496557854, + "industry_writing_and_literature_and_language": 1087.7338388400572, + "instruction_following": 1046.7755575884526, + "math": 1063.733761426013, + "multi_turn": 1045.0457619597842 + }, + "dolphin-2.2.1-mistral-7b": { + "overall": 1081.0498765777618, + "creative_writing": 1086.2689737769047, + "english": 1112.5261177108498, + "exclude_ties": 906.5200376476159, + "hard_prompts": 1054.9480930521718, + "hard_prompts_english": 1069.8412438577257, + "industry_entertainment_and_sports_and_media": 1049.9533614061481, + "industry_life_and_physical_and_social_science": 1065.783382553548, + "industry_mathematical": 1055.2182443383658, + "industry_software_and_it_services": 1047.5102794714744, + "industry_writing_and_literature_and_language": 1085.8418612438686, + "instruction_following": 1053.652439583451, + "math": 1076.45407757543, + "multi_turn": 1020.5099222145072 + }, + "phi-3-mini-4k-instruct-june-2024": { + "overall": 1080.0332982040084, + "chinese": 1032.4582117311168, + "coding": 1093.5980719207741, + "creative_writing": 1045.1753276333634, + "english": 1128.8286519323124, + "exclude_ties": 874.5010374694403, + "expert": 1050.3343013538636, + "german": 1030.6956941350572, + "hard_prompts": 1087.8212587892608, + "hard_prompts_english": 1116.2603568619538, + "industry_business_and_management_and_financial_operations": 1022.7815403859355, + "industry_entertainment_and_sports_and_media": 1031.6993651135372, + "industry_legal_and_government": 1068.5121872025345, + "industry_life_and_physical_and_social_science": 1068.4387749176994, + "industry_mathematical": 1146.8847501398575, + "industry_medicine_and_healthcare": 1041.2106464129447, + "industry_software_and_it_services": 1089.0496349391394, + "industry_writing_and_literature_and_language": 1056.5697319089395, + "instruction_following": 1058.0323891285764, + "japanese": 954.8063284341008, + "korean": 879.5202503596092, + "longer_query": 1041.6256813432765, + "math": 1151.841566414538, + "multi_turn": 1048.990640187021, + "russian": 1017.9773028955915 + }, + "granite-3.0-2b-instruct": { + "overall": 1079.9192209986152, + "chinese": 1069.04344156468, + "coding": 1090.8548062940588, + "creative_writing": 1046.7296534061288, + "english": 1119.5229622982781, + "exclude_ties": 873.3309450404882, + "expert": 1063.8746157015066, + "hard_prompts": 1073.6630363176978, + "hard_prompts_english": 1088.7777236773409, + "industry_business_and_management_and_financial_operations": 1035.126628654837, + "industry_entertainment_and_sports_and_media": 1038.0571808766804, + "industry_legal_and_government": 1083.9616692195893, + "industry_life_and_physical_and_social_science": 1071.9565728095508, + "industry_mathematical": 1130.4115036710684, + "industry_medicine_and_healthcare": 1002.3118308648476, + "industry_software_and_it_services": 1079.9706565031724, + "industry_writing_and_literature_and_language": 1050.4443502054103, + "instruction_following": 1055.938898235112, + "longer_query": 1070.459161278874, + "math": 1116.75098581798, + "multi_turn": 1052.7860455220637, + "russian": 1044.071510299044 + }, + "wizardlm-13b": { + "overall": 1076.6144968263775, + "chinese": 1021.7882062941786, + "coding": 1035.3904655488923, + "creative_writing": 1090.9838266397414, + "english": 1101.764013634055, + "exclude_ties": 904.5589236939608, + "hard_prompts": 1018.0371246993293, + "hard_prompts_english": 1035.5315683670553, + "industry_business_and_management_and_financial_operations": 1006.252563908569, + "industry_entertainment_and_sports_and_media": 1075.1977895994537, + "industry_legal_and_government": 1099.8572784203925, + "industry_life_and_physical_and_social_science": 1069.2437378352436, + "industry_mathematical": 1015.0455396441218, + "industry_medicine_and_healthcare": 1063.785180811927, + "industry_software_and_it_services": 1053.250864957543, + "industry_writing_and_literature_and_language": 1098.118254547608, + "instruction_following": 1048.7233617883728, + "longer_query": 1053.673514683569, + "math": 1016.9418982158849, + "multi_turn": 1046.9316166514232 + }, + "phi-3-mini-4k-instruct": { + "overall": 1073.3831666250737, + "chinese": 1020.0612094554074, + "coding": 1093.1266640542924, + "creative_writing": 1037.4603196792016, + "english": 1117.2884714106372, + "exclude_ties": 872.4904473803701, + "expert": 1044.7893668492436, + "french": 1076.7695206072526, + "german": 1043.75972775282, + "hard_prompts": 1072.563347328538, + "hard_prompts_english": 1099.4977039857242, + "industry_business_and_management_and_financial_operations": 1031.98552628478, + "industry_entertainment_and_sports_and_media": 1019.2878777111007, + "industry_legal_and_government": 1071.0719094811677, + "industry_life_and_physical_and_social_science": 1062.295097752994, + "industry_mathematical": 1109.105564707033, + "industry_medicine_and_healthcare": 1028.916818727131, + "industry_software_and_it_services": 1082.5387351562822, + "industry_writing_and_literature_and_language": 1054.2767837988906, + "instruction_following": 1053.563435082452, + "japanese": 935.4006586976568, + "korean": 905.1882570283535, + "longer_query": 1044.1048040532205, + "math": 1111.2626523919216, + "multi_turn": 1018.1503793468789, + "russian": 1021.0499828048607, + "spanish": 1083.5840426181815 + }, + "zephyr-7b-beta": { + "overall": 1069.4983677083846, + "chinese": 959.0194381642341, + "coding": 1045.1141439011312, + "creative_writing": 1102.8298405002797, + "english": 1107.425763475955, + "exclude_ties": 888.7826148761656, + "expert": 1005.372643037035, + "french": 1031.5974549007085, + "german": 988.8759134300633, + "hard_prompts": 1026.987376134216, + "hard_prompts_english": 1049.4448133310752, + "industry_business_and_management_and_financial_operations": 1000.3256440720311, + "industry_entertainment_and_sports_and_media": 1062.9949421168174, + "industry_legal_and_government": 1095.138140420419, + "industry_life_and_physical_and_social_science": 1060.4013707953843, + "industry_mathematical": 1044.3280747450053, + "industry_medicine_and_healthcare": 1025.992142736152, + "industry_software_and_it_services": 1049.196442493359, + "industry_writing_and_literature_and_language": 1085.6193672654574, + "instruction_following": 1031.8143782135176, + "longer_query": 1041.7415645699982, + "math": 1040.776717395585, + "multi_turn": 1037.8346531844559, + "russian": 1060.890920304884, + "spanish": 1072.2849784381633 + }, + "mpt-30b-chat": { + "overall": 1068.685462746424, + "coding": 1039.8100524927756, + "creative_writing": 1074.5826499932296, + "english": 1102.2672145350703, + "exclude_ties": 895.620850023177, + "hard_prompts": 1051.941757464771, + "hard_prompts_english": 1084.1759793195854, + "industry_business_and_management_and_financial_operations": 1008.2943826387234, + "industry_entertainment_and_sports_and_media": 1055.5543153630044, + "industry_life_and_physical_and_social_science": 1038.6370750183485, + "industry_mathematical": 1053.0982811937356, + "industry_software_and_it_services": 1058.600442185014, + "industry_writing_and_literature_and_language": 1079.1970212013896, + "instruction_following": 1054.23336358174, + "math": 1046.4595391883067, + "multi_turn": 1054.4262359154575 + }, + "codellama-34b-instruct": { + "overall": 1065.560273470057, + "chinese": 975.4984146065201, + "coding": 1046.3149806903475, + "creative_writing": 1032.1337732983402, + "english": 1092.0559254926566, + "exclude_ties": 883.0955140616962, + "hard_prompts": 1032.4645069080807, + "hard_prompts_english": 1048.7024582812414, + "industry_business_and_management_and_financial_operations": 1033.5036188104361, + "industry_entertainment_and_sports_and_media": 1015.0492538925289, + "industry_legal_and_government": 1064.3269560714002, + "industry_life_and_physical_and_social_science": 1066.8693436412286, + "industry_mathematical": 1054.9007121861941, + "industry_medicine_and_healthcare": 1024.7641822813018, + "industry_software_and_it_services": 1063.9001886536469, + "industry_writing_and_literature_and_language": 1048.1923172677816, + "instruction_following": 1028.2069086147308, + "longer_query": 1013.4996064314305, + "math": 1056.2814959279413, + "multi_turn": 1015.2576173336222 + }, + "zephyr-7b-alpha": { + "overall": 1058.4456788903888, + "coding": 1053.2618420622816, + "creative_writing": 1069.2781948257443, + "english": 1091.7015906820072, + "exclude_ties": 875.7478094738207, + "hard_prompts": 1024.2132903119232, + "hard_prompts_english": 1043.7203984643927, + "industry_entertainment_and_sports_and_media": 1048.622613791843, + "industry_life_and_physical_and_social_science": 1028.401950679201, + "industry_software_and_it_services": 1054.261425284502, + "industry_writing_and_literature_and_language": 1081.5054145274075, + "instruction_following": 1034.6594197531556, + "multi_turn": 1017.4977636184519 + }, + "vicuna-13b": { + "overall": 1057.7342591692022, + "chinese": 1038.5648913539321, + "coding": 1041.1682582261674, + "creative_writing": 1058.6441075976522, + "english": 1080.5981129176375, + "exclude_ties": 868.695243467233, + "expert": 993.5083438647391, + "french": 1020.81110951933, + "german": 992.2116037715944, + "hard_prompts": 1023.9554347858295, + "hard_prompts_english": 1037.392191568201, + "industry_business_and_management_and_financial_operations": 1005.8533172736143, + "industry_entertainment_and_sports_and_media": 1031.2397024038319, + "industry_legal_and_government": 1057.0661117442264, + "industry_life_and_physical_and_social_science": 1034.866082253417, + "industry_mathematical": 1021.709508785219, + "industry_medicine_and_healthcare": 1018.0260361891815, + "industry_software_and_it_services": 1048.8146198491268, + "industry_writing_and_literature_and_language": 1078.787904423153, + "instruction_following": 1035.3937099968855, + "japanese": 918.7502167124079, + "longer_query": 1047.4401230746987, + "math": 1030.2365000300756, + "multi_turn": 1030.7376412947463, + "russian": 1041.0059825945066, + "spanish": 1075.362838115959 + }, + "codellama-70b-instruct": { + "overall": 1057.3184970628704, + "english": 1095.9356762829198, + "exclude_ties": 872.7636447100203, + "hard_prompts": 1052.5301939124483, + "hard_prompts_english": 1079.2893071864821, + "industry_software_and_it_services": 1058.5926881810042, + "industry_writing_and_literature_and_language": 1014.6699230431489, + "instruction_following": 1023.9838192941188 + }, + "gemma-7b-it": { + "overall": 1056.231279740642, + "chinese": 1034.4973545637617, + "coding": 1048.646853091776, + "creative_writing": 1024.0329481874155, + "english": 1090.1153455998456, + "exclude_ties": 864.2278639744213, + "expert": 1000.8787573548473, + "french": 1025.7749903567183, + "hard_prompts": 1042.0179475770494, + "hard_prompts_english": 1062.3709377572852, + "industry_business_and_management_and_financial_operations": 1024.9053151237724, + "industry_entertainment_and_sports_and_media": 1001.7847181010732, + "industry_legal_and_government": 1044.4327742367855, + "industry_life_and_physical_and_social_science": 1020.7252242752955, + "industry_mathematical": 1061.2071328892225, + "industry_medicine_and_healthcare": 1026.751883303239, + "industry_software_and_it_services": 1060.6898929789904, + "industry_writing_and_literature_and_language": 1033.6773173727056, + "instruction_following": 1017.3193407485066, + "longer_query": 1021.7919233209932, + "math": 1066.2007264009212, + "multi_turn": 963.1116200229072, + "russian": 992.1463525053539 + }, + "llama-3.2-1b-instruct": { + "overall": 1054.7127048639052, + "chinese": 957.7126835350896, + "coding": 1071.0247142746848, + "creative_writing": 1033.313275974252, + "english": 1116.6936262109098, + "exclude_ties": 823.2908267941807, + "expert": 1006.6372068011768, + "german": 1013.5637887502621, + "hard_prompts": 1044.6411814547687, + "hard_prompts_english": 1088.5052409248153, + "industry_business_and_management_and_financial_operations": 999.530291491554, + "industry_entertainment_and_sports_and_media": 1025.4703121577782, + "industry_legal_and_government": 1070.7383772151152, + "industry_life_and_physical_and_social_science": 1055.018532474315, + "industry_mathematical": 1097.892574654631, + "industry_medicine_and_healthcare": 1008.0275867683732, + "industry_software_and_it_services": 1057.2315543006603, + "industry_writing_and_literature_and_language": 1029.6405755586136, + "instruction_following": 1031.3780220460237, + "longer_query": 1049.893205850833, + "math": 1085.569394093318, + "multi_turn": 1029.7387205133687, + "russian": 940.0425916446145 + }, + "falcon-180b-chat": { + "overall": 1054.4297221437066, + "creative_writing": 1089.079887781428, + "english": 1080.711364912463, + "exclude_ties": 868.309931276817, + "hard_prompts": 1007.6474855712164, + "industry_entertainment_and_sports_and_media": 1058.6388998129087, + "industry_life_and_physical_and_social_science": 1018.9547108568258, + "industry_software_and_it_services": 1025.3187096823679, + "industry_writing_and_literature_and_language": 1104.9716657363897, + "instruction_following": 1046.8701975116692, + "multi_turn": 1012.8030607535521 + }, + "llama-2-7b-chat": { + "overall": 1053.4747954139157, + "chinese": 972.0077057478939, + "coding": 1002.434888645548, + "creative_writing": 1033.7718513029834, + "english": 1095.9173089696162, + "exclude_ties": 861.6353076951144, + "expert": 1035.5404099912766, + "french": 970.4188975540453, + "german": 977.3578571177668, + "hard_prompts": 1009.5809402813207, + "hard_prompts_english": 1033.411675702915, + "industry_business_and_management_and_financial_operations": 1027.7604401647548, + "industry_entertainment_and_sports_and_media": 1020.1138956102563, + "industry_legal_and_government": 1064.9294725256416, + "industry_life_and_physical_and_social_science": 1047.3283565220254, + "industry_mathematical": 1038.5817796522317, + "industry_medicine_and_healthcare": 1034.9269987667417, + "industry_software_and_it_services": 1029.9614721604898, + "industry_writing_and_literature_and_language": 1036.9862185742472, + "instruction_following": 1006.1335314184189, + "longer_query": 999.1659024555036, + "math": 1041.7712044290015, + "multi_turn": 1028.9789862343418, + "russian": 993.8750543975979, + "spanish": 1006.1108678864042 + }, + "guanaco-33b": { + "overall": 1053.4546517949543, + "coding": 996.0409063229174, + "creative_writing": 1080.5224548211754, + "english": 1085.4321695561403, + "exclude_ties": 872.4574287878468, + "hard_prompts": 1005.4226475004812, + "hard_prompts_english": 1022.036357557141, + "industry_business_and_management_and_financial_operations": 972.6864798537092, + "industry_entertainment_and_sports_and_media": 1073.1868479371742, + "industry_life_and_physical_and_social_science": 1059.799528348061, + "industry_mathematical": 1019.8983298673546, + "industry_software_and_it_services": 1013.7061720505983, + "industry_writing_and_literature_and_language": 1066.1747776452635, + "instruction_following": 991.5992977127268, + "math": 1032.4457703423018, + "multi_turn": 1005.8683346175746 + }, + "qwen-14b-chat": { + "overall": 1051.103485378696, + "chinese": 1075.9406545586385, + "coding": 1072.0463068423178, + "creative_writing": 1028.212196620616, + "english": 1068.5464691277616, + "exclude_ties": 865.6805281160875, + "hard_prompts": 1027.4547828521208, + "hard_prompts_english": 1042.7504813053824, + "industry_business_and_management_and_financial_operations": 1001.3738099882306, + "industry_entertainment_and_sports_and_media": 996.8893358461377, + "industry_legal_and_government": 1050.7544145960578, + "industry_life_and_physical_and_social_science": 1010.5862567899217, + "industry_mathematical": 1073.0909896212302, + "industry_medicine_and_healthcare": 1008.0023609157779, + "industry_software_and_it_services": 1053.8146669992109, + "industry_writing_and_literature_and_language": 1049.8016144058895, + "instruction_following": 1031.6016804214573, + "longer_query": 1028.6077901786161, + "math": 1067.565810280229, + "multi_turn": 1021.9796664144417 + }, + "phi-3-mini-128k-instruct": { + "overall": 1050.4658311539322, + "chinese": 1015.3393943722024, + "coding": 1039.9507014547337, + "creative_writing": 1024.7268953714506, + "english": 1082.9938332162455, + "exclude_ties": 845.05274321256, + "expert": 983.3776062612226, + "french": 1039.6569569085104, + "german": 1005.9139008266112, + "hard_prompts": 1028.3945445703844, + "hard_prompts_english": 1052.9206545731095, + "industry_business_and_management_and_financial_operations": 989.9388658390831, + "industry_entertainment_and_sports_and_media": 1009.4485032308821, + "industry_legal_and_government": 1006.2494684635652, + "industry_life_and_physical_and_social_science": 1023.1474471937581, + "industry_mathematical": 1082.6754570872267, + "industry_medicine_and_healthcare": 994.7731257874918, + "industry_software_and_it_services": 1034.7083843687356, + "industry_writing_and_literature_and_language": 1040.2133844064115, + "instruction_following": 1023.1183790070108, + "japanese": 899.5222333484533, + "korean": 855.7747757878151, + "longer_query": 995.8472862781553, + "math": 1088.8801606699376, + "multi_turn": 988.791268000357, + "russian": 1002.7522031575147, + "spanish": 1057.3122360035254 + }, + "smollm2-1.7b-instruct": { + "overall": 1042.1059184710602, + "coding": 1048.3507682048585, + "creative_writing": 1031.9561116305329, + "english": 1092.7261323008056, + "exclude_ties": 797.4574987830075, + "hard_prompts": 1053.4842714014305, + "hard_prompts_english": 1070.2037422693434, + "industry_business_and_management_and_financial_operations": 965.1037746803128, + "industry_entertainment_and_sports_and_media": 1008.0265939081585, + "industry_life_and_physical_and_social_science": 1046.5929401804306, + "industry_mathematical": 1121.9192640725419, + "industry_software_and_it_services": 1047.5866868433682, + "industry_writing_and_literature_and_language": 1024.4052327261056, + "instruction_following": 1029.4951091501064, + "longer_query": 1056.4548965580207, + "math": 1104.6087128419344, + "multi_turn": 992.9065767749468, + "russian": 980.1062364917857 + }, + "stripedhyena-nous-7b": { + "overall": 1038.515789255196, + "coding": 1001.5700015219843, + "creative_writing": 1029.821089630304, + "english": 1070.5492708728589, + "exclude_ties": 840.3284434900694, + "french": 1032.7578914652559, + "hard_prompts": 998.5962260950369, + "hard_prompts_english": 1016.7657774798715, + "industry_business_and_management_and_financial_operations": 969.9959746466477, + "industry_entertainment_and_sports_and_media": 1011.7727496015641, + "industry_legal_and_government": 1000.3787455682789, + "industry_life_and_physical_and_social_science": 1012.2932104432737, + "industry_mathematical": 1020.9837731666885, + "industry_medicine_and_healthcare": 983.5733370268758, + "industry_software_and_it_services": 1022.5966180520903, + "industry_writing_and_literature_and_language": 1034.560406264493, + "instruction_following": 1010.5995249231537, + "longer_query": 982.6533969592303, + "math": 1032.9165849799017, + "multi_turn": 1007.4827087276091 + }, + "olmo-7b-instruct": { + "overall": 1031.887069929568, + "chinese": 1012.6414971910008, + "coding": 1016.1845293158274, + "creative_writing": 990.2614625025026, + "english": 1066.7884820551906, + "exclude_ties": 827.7111686419615, + "hard_prompts": 993.1918216638832, + "hard_prompts_english": 1011.6803644408669, + "industry_business_and_management_and_financial_operations": 1019.504279504084, + "industry_entertainment_and_sports_and_media": 991.209251908237, + "industry_legal_and_government": 1046.7841224954677, + "industry_life_and_physical_and_social_science": 1008.3008704521635, + "industry_mathematical": 1017.9730690333306, + "industry_medicine_and_healthcare": 1033.689218687108, + "industry_software_and_it_services": 1038.1517858165794, + "industry_writing_and_literature_and_language": 1003.0276847492704, + "instruction_following": 978.5438685840585, + "math": 1017.6150557594999, + "multi_turn": 1006.9209090237159, + "russian": 945.296185912752 + }, + "vicuna-7b": { + "overall": 1031.0224182981867, + "chinese": 978.0763322728462, + "coding": 1010.5377027875854, + "creative_writing": 1021.616528768014, + "english": 1051.3627741495418, + "exclude_ties": 841.3980432875894, + "hard_prompts": 995.2662748053059, + "hard_prompts_english": 1006.1401460691468, + "industry_business_and_management_and_financial_operations": 972.5368699208993, + "industry_entertainment_and_sports_and_media": 1016.9379461091405, + "industry_legal_and_government": 1032.0899784510482, + "industry_life_and_physical_and_social_science": 1001.3242332893624, + "industry_mathematical": 979.8888884560683, + "industry_medicine_and_healthcare": 996.7197559773041, + "industry_software_and_it_services": 1018.1186593135742, + "industry_writing_and_literature_and_language": 1048.4682856500951, + "instruction_following": 994.0911880899187, + "longer_query": 1035.649148746009, + "math": 993.6604265463615, + "multi_turn": 987.7157750605306 + }, + "palm-2": { + "overall": 1027.313410061157, + "chinese": 886.3022790745308, + "coding": 994.9623798204241, + "creative_writing": 991.9469372994555, + "english": 1063.492153425832, + "exclude_ties": 830.5676349400819, + "hard_prompts": 1005.3320500628759, + "hard_prompts_english": 1037.0223168365187, + "industry_business_and_management_and_financial_operations": 958.5662668687819, + "industry_entertainment_and_sports_and_media": 992.5969056934582, + "industry_legal_and_government": 1005.6392161145525, + "industry_life_and_physical_and_social_science": 1011.5087141540564, + "industry_mathematical": 1042.9966760765321, + "industry_medicine_and_healthcare": 960.604354141791, + "industry_software_and_it_services": 1000.6682946046035, + "industry_writing_and_literature_and_language": 1006.7104631817889, + "instruction_following": 1010.3572692272221, + "longer_query": 1009.8518809164862, + "math": 1048.716387114016, + "multi_turn": 994.236717279557 + }, + "mistral-7b-instruct": { + "overall": 1023.5834573061757, + "chinese": 930.5152477091331, + "coding": 1018.9493622262164, + "creative_writing": 1028.9034888006581, + "english": 1058.883067050495, + "exclude_ties": 819.4145231980289, + "expert": 953.3877198961302, + "french": 946.2499073612814, + "german": 930.9315492488176, + "hard_prompts": 1001.9479315482591, + "hard_prompts_english": 1023.546014175955, + "industry_business_and_management_and_financial_operations": 952.2981273540933, + "industry_entertainment_and_sports_and_media": 999.898936715325, + "industry_legal_and_government": 1024.5622402634053, + "industry_life_and_physical_and_social_science": 998.8444626056287, + "industry_mathematical": 1009.1220120697036, + "industry_medicine_and_healthcare": 992.5969151247151, + "industry_software_and_it_services": 1015.5901482022023, + "industry_writing_and_literature_and_language": 1030.233732701887, + "instruction_following": 1003.5641765183909, + "longer_query": 1000.7339928621706, + "math": 1027.3033445794601, + "multi_turn": 1007.998099609255, + "russian": 1000.2206420356767 + }, + "gemma-1.1-2b-it": { + "overall": 1022.3000590581174, + "chinese": 1010.5783107499406, + "coding": 1034.6538856950215, + "creative_writing": 998.5142439244681, + "english": 1050.2889366502588, + "exclude_ties": 796.858893462686, + "expert": 969.7298765926732, + "german": 943.6962192546603, + "hard_prompts": 1005.3421258084159, + "hard_prompts_english": 1027.6033752697303, + "industry_business_and_management_and_financial_operations": 980.7577803031256, + "industry_entertainment_and_sports_and_media": 980.2834854441265, + "industry_legal_and_government": 995.3869182168446, + "industry_life_and_physical_and_social_science": 996.3516986006819, + "industry_mathematical": 1042.7073354945762, + "industry_medicine_and_healthcare": 989.298044451606, + "industry_software_and_it_services": 1032.8927826116524, + "industry_writing_and_literature_and_language": 1005.9085294703569, + "instruction_following": 992.1105084369028, + "korean": 898.6811808123138, + "longer_query": 1002.7675657524633, + "math": 1047.0341359586912, + "multi_turn": 958.3959199396065, + "russian": 989.197188284552 + }, + "gemma-2b-it": { + "overall": 1002.2230063969662, + "chinese": 984.9388423092714, + "coding": 1010.124306351245, + "creative_writing": 986.9772075881311, + "english": 1032.8154845870665, + "exclude_ties": 780.089477869021, + "hard_prompts": 989.1471882837691, + "hard_prompts_english": 997.7092971104032, + "industry_business_and_management_and_financial_operations": 998.7035109793754, + "industry_entertainment_and_sports_and_media": 968.0605541304246, + "industry_legal_and_government": 1019.631361971572, + "industry_life_and_physical_and_social_science": 950.128590208941, + "industry_mathematical": 1004.1536558919311, + "industry_medicine_and_healthcare": 937.1924557380203, + "industry_software_and_it_services": 1020.8091417438709, + "industry_writing_and_literature_and_language": 978.177751690559, + "instruction_following": 970.7300957132406, + "longer_query": 980.7854062498463, + "math": 1008.5349374713638, + "multi_turn": 944.7341602657732, + "russian": 935.4773366728998 + }, + "qwen1.5-4b-chat": { + "overall": 997.3103594080544, + "chinese": 1022.913716311389, + "coding": 999.3255403830451, + "creative_writing": 969.7786053840211, + "english": 1015.1648856943864, + "exclude_ties": 763.5666416073791, + "expert": 979.4441928275812, + "german": 901.7946035259145, + "hard_prompts": 976.5862835533059, + "hard_prompts_english": 985.1264388961199, + "industry_business_and_management_and_financial_operations": 964.7403183313476, + "industry_entertainment_and_sports_and_media": 953.0157094949898, + "industry_legal_and_government": 1010.3381592896822, + "industry_life_and_physical_and_social_science": 976.659130720669, + "industry_mathematical": 998.2077538993387, + "industry_medicine_and_healthcare": 954.4135730308772, + "industry_software_and_it_services": 995.55540622134, + "industry_writing_and_literature_and_language": 1000.7362233980668, + "instruction_following": 978.7656958819931, + "longer_query": 987.7916847631026, + "math": 1025.7495691424851, + "multi_turn": 977.3674150802408, + "russian": 950.8569858170556 + }, + "koala-13b": { + "overall": 989.6838499343542, + "chinese": 879.9618296310721, + "coding": 945.4167253156322, + "creative_writing": 973.5051265316451, + "english": 1021.8439754483732, + "exclude_ties": 782.4950549133473, + "hard_prompts": 930.8089496867268, + "hard_prompts_english": 944.8655229173654, + "industry_business_and_management_and_financial_operations": 936.3213622833651, + "industry_entertainment_and_sports_and_media": 945.1131330006702, + "industry_legal_and_government": 963.8861004707181, + "industry_life_and_physical_and_social_science": 986.6774012653896, + "industry_mathematical": 928.5081941491901, + "industry_medicine_and_healthcare": 954.9539089103159, + "industry_software_and_it_services": 961.8489067471971, + "industry_writing_and_literature_and_language": 979.3028619340307, + "instruction_following": 943.9592848513664, + "math": 932.2189731312508, + "multi_turn": 923.3377779184411 + }, + "chatglm3-6b": { + "overall": 972.113320654106, + "chinese": 1063.1937614038497, + "coding": 964.6933308385453, + "creative_writing": 970.5634781935934, + "english": 1002.9664152509966, + "exclude_ties": 742.0373194980004, + "hard_prompts": 953.5495616587848, + "hard_prompts_english": 978.6841115793832, + "industry_business_and_management_and_financial_operations": 931.217821944067, + "industry_entertainment_and_sports_and_media": 928.8866951264545, + "industry_legal_and_government": 973.5312261495666, + "industry_life_and_physical_and_social_science": 932.1682822115711, + "industry_mathematical": 964.1340547060167, + "industry_medicine_and_healthcare": 891.251820421021, + "industry_software_and_it_services": 970.3206036890997, + "industry_writing_and_literature_and_language": 972.4291556462708, + "instruction_following": 954.2951640639249, + "longer_query": 951.9775111075639, + "math": 988.9401383209336, + "multi_turn": 961.6806007581822 + }, + "gpt4all-13b-snoozy": { + "overall": 956.2560414888144, + "creative_writing": 972.5700657113111, + "english": 988.9308610081234, + "exclude_ties": 727.594801936198, + "hard_prompts": 936.5701493202239, + "hard_prompts_english": 957.0999421182337, + "industry_entertainment_and_sports_and_media": 941.7524927854852, + "industry_life_and_physical_and_social_science": 902.2373787492188, + "industry_software_and_it_services": 932.1200853682712, + "industry_writing_and_literature_and_language": 972.999119269339, + "instruction_following": 945.7861156779167, + "math": 940.7047179171366, + "multi_turn": 926.5512956560719 + }, + "mpt-7b-chat": { + "overall": 955.2936365439722, + "chinese": 953.7637107375363, + "coding": 913.645613352639, + "creative_writing": 958.2058469290982, + "english": 980.7380781169771, + "exclude_ties": 727.3847719652756, + "hard_prompts": 895.9235071309004, + "hard_prompts_english": 910.130821935882, + "industry_business_and_management_and_financial_operations": 864.3855235551789, + "industry_entertainment_and_sports_and_media": 928.577932552273, + "industry_life_and_physical_and_social_science": 913.5900163082586, + "industry_mathematical": 906.7800699194856, + "industry_software_and_it_services": 913.3139884436365, + "industry_writing_and_literature_and_language": 972.6864737491477, + "instruction_following": 907.0396004700353, + "math": 919.0706777474575, + "multi_turn": 901.4249164305536 + }, + "RWKV-4-Raven-14B": { + "overall": 948.4605155999152, + "chinese": 909.6242718139929, + "coding": 928.3638575162714, + "creative_writing": 934.2965319587929, + "english": 968.0501744399301, + "exclude_ties": 721.5198812626726, + "hard_prompts": 896.4637402556223, + "hard_prompts_english": 903.077254750945, + "industry_business_and_management_and_financial_operations": 888.2193385745966, + "industry_entertainment_and_sports_and_media": 903.0287746057461, + "industry_legal_and_government": 975.8896414806602, + "industry_life_and_physical_and_social_science": 928.1464763069979, + "industry_mathematical": 918.9029550818777, + "industry_medicine_and_healthcare": 943.4156789337608, + "industry_software_and_it_services": 915.659440547699, + "industry_writing_and_literature_and_language": 933.8983087463071, + "instruction_following": 882.8801244716594, + "math": 921.7801746949142, + "multi_turn": 893.6210448131071 + }, + "chatglm2-6b": { + "overall": 938.702180025732, + "coding": 902.4343074782125, + "creative_writing": 941.6643068772632, + "english": 973.4768169653048, + "exclude_ties": 685.2399846392466, + "hard_prompts": 917.4129948684727, + "hard_prompts_english": 940.6190101168615, + "industry_business_and_management_and_financial_operations": 886.9434586150169, + "industry_entertainment_and_sports_and_media": 922.1423833377155, + "industry_life_and_physical_and_social_science": 916.2092533522755, + "industry_software_and_it_services": 918.3991662572246, + "industry_writing_and_literature_and_language": 952.5706937723314, + "instruction_following": 902.3950683710146, + "math": 915.0127948002124, + "multi_turn": 901.9149311885312 + }, + "alpaca-13b": { + "overall": 932.8851207487069, + "chinese": 780.9605332118033, + "coding": 799.3132113822606, + "creative_writing": 968.1192018880231, + "english": 957.3543807430705, + "exclude_ties": 697.8510711720514, + "hard_prompts": 826.0582809664122, + "hard_prompts_english": 841.2122738229714, + "industry_business_and_management_and_financial_operations": 812.1507994779132, + "industry_entertainment_and_sports_and_media": 932.703425637372, + "industry_legal_and_government": 911.0412499451361, + "industry_life_and_physical_and_social_science": 883.1649233055095, + "industry_mathematical": 879.2545196677816, + "industry_medicine_and_healthcare": 818.926426812147, + "industry_software_and_it_services": 827.3258983276351, + "industry_writing_and_literature_and_language": 968.082495497245, + "instruction_following": 891.5038574193395, + "math": 908.0048543520443, + "multi_turn": 884.1923564724779 + }, + "chatglm-6b": { + "overall": 918.538960030424, + "chinese": 1076.367903520699, + "coding": 919.8879520221484, + "creative_writing": 900.5870995653112, + "english": 938.257491740166, + "exclude_ties": 662.1009278101355, + "hard_prompts": 901.450757214689, + "hard_prompts_english": 898.0357907579355, + "industry_business_and_management_and_financial_operations": 845.4423176951511, + "industry_entertainment_and_sports_and_media": 841.3020570120011, + "industry_legal_and_government": 895.5711065171183, + "industry_life_and_physical_and_social_science": 849.8252950819943, + "industry_mathematical": 911.8955719735906, + "industry_medicine_and_healthcare": 822.9277533695266, + "industry_software_and_it_services": 889.3963478240303, + "industry_writing_and_literature_and_language": 925.2631770587293, + "instruction_following": 902.1507855127948, + "math": 925.5688514384098, + "multi_turn": 856.3243178150822 + }, + "oasst-pythia-12b": { + "overall": 916.1408839395293, + "chinese": 802.009907595136, + "coding": 900.8295362192712, + "creative_writing": 923.8290610787064, + "english": 945.4692196025103, + "exclude_ties": 672.0473998423398, + "hard_prompts": 884.9583477973092, + "hard_prompts_english": 899.2866790234223, + "industry_business_and_management_and_financial_operations": 822.2938602833956, + "industry_entertainment_and_sports_and_media": 885.6619016202253, + "industry_legal_and_government": 922.4745617157834, + "industry_life_and_physical_and_social_science": 897.0296221010481, + "industry_mathematical": 890.5834750033467, + "industry_medicine_and_healthcare": 818.0111976237273, + "industry_software_and_it_services": 882.3469988597146, + "industry_writing_and_literature_and_language": 928.1102563879577, + "instruction_following": 887.6705501624347, + "math": 891.7223915468036, + "multi_turn": 869.1441626836634 + }, + "fastchat-t5-3b": { + "overall": 894.4734317575424, + "chinese": 717.9213063963977, + "coding": 764.4074943821385, + "creative_writing": 902.6573378010041, + "english": 934.6470223034632, + "exclude_ties": 635.4024954304946, + "hard_prompts": 814.5994671473043, + "hard_prompts_english": 831.5251595581499, + "industry_business_and_management_and_financial_operations": 868.2702483137198, + "industry_entertainment_and_sports_and_media": 860.2583109263294, + "industry_legal_and_government": 898.926768105543, + "industry_life_and_physical_and_social_science": 865.8679890043468, + "industry_mathematical": 851.1859889518264, + "industry_software_and_it_services": 836.8628254259595, + "industry_writing_and_literature_and_language": 907.4675523404637, + "instruction_following": 858.3428767214266, + "math": 861.6169924488922, + "multi_turn": 852.8632543226367 + }, + "stablelm-tuned-alpha-7b": { + "overall": 866.6474187555608, + "coding": 886.5316891905261, + "creative_writing": 868.4243897843589, + "english": 889.6857082700818, + "exclude_ties": 583.893118094984, + "hard_prompts": 835.5281044006632, + "hard_prompts_english": 851.22401597746, + "industry_business_and_management_and_financial_operations": 786.0665041113928, + "industry_entertainment_and_sports_and_media": 823.3901318721034, + "industry_life_and_physical_and_social_science": 803.6882092596481, + "industry_mathematical": 834.9624542081699, + "industry_software_and_it_services": 858.7249069268798, + "industry_writing_and_literature_and_language": 867.8082218310165, + "instruction_following": 831.3229009923303, + "math": 838.9032355524978, + "multi_turn": 820.5251209635803 + }, + "dolly-v2-12b": { + "overall": 851.2017707466475, + "chinese": 835.4453441687741, + "coding": 776.7916703789903, + "creative_writing": 864.2505673607096, + "english": 866.7675741696949, + "exclude_ties": 568.2389566279836, + "hard_prompts": 804.1219692389766, + "hard_prompts_english": 811.1423566987501, + "industry_business_and_management_and_financial_operations": 734.077095518022, + "industry_entertainment_and_sports_and_media": 809.3863389959648, + "industry_life_and_physical_and_social_science": 801.2497183479315, + "industry_mathematical": 839.8441748198422, + "industry_software_and_it_services": 790.0432062998876, + "industry_writing_and_literature_and_language": 872.1419727591103, + "instruction_following": 814.020663518399, + "math": 870.8028653498563, + "multi_turn": 740.2568000461703 + }, + "llama-13b": { + "overall": 833.5846216837616, + "coding": 683.4434115529443, + "creative_writing": 794.5130702205912, + "english": 845.8854957363097, + "exclude_ties": 534.1716390592597, + "hard_prompts": 728.4084244887163, + "hard_prompts_english": 736.4357538033521, + "industry_entertainment_and_sports_and_media": 785.1855216981465, + "industry_life_and_physical_and_social_science": 806.5865083401636, + "industry_mathematical": 833.3534592977525, + "industry_software_and_it_services": 720.2984164448287, + "industry_writing_and_literature_and_language": 808.9634209014641, + "instruction_following": 781.1393565681981, + "math": 837.6357216869443, + "multi_turn": 753.0469412649679 + } + } + }, + "namespace_to_type": [ + { + "prefix": "nim/", + "nat_type": "nim", + "note": "NVIDIA NIM \u2014 the platform's native provider type." + }, + { + "prefix": "openai/", + "nat_type": "openai", + "note": "OpenAI direct API." + }, + { + "prefix": "anthropic/", + "nat_type": "anthropic", + "note": "Anthropic direct API. Verify with `nat info components -t llm_provider`." + }, + { + "prefix": "bedrock/", + "nat_type": "aws_bedrock", + "note": "AWS Bedrock. Verify with `nat info components -t llm_provider`; exact type may be `aws_bedrock` or `bedrock`." + }, + { + "prefix": "ollama/", + "nat_type": "openai", + "note": "Local Ollama exposes an OpenAI-compatible endpoint; use `_type: openai` with the Ollama base_url." + }, + { + "prefix": "qwen/", + "nat_type": "nim", + "note": "Qwen models served through NIM." + }, + { + "prefix": "meta/", + "nat_type": "nim", + "note": "Meta Llama models served through NIM." + }, + { + "prefix": "nvidia/", + "nat_type": "nim", + "note": "NVIDIA-published models (Nemotron family) served through NIM." + }, + { + "prefix": "microsoft/", + "nat_type": "nim", + "note": "Microsoft Phi models served through NIM." + }, + { + "prefix": "mistralai/", + "nat_type": "nim", + "note": "Mistral models served through NIM when published there." + } + ], + "name_decomposition_rules": [ + { + "pattern": "qwen3", + "category": "family", + "hint": "Qwen3 family \u2014 Alibaba's instruction-tuned series with a public tool-calling track record" + }, + { + "pattern": "qwen2", + "category": "family", + "hint": "Qwen2 family \u2014 older Alibaba generation, generally superseded by Qwen3 where available" + }, + { + "pattern": "llama", + "category": "family", + "hint": "Meta Llama family \u2014 widely deployed, well-characterized baseline" + }, + { + "pattern": "nemotron", + "category": "family", + "hint": "NVIDIA Nemotron \u2014 Llama-derived post-trained variant emphasizing reasoning over function calling" + }, + { + "pattern": "phi", + "category": "family", + "hint": "Microsoft Phi family \u2014 small-but-capable models tuned for quality-per-parameter" + }, + { + "pattern": "mistral", + "category": "family", + "hint": "Mistral family \u2014 French open-weight models, generally strong on European languages" + }, + { + "pattern": "mixtral", + "category": "family", + "hint": "Mistral Mixtral \u2014 sparse MoE variant of the Mistral line" + }, + { + "pattern": "claude", + "category": "family", + "hint": "Anthropic Claude family \u2014 frontier closed-weight model, strong on instruction-following and code" + }, + { + "pattern": "gpt", + "category": "family", + "hint": "OpenAI GPT family \u2014 frontier closed-weight model" + }, + { + "pattern": "gemini", + "category": "family", + "hint": "Google Gemini family \u2014 frontier closed-weight model with strong multimodal support" + }, + { + "pattern": "deepseek", + "category": "family", + "hint": "DeepSeek family \u2014 Chinese open-weight models with strong code and reasoning track record" + }, + { + "pattern": "0.6b", + "category": "size", + "hint": "0.6B parameters \u2014 very small, edge/embedded class" + }, + { + "pattern": "1.7b", + "category": "size", + "hint": "1.7B parameters \u2014 small, low-VRAM" + }, + { + "pattern": "3b", + "category": "size", + "hint": "~3B parameters \u2014 small dense, fits single consumer GPU" + }, + { + "pattern": "4b", + "category": "size", + "hint": "~4B parameters \u2014 small dense, fits single consumer GPU" + }, + { + "pattern": "7b", + "category": "size", + "hint": "~7B parameters \u2014 small-mid dense, fits 12\u201316 GB VRAM" + }, + { + "pattern": "8b", + "category": "size", + "hint": "~8B parameters \u2014 small-mid dense, fits single consumer GPU" + }, + { + "pattern": "14b", + "category": "size", + "hint": "~14B parameters \u2014 mid dense, needs 24+ GB for self-hosting" + }, + { + "pattern": "30b", + "category": "size", + "hint": "~30B parameters \u2014 mid-large; pair with active-param suffix for MoE variants" + }, + { + "pattern": "32b", + "category": "size", + "hint": "~32B parameters \u2014 mid-large dense" + }, + { + "pattern": "49b", + "category": "size", + "hint": "~49B parameters \u2014 large dense, distilled-from-larger variants are common" + }, + { + "pattern": "70b", + "category": "size", + "hint": "~70B parameters \u2014 large dense, cloud or multi-GPU self-host" + }, + { + "pattern": "235b", + "category": "size", + "hint": "~235B parameters total \u2014 very large; check for MoE active-param suffix" + }, + { + "pattern": "253b", + "category": "size", + "hint": "~253B parameters \u2014 very large dense" + }, + { + "pattern": "405b", + "category": "size", + "hint": "~405B parameters \u2014 frontier dense scale" + }, + { + "pattern": "a3b", + "category": "size", + "hint": "Sparse MoE with ~3B active parameters per forward pass \u2014 fast inference at moderate quality" + }, + { + "pattern": "a22b", + "category": "size", + "hint": "Sparse MoE with ~22B active parameters per forward pass" + }, + { + "pattern": "instruct", + "category": "tuning", + "hint": "Instruction-tuned \u2014 designed to follow user directions, not base-model completion" + }, + { + "pattern": "chat", + "category": "tuning", + "hint": "Chat-tuned \u2014 optimized for conversational back-and-forth" + }, + { + "pattern": "base", + "category": "tuning", + "hint": "Base model \u2014 pretraining only, no instruction tuning. Usually wrong for agents." + }, + { + "pattern": "rlhf", + "category": "tuning", + "hint": "RLHF-tuned \u2014 aligned via reinforcement learning from human feedback" + }, + { + "pattern": "dpo", + "category": "tuning", + "hint": "DPO-tuned \u2014 aligned via direct preference optimization" + }, + { + "pattern": "thinking", + "category": "tuning", + "hint": "Reasoning/thinking variant \u2014 exposes intermediate chain-of-thought; usually slower but better on hard problems" + }, + { + "pattern": "coder", + "category": "specialization", + "hint": "Code-specialized fine-tune \u2014 strong on software engineering tasks, weaker on general reasoning" + }, + { + "pattern": "code", + "category": "specialization", + "hint": "Code-specialized variant \u2014 strong on software engineering tasks" + }, + { + "pattern": "vl", + "category": "specialization", + "hint": "Vision-language variant \u2014 accepts image input" + }, + { + "pattern": "vision", + "category": "specialization", + "hint": "Vision-language variant \u2014 accepts image input" + }, + { + "pattern": "math", + "category": "specialization", + "hint": "Math-specialized variant \u2014 strong on numerical/symbolic reasoning" + }, + { + "pattern": "mini", + "category": "size", + "hint": "'Mini' variant \u2014 small parameter count within the family; distinct from the same family's full-size model" + }, + { + "pattern": "super", + "category": "size", + "hint": "'Super' tier \u2014 middle ground within NVIDIA's Nemotron lineup" + }, + { + "pattern": "ultra", + "category": "size", + "hint": "'Ultra' tier \u2014 largest size within NVIDIA's Nemotron lineup" + } + ] +} \ No newline at end of file diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-skill-selection/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-skill-selection/SKILL.md index b0597932805..dc5f3318626 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-skill-selection/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-skill-selection/SKILL.md @@ -1,6 +1,6 @@ --- name: nemo-skill-selection -description: Top-level skill selector for any task involving NeMo Platform (NVIDIA's agent platform). Picks the right downstream skill (setup, explore, spec, build, try, status, teardown, customization training) from natural-language intent. Use over generic brainstorming, planning, or onboarding skills for any NeMo Platform task. +description: Top-level skill selector for ambiguous tasks involving NeMo Platform (NVIDIA's agent platform). Picks the right downstream skill for setup, design, specification, agent configuration, build, deployment, testing, status, teardown, evaluation, optimization, security, or model customization. Use when the user needs help deciding where a NeMo Platform task should start. triggers: - build an agent - create an agent @@ -33,7 +33,15 @@ allowed-tools: [Read] You are deciding which downstream NeMo Platform skill should run. This skill never executes commands. It picks the next skill, announces the choice, and hands off. -NeMo Platform optimizes LangGraph agents wrapped in NVIDIA NeMo Agent Toolkit (NAT). State that constraint when the user describes an agent in another framework (CrewAI, AutoGen, plain LangChain, Pydantic AI). Those frameworks need a user-written NAT wrapper before the platform's optimization, evaluation, and guardrails surfaces apply. +New NeMo Platform agent builds use a Platform-owned `agent.yaml` with +`config_format: nemo-agents-spec-v1` and a supported harness. NVIDIA NeMo Agent +Toolkit (NAT) workflow YAML remains a compatibility path. Do not describe NAT +as the only supported implementation model. + +If an existing agent does not fit a supported harness contract, route based on +the user's goal: preserve an existing NAT workflow, identify a custom adapter, +or use `nemo-agent-config` for a best-effort migration. Do not promise that an +arbitrary Python entrypoint can be converted mechanically. ## Decision table @@ -44,8 +52,9 @@ Match the user's intent to one downstream skill. Pick exactly one. | "set up", "install", "get started", "try NeMo", "first time" | `setup` | Verify the platform is installed and running. If not, the skill tells the user how to run the CLI install (`make bootstrap` + `nemo setup`). Install itself is CLI-only. | | "design an agent", "I want an agent that handles X", "what should my agent do" | `nemo-explore` | Capture the agent's job, audience, categories, tools, model, constraints before any code | | "write the spec", "save the design", "capture what we agreed" | `nemo-spec` | Persist the explore answers as `agents/-spec/AGENT-SPEC.md` | -| "build the agent", "create the agent", "deploy", "scaffold from spec" | `nemo-build-agent` | Scaffold the NAT workflow YAML, deploy, eval, optional guardrails | -| "ask my agent", "try the agent", "test it" | `nemo-try-agent` | Send a query to a deployed agent or fall back to model chat | +| "write agent.yaml", "validate agent.yaml", "choose a harness", "migrate this NAT YAML", "convert to nemo-agents-spec-v1" | `nemo-agent-config` | Author or migrate the Platform-owned machine-readable config without running the full build | +| "build the agent", "create the agent", "deploy", "scaffold from spec" | `nemo-build-agent` | Build from the approved spec, default to Platform `agent.yaml`, register, deploy, evaluate, and optionally apply guardrails | +| "ask my agent", "try the agent", "test it", "invoke this agent.yaml" | `nemo-try-agent` | Invoke a named deployment or run a local agent YAML config once | | "status", "what is running", "platform health", "is the platform up", "what's deployed", "show me what's running" | `nemo-status` | Read-only dashboard: platform, agents, providers, models | | "shut down", "stop NeMo", "tear down", "clean up" | `nemo-teardown` | Stop the cluster (keep data, delete platform data, or full cleanup) | | "fine-tune", "customize the model", "train on my data", "SFT", "LoRA" | `nemo-customizer` | Model customization via installed customization contributor plugins (`nemo-customizer-plugin`). Requires plugin skills to be installed (`nemo skills install` / enabled-plugins). | @@ -55,7 +64,12 @@ Match the user's intent to one downstream skill. Pick exactly one. **Optimize vs build:** Do NOT route optimize asks to `nemo-build-agent`. Build is for creating new agents from a spec; optimize is for tuning **already deployed** agents. If the user says "make my agent faster" or "use a cheaper model," that is `agents-optimize`, not `nemo-build-agent`. -If two rows fit, pick the earliest one in the lifecycle (setup before build before try). If nothing matches, ask one disambiguating question with the relevant rows as a numbered list. +If a request includes both config authoring and deployment, choose +`nemo-build-agent`; it delegates the config portion to `nemo-agent-config`. +Choose `nemo-agent-config` when the requested output stops at a validated config +or migration. Otherwise, if two rows fit, pick the earliest one in the +lifecycle. If nothing matches, ask one disambiguating question with the +relevant rows as a numbered list. ## Pre-flight @@ -100,8 +114,9 @@ NeMo Platform skills I can route to: setup verify install or get the CLI install command nemo-explore design conversation: capture goal, audience, tools, constraints nemo-spec write the design to agents/-spec/AGENT-SPEC.md - nemo-build-agent scaffold the NAT workflow YAML and deploy - nemo-try-agent query a deployed agent or chat with a model + nemo-agent-config author, validate, or migrate Platform agent.yaml + nemo-build-agent build from the spec, register, deploy, evaluate, and sign off + nemo-try-agent invoke a named deployment or local agent YAML config nemo-status read-only platform health dashboard nemo-teardown guided shutdown @@ -143,4 +158,8 @@ Do not proactively suggest Studio as the path for anything a skill already cover - **Install must happen before any skill can do useful work.** Build, try, and status all assume the platform is up. If the user has not run the CLI install (`make bootstrap` + `nemo setup`), the skills cannot work around that; hand them to `setup` for instructions. - **NeMo Platform is the product name.** Capital N, e, M, o, P. Not "nemo" or "Nemo." NAT on first mention is "NVIDIA NeMo Agent Toolkit (NAT)." - **Model customization** goes to the `nemo-customizer` plugin skill when `nemo-customizer-plugin` (and a training backend) are installed. If that skill is not available, tell the user to enable customization plugins and install skills — do not improvise training with an external library. -- **Framework honesty.** If the user describes an agent in CrewAI, AutoGen, plain LangChain, or Pydantic AI, tell them up front that NeMo Platform's optimization and evaluation surfaces operate on NAT-wrapped LangGraph agents. They will need to wrap their agent before the build path produces value. +- **Execution compatibility.** New Platform configs must select a supported + harness. Existing NAT workflows may remain on the NAT compatibility path. + For another framework or an arbitrary Python entrypoint, inspect whether a + supported harness owns its lifecycle; otherwise identify a custom adapter or + NAT wrapper instead of claiming direct support. diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/SKILL.md index 42612124b50..970c5d1b2ea 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/SKILL.md @@ -23,7 +23,8 @@ allowed-tools: [Read, Write, Edit, Bash] # NeMo Platform agent spec Turn the answers from `nemo-explore` into a durable artifact. The spec is -the contract `nemo-build-agent` reads to scaffold the NAT workflow YAML and +the contract `nemo-build-agent` reads before producing the Platform-owned +`agent.yaml` or preserving an existing NAT compatibility workflow, and the `AGENT-SPEC.md` that downstream optimization agents read as their primary context. Without it, downstream skills have to re-ask everything and the optimization loop has no contract for what the agent is @@ -34,12 +35,13 @@ supposed to do or what may be changed. Two copies of the spec exist intentionally: * **Canonical**: a NeMo Filesets fileset named `-spec` in the - active workspace, holding a single file `AGENT-SPEC.md`. Downstream - optimization services read this copy server-side; the platform stores it - durably. -* **Local cache**: `agents/-spec/AGENT-SPEC.md` in the developer's - working directory. Hand-editable, version-controlled with the agent's repo, - used by this skill and by `nemo-build-agent`. + active workspace. It holds `AGENT-SPEC.md` and may also hold `agent.yaml` + plus relative artifacts used by the executable agent package. Downstream + services derive the relevant file ref from workspace and agent name. +* **Local cache**: `agents/-spec/` in the developer's working directory. + `AGENT-SPEC.md` is the human-readable contract; `agent.yaml` is the optional + machine-readable Platform config created by `nemo-agent-config` during the + build path. The Fileset wins on conflict. If a developer edits the local file, this skill re-uploads to refresh the Fileset. If the platform copy has drifted @@ -64,10 +66,10 @@ default. 1. **Role** — one concrete sentence describing the role this agent plays. Vague answers ("help with stuff", "answer questions") are rejected at write time by the `AgentSpec` validator and will fail the file write. -2. **Framework** — temporary NeMo Platform compatibility status, resolved to - one of `langgraph-nat` or `needs-wrapper` (with source-framework context - when `needs-wrapper`). The lightweight parser refuses unresolved framework - sections. +2. **Framework** — NeMo Platform execution compatibility, resolved to one of + `supported-harness`, `nat-workflow`, or `needs-adapter`. Include source + framework context when known. The lightweight parser refuses an empty or + unresolved framework section. The AGENTSpec parser (`nemo_agents_plugin.spec_parse.parse_spec`) enforces both at validation time; this skill enforces them upstream so the user sees a @@ -114,9 +116,10 @@ clear gap-question rather than a parser error. capability/source when they share credentials, side effects, freshness, and failure modes. Keep only details that change how downstream agents evaluate behavior. - - `Framework` should be binary: `langgraph-nat` or `needs-wrapper`, with - source-framework context only for `needs-wrapper`. Do not expand it into a - platform compatibility essay. + - `Framework` should use `supported-harness`, `nat-workflow`, or + `needs-adapter`. Record the source framework separately when known. Do not + infer execution compatibility from a framework import alone or expand the + section into a platform compatibility essay. - Avoid public shorthand like `AUT` or "agent under test." Use "this agent" for the agent being specified. Use "target agent" only when this agent's job is explicitly to inspect or modify another agent. @@ -169,9 +172,11 @@ clear gap-question rather than a parser error. 10. **Hand off.** Once confirmed, tell the user the next skill: - - `nemo-build-agent` will read `agents/-spec/AGENT-SPEC.md`, produce the - workflow YAML, and call `nemo agents create`. It does not need a - `--spec-file-ref` flag — the spec's location is derivable. + - `nemo-build-agent` will read `agents/-spec/AGENT-SPEC.md`, use + `nemo-agent-config` to produce `agent.yaml` by default, and call + `nemo agents create`. Existing NAT workflow YAML may remain on the + compatibility path. No `--spec-file-ref` flag is needed because the + spec's location is derivable. - The `eval-setup` skill (M2) will fill in the `Evaluation Setup` section when ready. - The insights plugin reads the same canonical fileset server-side once @@ -213,10 +218,10 @@ all print, and the user has confirmed the contents. ## What this skill is not -This skill does not produce NAT workflow YAML. The spec is the -human-readable design; the YAML is generated downstream by -`nemo-build-agent`. It also does not create the `Agent` entity on the -platform — that happens in `nemo-build-agent` via `nemo agents create`. +This skill does not produce `agent.yaml`, migrate NAT workflow YAML, or create +the `Agent` entity. The spec is the human-readable design. Machine-readable +config authoring belongs to `nemo-agent-config`, while registration and +deployment belong to `nemo-build-agent`. ## Gotchas @@ -224,9 +229,9 @@ platform — that happens in `nemo-build-agent` via `nemo agents create`. section headings intact. The parser in `nemo_agents_plugin.spec_parse` rejects missing or duplicate required sections, but section bodies remain markdown for humans and agents to read directly. -- **Spec lives next to the workflow YAML.** Local copies of both files end - up in `agents/`. Keep them adjacent so a future read of the directory - shows design and implementation together. +- **Spec lives next to the implementation config.** Keep `AGENT-SPEC.md`, + Platform `agent.yaml`, and their relative artifacts under + `agents/-spec/` so local and Filesets consumers share one package root. - **The Fileset is canonical, not the local file.** If the two disagree, the Fileset wins. Re-pull before editing if you suspect server-side drift. @@ -239,6 +244,10 @@ platform — that happens in `nemo-build-agent` via `nemo agents create`. `[a-z][a-z0-9-]*`. - **Role and Framework are hard requirements.** Do not write the spec with either missing. Route back to `nemo-explore` for the missing field only. +- **Framework is execution readiness, not a library label.** Use + `supported-harness` only when a supported harness can own the lifecycle, + `nat-workflow` for the existing NAT compatibility path, and `needs-adapter` + when no supported lifecycle contract exists. - **Purpose cannot be implementation-only by accident.** If goal context was not found in the codebase and the user did not provide outside context, make that provenance clear instead of letting implementation details masquerade as diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/references/templates/agent-spec.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/references/templates/agent-spec.md index baa405053f3..3393e82d254 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/references/templates/agent-spec.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-spec/references/templates/agent-spec.md @@ -67,12 +67,13 @@ bad agent behavior or a normal tool/source limitation.> ## Framework -- Resolution: -- Source framework: -- Notes: +- Resolution: +- Source framework: +- Notes: ## Harness +- Selection: - Description: - Agent loop: - Tool dispatch: diff --git a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-try-agent/SKILL.md b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-try-agent/SKILL.md index 228eb69cb2c..d26091eac7b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-try-agent/SKILL.md +++ b/sdk/python/nemo-platform/src/nemo_platform/skills/nemo-try-agent/SKILL.md @@ -1,19 +1,28 @@ --- name: nemo-try-agent -description: Sends a query to a deployed NeMo Platform agent (or falls back to direct model chat) and announces the routing decision before sending. Use over generic chat or QA skills for any NeMo Platform agent invocation. +description: Invokes an existing NeMo Platform agent through a named deployment or directly from a local agent YAML config. Use to try, test, or query an agent and inspect its response. triggers: + - nemo-try-agent - ask my agent + - ask my NeMo agent - try the agent - test it out + - test support agent with real question - query my agent - what does my agent say - send to the agent - try my nemo agent + - invoke deployed agent + - query running deployment + - invoke local agent config + - query local agent config + - invoke legacy NAT workflow + - test agent and show raw output not-for: - nemo-build-agent (use to deploy an agent before querying) - nemo-skill-selection (use to dispatch when intent is unclear) - nemo-status (use for read-only platform health) -compatibility: nemo-platform >= 0.1.0; running platform with at least one deployed agent (or a registered model for fallback); requires agents plugin; no destructive ops; safe under any sandbox. +compatibility: nemo-platform >= 0.1.0; requires agents plugin and either a local agent YAML config or a running platform with a deployed agent; no destructive ops; safe under any sandbox. maturity: active license: Apache-2.0 user-invocable: true @@ -22,57 +31,94 @@ allowed-tools: [Bash, Read] # NeMo Platform try-agent -Route a user query to a deployed agent or direct model. Announce the routing decision before sending. Never invoke silently. +Invoke an existing NeMo agent through a deployment or directly from a local YAML config. Announce the target before sending. Never invoke silently. ## Pre-flight -Confirm the platform is up and check what deployed agents exist before doing anything: +Choose the invocation mode from the user's target: + +- **Local one-shot:** the user provides an `agent.yaml` or legacy NAT workflow + YAML path. Read the config format and model settings before deciding whether + Platform readiness is required: + - A Platform-owned `nemo-agents-spec-v1` config invokes Fabric directly and + does not require Platform readiness. + - A legacy NAT config requires Platform readiness when any `openai` or `nim` + LLM omits `base_url`; local invocation injects the Platform IGW URL for + those entries. + - A legacy NAT config whose applicable LLMs all provide explicit `base_url` + values may invoke those endpoints directly without Platform readiness. +- **Deployed agent:** the user names a deployment or asks to use an already + deployed agent. Preserve the active CLI context and confirm the target + Platform is reachable by listing deployments: ```bash -# Ground truth: anything bound to :8080? +.venv/bin/nemo agents deployments list 2>/dev/null || { echo "PLATFORM_UNREACHABLE"; exit 1; } +``` + +When the user explicitly selects a local Platform, override any remote CLI +context for the current shell and add local process and health checks before +listing deployments: + +```bash +export NMP_BASE_URL=http://127.0.0.1:8080 lsof -iTCP:8080 -sTCP:LISTEN >/dev/null 2>&1 || { echo "PLATFORM_DOWN"; exit 1; } -# Functional check: platform readiness endpoint answers? -curl -sS --connect-timeout 2 --max-time 5 http://localhost:8080/health/ready -o /dev/null -w "%{http_code}\n" 2>/dev/null | grep -q "^200$" || { echo "PLATFORM_WEDGED"; exit 1; } +curl -sS --connect-timeout 2 --max-time 5 "$NMP_BASE_URL/health/ready" -o /dev/null -w "%{http_code}\n" 2>/dev/null | grep -q "^200$" || { echo "PLATFORM_WEDGED"; exit 1; } .venv/bin/nemo agents deployments list 2>/dev/null ``` -Do not use `nemo services status` for this check — it reports stale "running" from held locks after the process has died. +Do not use `nemo services status` for the local process check; it can report +stale "running" state from held locks after the process has died. -If `PLATFORM_DOWN`: route to `nemo-setup` and stop. If `PLATFORM_WEDGED`: route to `nemo-status` to surface the underlying error and stop. +Require these checks for a deployed invocation and for a local NAT invocation +that depends on injected Platform IGW routing. If `PLATFORM_UNREACHABLE` or +`PLATFORM_DOWN`, route to `nemo-setup` and stop. If `PLATFORM_WEDGED`, route to +`nemo-status` and stop. Do not require the checks for Platform-owned Fabric +local invocation or a NAT config with directly usable explicit endpoints. ## What you do 1. **Find the target.** - - One agent deployed: pick it as the target. - - Multiple agents deployed: ask the user which one. List names + statuses. - - No agents deployed: fall back to `nemo chat` against a model from `nemo models list`. Tell the user this is a model query, not an agent query. + - Local YAML path supplied: set `INVOCATION_MODE=local` and + `AGENT_CONFIG_PATH` to that config. + - Deployment named: confirm it is `running`, set + `INVOCATION_MODE=deployed`, and set `DEPLOYMENT_NAME` to its name. + - One running deployment and no target named: use it and set the deployed + mode variables above. + - Multiple running deployments: list their names, ask the user which one, + then set the deployed mode variables above. + - No running deployments: report that no deployed agent is available. Do not silently replace an agent invocation with `nemo chat`. 2. **Announce.** Say one of: - - "Sending to agent ``." - - "No agents deployed; sending to model `` via `nemo chat`." - - "Multiple agents deployed; which one: , ?" (then wait) + - "Invoking local agent config ``." + - "Sending to deployment ``." + - "Multiple deployments are running; which one: , ?" Then wait. 3. **Send the query.** ```bash -# Agent path -.venv/bin/nemo agents invoke --agent --input "" - -# Model fallback path -.venv/bin/nemo chat "" +if [ "$INVOCATION_MODE" = "local" ]; then + RESP=$(.venv/bin/nemo agents invoke \ + --agent-config "$AGENT_CONFIG_PATH" \ + --input "") +else + RESP=$(.venv/bin/nemo agents invoke \ + --agent-deployment "$DEPLOYMENT_NAME" \ + --input "") +fi +RC=$? ``` -4. **Show the verbatim response.** Code block, no paraphrase. If the agent used tool calls, list which tools and their outputs before the final answer. +1. **Show the verbatim response.** Print `RESP` in a code block without + paraphrasing it. If the agent used tool calls, list which tools and their + outputs before the final answer. -5. **Loop for follow-ups.** After the response, ask: "Another question, or done?" Keep the same target until the user changes it. +2. **Offer another invocation.** Keep the same target unless the user changes it. Do not claim that separate CLI invocations preserve a conversation session. ## Verification A "successful" invocation requires both: (a) the CLI returns exit code 0, and (b) the response body is non-empty. An empty body on a question the spec says the agent should handle is a quality signal, not a success. ```bash -RESP=$(.venv/bin/nemo agents invoke --agent --input "") -RC=$? if [ $RC -ne 0 ]; then echo "INVOKE_FAILED (exit $RC)" elif [ -z "$RESP" ]; then @@ -82,14 +128,19 @@ else fi ``` +Do not switch targets for verification. + If `INVOKE_FAILED` or `EMPTY_RESPONSE`: surface that to the user and stop. Do not claim the invocation succeeded. ## If verification fails | Symptom | Cause | Recovery | |---|---|---| -| 404 "agent not found" | Agent undeployed since last list | Re-run `.venv/bin/nemo agents deployments list`; ask user to pick from the new list | -| 5xx or platform error | Platform unhealthy | Route to `nemo-status` to surface the underlying error; offer to fall back to model chat | +| 404 "deployment not found" | Deployment was removed or the wrong name was used | Re-run `.venv/bin/nemo agents deployments list`; ask the user to pick from the new list | +| Deployment is not `running` | Deployment is still starting or failed | Inspect it with `.venv/bin/nemo agents deployments get `; do not invoke until it is running | +| Config validation error | Local YAML is invalid or references missing artifacts | Surface the validation details; route local YAML fixes or migrations to `nemo-agent-config`; use `nemo-build-agent` only when the user explicitly requests redeployment | +| Adapter or runtime error | Required harness package or runtime dependency is unavailable | Surface the structured invocation error and required dependency; do not substitute model chat | +| 5xx or platform error | Platform or deployed runtime is unhealthy | Route to `nemo-status` to surface the underlying error | | Empty response on a spec-handled question | Quality issue, not invocation issue | Stop and report; do not loop until the user decides next step | | "I cannot help" on every question | System prompt or tool wiring wrong in YAML | Route to `nemo-build-agent` to inspect and redeploy | | `agents plugin unavailable` | Plugin not installed | Route to `nemo-setup` Step 3 | @@ -97,6 +148,7 @@ If `INVOKE_FAILED` or `EMPTY_RESPONSE`: surface that to the user and stop. Do no ## Gotchas - **Routing must be explicit.** Silently picking a target and sending a query is the failure mode this skill exists to prevent. Announce first. -- **`nemo chat` and `nemo agents invoke` take different model id formats.** Chat uses entity-name (hyphens). Agents use whatever the YAML specifies. Pass through what the user says; do not auto-translate. -- **Use `curl` only for the pre-flight health probe.** The CLI is the documented interface for agent and model operations. Hand-rolled HTTP is not a substitute. -- **Loop in this skill, not in another.** Do not invoke `nemo-skill-selection` between turns. Stay here until the user asks to do something else. +- **Use an explicit deployment name.** `--agent-deployment` avoids ambiguity when one Agent entity has multiple deployments. +- **Keep agent and model chat distinct.** Offer `nemo chat` only when the user explicitly chooses a raw model query. +- **Use `curl` only for the pre-flight health probe.** The CLI is the documented interface for agent operations. Hand-rolled HTTP is not a substitute. +- **Do not promise session continuity.** Reusing a deployment target is not the same as resuming a specific multi-turn session. diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_skill_content.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_skill_content.py index 87b7fec7fd1..9a1eee1b02d 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_skill_content.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/skills/test_skill_content.py @@ -89,6 +89,19 @@ def test_each_skill_has_source_dir(self): assert skill.source_dir.is_dir() assert (skill.source_dir / "SKILL.md").exists() + def test_build_agent_templates_are_packaged(self): + skill = load_skills()["nemo-build-agent"] + assert skill.source_dir is not None + + templates_dir = skill.source_dir / "references" / "templates" + assert (templates_dir / "agent.yml").is_file() + assert (templates_dir / "eval-job.json").is_file() + + def test_model_selection_benchmark_cache_is_packaged(self): + skill = load_skills()["nemo-model-selection"] + assert skill.source_dir is not None + assert (skill.source_dir / "references" / "benchmark_cache.json").is_file() + def test_returns_new_dict_each_call(self): """Verify callers can't corrupt the cached data.""" skills1 = load_skills()