From e3c7ddc434876afcc18ba1b58f151c101aeede57 Mon Sep 17 00:00:00 2001 From: mschwab Date: Tue, 9 Jun 2026 15:36:14 -0700 Subject: [PATCH 1/9] fix(studio): wire evaluator plugin into OpenAPI gen + web SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nemo-evaluator plugin was missing [tool.nemo.openapi] so it was never discovered by script/generate-openapi-spec.sh, and therefore never contributed routes to a plugin openapi.yaml. Studio's evaluation UI references evaluator types that have been removed from the platform spec (PR #228 removed the legacy evaluator service). This PR establishes the generation pipeline so the evaluator plugin contributes its own SDK. Changes: - plugins/nemo-evaluator/pyproject.toml: add [tool.nemo.openapi] to opt in to OpenAPI spec generation - plugins/nemo-evaluator/openapi/openapi.yaml: generated evaluator plugin spec (EvaluateJob CRUD, health, hello routes) - web/packages/sdk/orval/constants.ts: add evaluator to serviceConfigs - web/packages/sdk/package.json: add gen:evaluator + gen:evaluator-zod scripts so gen:all generates the evaluator SDK Studio typecheck still fails because the evaluation UI references the old platform evaluator API (MetricEvaluationJob, BenchmarkEvaluationJob, useEvaluationListBenchmarks, etc.) which no longer exists — the new EvaluateJob API is a different design. A follow-up PR will migrate the evaluation UI to the new API. Signed-off-by: mschwab --- plugins/nemo-evaluator/openapi/openapi.yaml | 1446 +++++++++++++++++++ plugins/nemo-evaluator/pyproject.toml | 3 + web/packages/sdk/orval/constants.ts | 6 + web/packages/sdk/package.json | 2 + 4 files changed, 1457 insertions(+) create mode 100644 plugins/nemo-evaluator/openapi/openapi.yaml diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml new file mode 100644 index 00000000000..c1d4e98da5d --- /dev/null +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -0,0 +1,1446 @@ +openapi: 3.1.0 +info: + title: evaluator (plugin) + version: 0.1.3 +paths: + /apis/evaluator/v1/healthz: + get: + tags: + - Evaluator Plugin + summary: Healthz + operationId: healthz_apis_evaluator_v1_healthz_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + additionalProperties: true + type: object + title: Response Healthz Apis Evaluator V1 Healthz Get + /apis/evaluator/v1/hello/{name}: + get: + tags: + - Evaluator Plugin Hello Routes + summary: Hello + description: "Greet a name.\n\nThe greeting style is controlled by ``EvaluatorConfig.greeting_style``:\n\ + \n- ``\"formal\"`` (default) \u2192 ``\"Hello, {name}!\"``\n- ``\"casual\"\ + `` \u2192 ``\"Hey, {name}!\"``\n\nOverride at runtime: ``NMP_EVALUATOR_GREETING_STYLE=casual``." + operationId: hello_apis_evaluator_v1_hello__name__get + parameters: + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/HelloResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs: + post: + tags: + - Evaluator Plugin Jobs Routes + summary: Create Job + operationId: create_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJobRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Evaluator Plugin Jobs Routes + summary: List Jobs + operationId: list_jobs_apis_evaluator_v2_workspaces__workspace__evaluate_jobs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + exclusiveMinimum: 0 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + exclusiveMinimum: 0 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/EvaluateJobsSortField' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: -created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluateJobsListFilter' + description: Filter jobs on various criteria. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJobsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{job}/results/{name}: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job Result + operationId: get_job_result_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__job__results__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: job + in: path + required: true + schema: + type: string + title: Job + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{job}/results/{name}/download: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Download Job Result + operationId: download_job_result_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__job__results__name__download_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: job + in: path + required: true + schema: + type: string + title: Job + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/octet-stream: + schema: + type: string + format: binary + '404': + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job + operationId: get_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluator Plugin Jobs Routes + summary: Delete Job + operationId: delete_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/cancel: + post: + tags: + - Evaluator Plugin Jobs Routes + summary: Cancel Job + operationId: cancel_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__cancel_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/logs: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job Logs + operationId: get_job_logs_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__logs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: limit + in: query + required: false + schema: + title: Limit + type: integer + - name: page_cursor + in: query + required: false + schema: + title: Page Cursor + type: string + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobLogPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/results: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: List Job Results + operationId: list_job_results_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobListResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/status: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job Status + operationId: get_job_status_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__status_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobStatusResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' +components: + schemas: + Agent: + properties: + url: + type: string + title: Url + description: Base URL of the agent endpoint. + name: + type: string + title: Name + description: Agent name / identifier. + format: + type: string + enum: + - generic + - nemo_agent_toolkit + title: Format + description: Agent format that determines the execution path. + default: generic + api_key_secret: + allOf: + - $ref: '#/components/schemas/SecretRef' + description: 'API key secret reference for the agent. Format: workspace/secret_name + or secret_name within the job workspace.' + body: + title: Body + description: Jinja template for the request payload. Required for generic + agents. + additionalProperties: true + type: object + response_path: + title: Response Path + description: JSONPath expression to extract the response text from the agent's + response body. Required for generic agents. + type: string + trajectory_path: + title: Trajectory Path + description: JSONPath expression to extract the trajectory from the agent's + response body. Optional. + type: string + additionalProperties: false + type: object + required: + - url + - name + title: Agent + description: "Agent definition for inference in online evaluation jobs.\n\n\ + An agent is an endpoint that accepts a request and returns a response,\npotentially\ + \ with a trajectory. Two formats are supported:\n\n- ``generic``: configurable\ + \ HTTP POST with Jinja-templated body and\n JSONPath extraction for response\ + \ and trajectory.\n- ``nemo_agent_toolkit``: NeMo Agent Toolkit SSE streaming\ + \ protocol\n (``/generate/full?filter_steps=none``)." + BundledMetricOutputSpec: + properties: + name: + type: string + title: Name + description: + title: Description + type: string + value_json_schema: + additionalProperties: true + type: object + title: Value Json Schema + additionalProperties: false + type: object + required: + - name + - value_json_schema + title: BundledMetricOutputSpec + description: JSON-safe projection of a runtime metric output spec. + DatetimeFilter: + additionalProperties: false + properties: + $gte: + description: Filter for results greater than or equal to this datetime. + title: $Gte + format: date-time + type: string + $lte: + description: Filter for results less than or equal to this datetime. + title: $Lte + format: date-time + type: string + title: DatetimeFilter + type: object + EvaluateInputSpec: + properties: + metrics: + items: + $ref: '#/components/schemas/MetricBundleInput' + type: array + minItems: 1 + title: Metrics + description: Bundled metric entities to evaluate. + dataset: + anyOf: + - items: + additionalProperties: true + type: object + type: array + minItems: 1 + - $ref: '#/components/schemas/FilesetRef' + title: Dataset + description: Inline dataset rows or a persisted FilesetRef dataset source + to evaluate. + params: + anyOf: + - $ref: '#/components/schemas/RunConfig' + - $ref: '#/components/schemas/RunConfigOnline' + - $ref: '#/components/schemas/RunConfigOnlineModel' + title: Params + description: Optional evaluator SDK execution parameters. + target: + anyOf: + - $ref: '#/components/schemas/Model' + - $ref: '#/components/schemas/Agent' + title: Target + description: Optional model or agent target for online evaluation. + prompt_template: + anyOf: + - type: string + - additionalProperties: true + type: object + title: Prompt Template + description: Optional prompt template for online target generation. + field_mapping: + allOf: + - $ref: '#/components/schemas/FieldMapping' + description: Optional mapping from canonical evaluator fields to dataset + columns. + additionalProperties: false + type: object + required: + - metrics + - dataset + title: EvaluateInputSpec + description: Submitter-facing SDK evaluation input for the evaluator plugin + job. + EvaluateJob: + properties: + id: + title: Id + type: string + name: + type: string + title: Name + description: + title: Description + type: string + project: + title: Project + type: string + workspace: + title: Workspace + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + spec: + $ref: '#/components/schemas/EvaluateSpec' + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + title: Status Details + additionalProperties: true + type: object + error_details: + title: Error Details + additionalProperties: true + type: object + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object + type: object + required: + - name + - spec + title: EvaluateJob + EvaluateJobRequest: + properties: + name: + title: Name + type: string + description: + title: Description + type: string + project: + title: Project + type: string + spec: + $ref: '#/components/schemas/EvaluateInputSpec' + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object + type: object + required: + - spec + title: EvaluateJobRequest + EvaluateJobsListFilter: + additionalProperties: false + properties: + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs created at 'gte' datetime or 'lte' datetime. + name: + description: Name of the job. + title: Name + type: string + workspace: + description: Workspace of the job. + title: Workspace + type: string + project: + description: Project containing the job. + title: Project + type: string + status: + allOf: + - $ref: '#/components/schemas/PlatformJobStatus' + description: The current status. + updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs updated at 'gte' datetime or 'lte' datetime. + title: EvaluateJobsListFilter + type: object + EvaluateJobsPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluateJob' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluateJobsPage + EvaluateJobsSortField: + type: string + enum: + - created_at + - -created_at + - updated_at + - -updated_at + title: EvaluateJobsSortField + EvaluateSpec: + properties: + metrics: + items: + $ref: '#/components/schemas/MetricBundleOutput' + type: array + minItems: 1 + title: Metrics + description: Bundled metric entities to evaluate. + dataset: + anyOf: + - items: + additionalProperties: true + type: object + type: array + minItems: 1 + - $ref: '#/components/schemas/FilesetRef' + title: Dataset + description: Inline dataset rows or a persisted FilesetRef dataset source + to evaluate. + params: + anyOf: + - $ref: '#/components/schemas/RunConfig' + - $ref: '#/components/schemas/RunConfigOnline' + - $ref: '#/components/schemas/RunConfigOnlineModel' + title: Params + description: Optional evaluator SDK execution parameters. + target: + anyOf: + - $ref: '#/components/schemas/Model' + - $ref: '#/components/schemas/Agent' + title: Target + description: Optional model or agent target for online evaluation. + prompt_template: + anyOf: + - type: string + - additionalProperties: true + type: object + title: Prompt Template + description: Optional prompt template for online target generation. + field_mapping: + allOf: + - $ref: '#/components/schemas/FieldMapping' + description: Optional mapping from canonical evaluator fields to dataset + columns. + additionalProperties: false + type: object + required: + - metrics + - dataset + title: EvaluateSpec + description: Canonical SDK evaluation spec with platform model references resolved. + FieldMapping: + properties: + input: + title: Input + description: Binding for the canonical 'input' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + output: + title: Output + description: Binding for the canonical 'output' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + context: + title: Context + description: Binding for the canonical 'context' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + reference: + title: Reference + description: Binding for the canonical 'reference' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + trajectory: + title: Trajectory + description: Binding for the canonical 'trajectory' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + messages: + title: Messages + description: Binding for the canonical 'messages' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + tool_calls: + title: Tool Calls + description: Binding for the canonical 'tool_calls' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + tools: + title: Tools + description: Binding for the canonical 'tools' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + custom: + additionalProperties: + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + type: object + title: Custom + description: Additional evaluator field bindings keyed by canonical field + name. + additionalProperties: false + type: object + title: FieldMapping + description: 'Maps canonical evaluator fields to raw dataset column paths. + + Example: {''input'': ''question'', ''output'': ''answer'', ''reference'': + ''gold'', ''trajectory'': ''steps''}' + FileStorageType: + type: string + enum: + - fileset + title: FileStorageType + FilesetRef: + type: string + title: FilesetRef + description: Reference to a persisted platform fileset, optionally with a file + fragment. + HTTPValidationError: + properties: + detail: + items: + $ref: '#/components/schemas/ValidationError' + type: array + title: Detail + type: object + title: HTTPValidationError + HelloResponse: + properties: + message: + type: string + title: Message + type: object + required: + - message + title: HelloResponse + InferenceParams: + properties: + temperature: + title: Temperature + description: Float value between 0 and 1. temp of 0 indicates greedy decoding, + where the token with highest prob is chosen. Temperature can't be set + to 0.0 currently + type: number + maximum: 2.0 + minimum: 0.0 + max_tokens: + title: Max Tokens + description: Max tokens to generate + type: integer + minimum: 1.0 + max_completion_tokens: + title: Max Completion Tokens + description: Max tokens to generate + type: integer + minimum: 1.0 + top_p: + title: Top P + description: Float value between 0 and 1; limits to the top tokens within + a certain probability. top_p=0 means the model will only consider the + single most likely token for the next prediction + type: number + maximum: 1.0 + minimum: 0.0 + stop: + title: Stop + items: + type: string + type: array + additionalProperties: true + type: object + title: InferenceParams + description: Parameters for model inference. Extra fields can be supplied for + additional options applied to the inference request directly. Fields not supported + by the model may cause inference errors during evaluation. + MetricBundleInput: + properties: + bundle_kind: + type: string + const: metric-bundle + title: Bundle Kind + default: metric-bundle + bundle_format_version: + type: string + const: v1 + title: Bundle Format Version + default: v1 + metric_type: + type: string + minLength: 1 + title: Metric Type + metadata: + $ref: '#/components/schemas/MetricMetadata' + outputs: + items: + $ref: '#/components/schemas/BundledMetricOutputSpec' + type: array + minItems: 1 + title: Outputs + secrets: + additionalProperties: + $ref: '#/components/schemas/SecretRef' + type: object + title: Secrets + payload: + $ref: '#/components/schemas/MetricBundlePayload' + additionalProperties: false + type: object + required: + - metric_type + - outputs + - payload + title: MetricBundleInput + description: Standalone executable metric bundle entity used by backend execution. + MetricBundleOutput: + properties: + bundle_kind: + type: string + const: metric-bundle + title: Bundle Kind + default: metric-bundle + bundle_format_version: + type: string + const: v1 + title: Bundle Format Version + default: v1 + metric_type: + type: string + minLength: 1 + title: Metric Type + metadata: + $ref: '#/components/schemas/MetricMetadata' + outputs: + items: + $ref: '#/components/schemas/BundledMetricOutputSpec' + type: array + minItems: 1 + title: Outputs + secrets: + additionalProperties: + $ref: '#/components/schemas/SecretRef' + type: object + title: Secrets + payload: + additionalProperties: true + type: object + additionalProperties: false + type: object + required: + - metric_type + - outputs + - payload + title: MetricBundleOutput + description: Standalone executable metric bundle entity used by backend execution. + MetricBundlePayload: + properties: {} + type: object + title: MetricBundlePayload + description: Base class for concrete Pydantic metric bundle payloads. + MetricMetadata: + properties: + description: + title: Description + type: string + labels: + additionalProperties: + type: string + type: object + title: Labels + additionalProperties: true + type: object + title: MetricMetadata + description: User-facing metadata captured with a bundled metric. + Model: + properties: + url: + type: string + title: Url + description: URL of the model. + name: + type: string + title: Name + description: Name of the model. + api_key_secret: + allOf: + - $ref: '#/components/schemas/SecretRef' + description: 'API key secret reference for the model. Format: workspace/secret_name + or secret_name within the job workspace.' + format: + type: string + enum: + - nim + - openai + - llama_stack + title: Format + description: API format of the model. + default: nim + additionalProperties: false + type: object + required: + - url + - name + title: Model + description: Model definition for use without persisting to the Models API. + PaginationData: + properties: + page: + type: integer + title: Page + description: The current page number. + page_size: + type: integer + title: Page Size + description: The page size used for the query. + current_page_size: + type: integer + title: Current Page Size + description: The size for the current page. + total_pages: + type: integer + title: Total Pages + description: The total number of pages. + total_results: + type: integer + title: Total Results + description: The total number of results. + type: object + required: + - page + - page_size + - current_page_size + - total_pages + - total_results + title: PaginationData + PlatformJobListResultResponse: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobResultResponse' + type: array + title: Data + type: object + required: + - data + title: PlatformJobListResultResponse + PlatformJobLog: + properties: + timestamp: + type: string + format: date-time + title: Timestamp + job: + type: string + title: Job + job_step: + type: string + title: Job Step + job_task: + type: string + title: Job Task + message: + type: string + title: Message + type: object + required: + - timestamp + - job + - job_step + - job_task + - message + title: PlatformJobLog + PlatformJobLogPage: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobLog' + type: array + title: Data + total: + type: integer + title: Total + next_page: + title: Next Page + type: string + prev_page: + title: Prev Page + type: string + type: object + required: + - data + - total + - next_page + - prev_page + title: PlatformJobLogPage + PlatformJobResultResponse: + properties: + name: + type: string + title: Name + job: + type: string + title: Job + workspace: + type: string + title: Workspace + project: + title: Project + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + artifact_url: + type: string + title: Artifact Url + artifact_storage_type: + $ref: '#/components/schemas/FileStorageType' + download_url: + title: Download Url + type: string + type: object + required: + - name + - job + - workspace + - artifact_url + - artifact_storage_type + title: PlatformJobResultResponse + PlatformJobStatus: + type: string + enum: + - created + - pending + - active + - cancelled + - cancelling + - error + - completed + - paused + - pausing + - resuming + title: PlatformJobStatus + description: 'Enumeration of possible job statuses. + + + This enum represents the various states a job can be in during its lifecycle, + + from creation to a terminal state.' + PlatformJobStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + steps: + items: + $ref: '#/components/schemas/PlatformJobStepStatusResponse' + type: array + title: Steps + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - steps + - created_at + - updated_at + title: PlatformJobStatusResponse + PlatformJobStepStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + tasks: + items: + $ref: '#/components/schemas/PlatformJobTaskStatusResponse' + type: array + title: Tasks + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - tasks + - created_at + - updated_at + title: PlatformJobStepStatusResponse + PlatformJobTaskStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + error_stack: + title: Error Stack + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - error_stack + - created_at + - updated_at + title: PlatformJobTaskStatusResponse + ReasoningParams: + properties: + end_token: + title: End Token + description: 'Configure the end token to trim reasoning context based on + the model''s reasoning API. Example for Nemotron models: ''''' + type: string + include_if_not_finished: + title: Include If Not Finished + description: Configure whether to include reasoning context if the model + has not finished reasoning. + type: boolean + effort: + title: Effort + description: Option for OpenAI models to specify low, medium, or high reasoning + effort. + type: string + type: object + title: ReasoningParams + description: Custom settings that control the model's reasoning behavior. + RunConfig: + properties: + parallelism: + type: integer + minimum: 1.0 + title: Parallelism + description: Parallelism to be used for the evaluation job. Typically, this + represents the maximum number of concurrent requests made to the model. + default: 8 + limit_samples: + title: Limit Samples + description: Limit number of evaluation samples, taking the first `limit` + samples from the dataset. + type: integer + minimum: 1.0 + additionalProperties: false + type: object + title: RunConfig + description: Job parameters. + RunConfigOnline: + properties: + parallelism: + type: integer + minimum: 1.0 + title: Parallelism + description: Parallelism to be used for the evaluation job. Typically, this + represents the maximum number of concurrent requests made to the model. + default: 8 + limit_samples: + title: Limit Samples + description: Limit number of evaluation samples, taking the first `limit` + samples from the dataset. + type: integer + minimum: 1.0 + ignore_request_failure: + type: boolean + title: Ignore Request Failure + description: If True, request failures will be ignored and the result will + be marked as NaN. If False (default), request failures will raise an exception. + default: false + request_timeout: + title: Request Timeout + description: The timeout to be used for requests made to the model. + type: integer + max_retries: + type: integer + minimum: 0.0 + title: Max Retries + description: Maximum number of retries for failed requests. + default: 3 + additionalProperties: false + type: object + title: RunConfigOnline + description: Job parameters for online evaluation. + RunConfigOnlineModel: + properties: + parallelism: + type: integer + minimum: 1.0 + title: Parallelism + description: Parallelism to be used for the evaluation job. Typically, this + represents the maximum number of concurrent requests made to the model. + default: 8 + limit_samples: + title: Limit Samples + description: Limit number of evaluation samples, taking the first `limit` + samples from the dataset. + type: integer + minimum: 1.0 + ignore_request_failure: + type: boolean + title: Ignore Request Failure + description: If True, request failures will be ignored and the result will + be marked as NaN. If False (default), request failures will raise an exception. + default: false + request_timeout: + title: Request Timeout + description: The timeout to be used for requests made to the model. + type: integer + max_retries: + type: integer + minimum: 0.0 + title: Max Retries + description: Maximum number of retries for failed requests. + default: 3 + inference: + allOf: + - $ref: '#/components/schemas/InferenceParams' + description: Custom settings that control the model's text generation behavior. + system_prompt: + title: System Prompt + description: Initial instructions that define the model's role and behavior + for the conversation. + type: string + reasoning: + allOf: + - $ref: '#/components/schemas/ReasoningParams' + description: Custom settings that control the model's reasoning behavior. + structured_output: + title: Structured Output + description: JSON schema to apply structured output for the model. + additionalProperties: true + type: object + additionalProperties: false + type: object + title: RunConfigOnlineModel + description: Job parameters for model online evaluation. + SecretRef: + type: string + pattern: ^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)?$ + title: SecretRef + description: 'Reference to a platform secret or local environment variable. + Format: ''secret_name'' (uses request workspace) or ''workspace/secret_name'' + (explicit workspace).' + ValidationError: + properties: + loc: + items: + anyOf: + - type: string + - type: integer + type: array + title: Location + msg: + type: string + title: Message + type: + type: string + title: Error Type + input: + title: Input + ctx: + type: object + title: Context + additionalProperties: true + type: object + required: + - loc + - msg + - type + title: ValidationError diff --git a/plugins/nemo-evaluator/pyproject.toml b/plugins/nemo-evaluator/pyproject.toml index 4449858dfa0..be206e01ff7 100644 --- a/plugins/nemo-evaluator/pyproject.toml +++ b/plugins/nemo-evaluator/pyproject.toml @@ -51,5 +51,8 @@ asyncio_mode = "auto" pythonpath = ["src"] testpaths = ["tests"] +# Opt this plugin into OpenAPI spec generation +[tool.nemo.openapi] + [tool.pyright] extraPaths = ["src"] diff --git a/web/packages/sdk/orval/constants.ts b/web/packages/sdk/orval/constants.ts index 348799fb899..99d84f57bab 100644 --- a/web/packages/sdk/orval/constants.ts +++ b/web/packages/sdk/orval/constants.ts @@ -43,6 +43,12 @@ export const serviceConfigs: Record = { apiEnvKeys: ['VITE_PLATFORM_BASE_URL'], zod: true, }, + evaluator: { + path: 'evaluator', + url: `../../../../plugins/nemo-evaluator/openapi/openapi.yaml`, + apiEnvKeys: ['VITE_PLATFORM_BASE_URL'], + zod: true, + }, }; export const serviceToConfig = { diff --git a/web/packages/sdk/package.json b/web/packages/sdk/package.json index c035c62cb6d..92b24264c2b 100644 --- a/web/packages/sdk/package.json +++ b/web/packages/sdk/package.json @@ -14,6 +14,8 @@ "gen:customizer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts customizer", "gen:data-designer": "tsx ./orval/generate.ts data-designer", "gen:data-designer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts data-designer", + "gen:evaluator": "tsx ./orval/generate.ts evaluator", + "gen:evaluator-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts evaluator", "gen:deployment-management": "tsx ./orval/generate.ts deployment-management", "gen:entity-store": "tsx ./orval/generate.ts entity-store", "gen:evaluator": "tsx ./orval/generate.ts evaluator", From 396245ed06a1d38e6f2ea0b830f94b3d810a911e Mon Sep 17 00:00:00 2001 From: mschwab Date: Tue, 9 Jun 2026 15:57:54 -0700 Subject: [PATCH 2/9] fix(studio): migrate evaluation UI from legacy platform API to EvaluateJob The legacy platform evaluator service was removed in #228, deleting all metric/benchmark entity management APIs (MetricEvaluationJob, useEvaluationListMetricJobs, useEvaluationListBenchmarks, etc.) from the platform SDK. Studio's evaluation UI referenced these exclusively, breaking studio-ci web-typecheck on main. Migrated to the new nemo-evaluator plugin SDK (EvaluateJob-based API): - Replaced MetricEvaluationJob/BenchmarkEvaluationJob with EvaluateJob - Replaced useEvaluationList/Get/DeleteJob with useEvaluatorList/Get/DeleteEvaluateJob - Removed metric entity management routes (no equivalent in new API): EvaluationBenchmarksRoute, EvaluationMetricsRoute, EvaluationMetricCreate/DetailsRoute - Removed benchmark/metric entity data views and side panels - Kept and migrated: EvaluationResultsRoute, EvaluationResultDetailsRoute, ActionMenu, JobBulkDeleteModal, ComparisonPanel, DetailsPanel, evaluationJob selectors, util/evaluations Signed-off-by: mschwab --- .../src/api/evaluation/EvaluationApiError.ts | 20 +- .../details/useAggregatedEvaluations.tsx | 38 +- .../useEvaluationDownloadResultsById.tsx | 36 +- .../studio/src/api/evaluation/index.ts | 21 +- .../api/evaluation/useEvaluationsByConfig.ts | 13 +- .../evaluation/useEvaluationsWithMetrics.ts | 30 +- .../index.spec.tsx | 116 ------ .../EvaluationBenchmarksDataView/index.tsx | 208 ----------- .../EvaluationBenchmarksDataView/types.ts | 12 - .../EvaluationMetricsDataView/index.spec.tsx | 115 ------ .../EvaluationMetricsDataView/index.tsx | 324 ---------------- .../EvaluationMetricsDataView/types.ts | 17 +- .../Configurations/LLMJudgeDisplay.tsx | 23 +- .../evaluation/Jobs/ActionMenu.spec.tsx | 35 +- .../components/evaluation/Jobs/ActionMenu.tsx | 31 +- .../evaluation/Jobs/ComparisonPanel.tsx | 90 ++++- .../evaluation/Jobs/DetailsPanel.tsx | 23 +- .../EvaluationJobBulkDeleteModal.spec.tsx | 44 +-- .../Jobs/EvaluationJobBulkDeleteModal.tsx | 4 +- .../evaluation/Jobs/ResultsPanel.tsx | 84 +++-- .../evaluation/Jobs/TestMetric/ResultsLog.tsx | 33 +- .../Jobs/form/MetricDetailsPanel.tsx | 18 - .../Jobs/form/MetricDetailsSection.spec.tsx | 39 -- .../Jobs/form/MetricDetailsSection.tsx | 32 -- .../Jobs/form/MetricFormPanels.spec.tsx | 187 ---------- .../evaluation/Jobs/form/MetricFormPanels.tsx | 129 ------- .../Jobs/form/MetricLLMJudgePanel.tsx | 100 ----- .../Jobs/form/MetricSelector.spec.tsx | 99 ----- .../evaluation/Jobs/form/MetricSelector.tsx | 204 ----------- .../Jobs/form/MetricTestPanel.spec.tsx | 331 ----------------- .../evaluation/Jobs/form/MetricTestPanel.tsx | 345 ------------------ .../evaluation/Jobs/form/ScoreDefinitions.tsx | 6 +- .../evaluation/Jobs/form/ScoreModal.tsx | 11 +- .../EvaluationMetricBulkDeleteModal.spec.tsx | 249 ------------- .../EvaluationMetricBulkDeleteModal.tsx | 65 ---- .../BenchmarkDetailsPanel/index.tsx | 92 ----- .../sidePanels/MetricDetailsPanel/index.tsx | 121 ------ .../MetricRunSidePanel.stories.tsx | 38 +- .../sidePanels/MetricRunSidePanel/index.tsx | 108 ++---- .../sidePanels/MetricRunSidePanel/utils.ts | 2 +- .../evaluation/useEvaluationJobResultV2.ts | 16 +- .../evaluation/useMetricPanelForm.spec.ts | 13 - .../hooks/evaluation/useMetricPanelForm.ts | 165 --------- .../useSelectedEvaluationConfigFromUrl.ts | 68 ---- .../src/hooks/useComparisonResults/index.tsx | 11 +- .../src/mocks/evaluation/v1/evaluations.ts | 31 +- web/packages/studio/src/mocks/handlers.ts | 65 +--- .../EvaluationBenchmarksRoute/index.tsx | 59 --- .../index.spec.tsx | 150 -------- .../EvaluationMetricCreateRoute/index.tsx | 96 ----- .../EvaluationMetricDetailsRoute/index.tsx | 219 ----------- .../EvaluationMetricsRoute/index.spec.tsx | 119 ------ .../EvaluationMetricsRoute/index.tsx | 70 ---- .../EvaluationResultDetailsRoute/index.tsx | 4 +- .../EvaluationResultsRoute/index.tsx | 4 +- web/packages/studio/src/routes/index.tsx | 57 +-- .../src/selectors/evaluationConfigMetrics.ts | 206 ----------- .../studio/src/selectors/evaluationJob.ts | 73 +--- .../studio/src/tests/evaluation.spec.tsx | 74 ---- web/packages/studio/src/util/evaluations.ts | 18 +- 60 files changed, 415 insertions(+), 4596 deletions(-) delete mode 100644 web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx delete mode 100644 web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx delete mode 100644 web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts delete mode 100644 web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx delete mode 100644 web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsPanel.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.spec.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.spec.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricTestPanel.spec.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Jobs/form/MetricTestPanel.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Metrics/EvaluationMetricBulkDeleteModal.spec.tsx delete mode 100644 web/packages/studio/src/components/evaluation/Metrics/EvaluationMetricBulkDeleteModal.tsx delete mode 100644 web/packages/studio/src/components/sidePanels/BenchmarkDetailsPanel/index.tsx delete mode 100644 web/packages/studio/src/components/sidePanels/MetricDetailsPanel/index.tsx delete mode 100644 web/packages/studio/src/hooks/evaluation/useMetricPanelForm.spec.ts delete mode 100644 web/packages/studio/src/hooks/evaluation/useMetricPanelForm.ts delete mode 100644 web/packages/studio/src/hooks/evaluation/useSelectedEvaluationConfigFromUrl.ts delete mode 100644 web/packages/studio/src/routes/evaluation/EvaluationBenchmarksRoute/index.tsx delete mode 100644 web/packages/studio/src/routes/evaluation/EvaluationMetricCreateRoute/index.spec.tsx delete mode 100644 web/packages/studio/src/routes/evaluation/EvaluationMetricCreateRoute/index.tsx delete mode 100644 web/packages/studio/src/routes/evaluation/EvaluationMetricDetailsRoute/index.tsx delete mode 100644 web/packages/studio/src/routes/evaluation/EvaluationMetricsRoute/index.spec.tsx delete mode 100644 web/packages/studio/src/routes/evaluation/EvaluationMetricsRoute/index.tsx delete mode 100644 web/packages/studio/src/selectors/evaluationConfigMetrics.ts delete mode 100644 web/packages/studio/src/tests/evaluation.spec.tsx diff --git a/web/packages/studio/src/api/evaluation/EvaluationApiError.ts b/web/packages/studio/src/api/evaluation/EvaluationApiError.ts index c178aa68e5e..a93d9db4abe 100644 --- a/web/packages/studio/src/api/evaluation/EvaluationApiError.ts +++ b/web/packages/studio/src/api/evaluation/EvaluationApiError.ts @@ -1,23 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { ErrorResponse, HTTPValidationError } from '@nemo/sdk/generated/platform/schema'; - -const isHttpValidationError = ( - response: HTTPValidationError | ErrorResponse -): response is HTTPValidationError => { - return 'detail' in response && Array.isArray(response.detail); -}; +import type { HTTPValidationError } from '@nemo/sdk/generated/evaluator/schema'; export class EvaluationApiError extends Error { - constructor(public response: ErrorResponse | HTTPValidationError) { - let message; - if (isHttpValidationError(response)) { - message = response.detail?.map((err: { msg?: string }) => err.msg).join(''); - } else { - // ErrorResponse has {detail: string} - message = response.detail; - } + constructor(public response: HTTPValidationError) { + const message = Array.isArray(response.detail) + ? response.detail.map((err: { msg?: string }) => err.msg).join('') + : response.detail; super(message || 'Unknown error'); } diff --git a/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx b/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx index e87f98833e7..a5f92135472 100644 --- a/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx +++ b/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx @@ -3,15 +3,15 @@ import { NamedEntity } from '@nemo/common/src/namedEntity'; import { - evaluationGetMetricJob, - evaluationListMetricJobsResults, - evaluationListMetricJobs, -} from '@nemo/sdk/generated/platform/api'; + evaluatorGetEvaluateJob, + evaluatorListEvaluateJobResults, + evaluatorListEvaluateJobs, +} from '@nemo/sdk/generated/evaluator/api'; import { - MetricEvaluationJob, - MetricEvaluationJobsPage, + EvaluateJob, + EvaluateJobsPage, PlatformJobListResultResponse, -} from '@nemo/sdk/generated/platform/schema'; +} from '@nemo/sdk/generated/evaluator/schema'; import { useDetailsChartsStore } from '@studio/api/evaluation/details/useDetailsChartsStore'; import { getEvaluationJobModel, isEvaluationJobSucceeded } from '@studio/selectors/evaluationJob'; import { useQuery } from '@tanstack/react-query'; @@ -23,24 +23,24 @@ import { useQuery } from '@tanstack/react-query'; * Example usage in a component: * fetchAllEvaluationsByConfig(workspace); * - * @returns {Object} Promise + * @returns {Object} Promise */ const fetchAllEvaluationsByConfig = async ( workspace: string | undefined -): Promise => { +): Promise => { if (!workspace) { return { data: [], }; } - let allEvaluations: MetricEvaluationJobsPage['data'] = []; + let allEvaluations: EvaluateJobsPage['data'] = []; let currentPage = 1; let totalPages = 1; // initialize to avoid infinite loop const pageSize = 100; while (currentPage <= totalPages) { - const response = await evaluationListMetricJobs(workspace, { + const response = await evaluatorListEvaluateJobs(workspace, { page_size: pageSize, page: currentPage, sort: '-created_at', @@ -77,11 +77,11 @@ const fetchAllEvaluationsByConfig = async ( * Example usage in a component: * useUniqueModelsByConfig(workspace, id); * - * @returns {Object} + * @returns {Object} */ export const useUniqueModelsByConfig = (workspace: string | undefined, evaluationId?: string) => { - return useQuery, Error>({ + return useQuery, Error>({ queryKey: ['uniqueModelsByConfig', workspace, evaluationId], queryFn: async () => { const evaluationsByConfig = await fetchAllEvaluationsByConfig(workspace); @@ -97,13 +97,13 @@ export const useUniqueModelsByConfig = (workspace: string | undefined, evaluatio if (modelName) { if (!acc[modelName]) { - acc[modelName] = [] as MetricEvaluationJob[]; + acc[modelName] = [] as EvaluateJob[]; } acc[modelName].push(job); } return acc; }, - {} as Record + {} as Record ); }, placeholderData: (prevData) => prevData ?? {}, @@ -130,7 +130,7 @@ export const useAggregatedEvaluationResults = (config: NamedEntity) => { } return await Promise.all( selectedEvaluations.map((evaluationId) => - evaluationListMetricJobsResults(config.workspace!, evaluationId) + evaluatorListEvaluateJobResults(config.workspace!, evaluationId) ) ); }, @@ -146,12 +146,12 @@ export const useAggregatedEvaluationResults = (config: NamedEntity) => { * Example usage in a component: * useAggregatedEvaluations(config); * - * @returns {Array} MetricEvaluationJob[] + * @returns {Array} EvaluateJob[] */ export const useAggregatedEvaluations = (config: NamedEntity) => { const { selectedEvaluations } = useDetailsChartsStore(); - return useQuery({ + return useQuery({ queryKey: ['aggregatedEvaluations', config, selectedEvaluations], queryFn: async () => { if (!config.workspace) { @@ -159,7 +159,7 @@ export const useAggregatedEvaluations = (config: NamedEntity) => { } return await Promise.all( selectedEvaluations.map((evaluationId) => - evaluationGetMetricJob(config.workspace!, evaluationId) + evaluatorGetEvaluateJob(config.workspace!, evaluationId) ) ); }, diff --git a/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx b/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx index 2c6e2d30a60..9129ac79d77 100644 --- a/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx +++ b/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { ExcludedChatCompletionMessageParam } from '@nemo/common/src/types/chat'; -import { evaluationDownloadMetricJobResultRowScores } from '@nemo/sdk/generated/platform/api'; +import { evaluatorDownloadEvaluateJobResult } from '@nemo/sdk/generated/evaluator/api'; import { useQuery } from '@tanstack/react-query'; export interface EvaluationMetrics { @@ -34,16 +34,14 @@ export interface EvaluationResultItem { metrics: EvaluationMetrics; } -// V2 row score shape from the platform API -interface V2RowScore { +interface RowScore { index: number; row: Record; scores?: Record; error?: string; } -// Map V2 row scores to the EvaluationResultItem format used by the UI -function mapV2RowScores(rows: V2RowScore[]): EvaluationResultItem[] { +function mapRowScores(rows: RowScore[]): EvaluationResultItem[] { return rows.map((row) => ({ item: { prompt: String(row.row?.prompt ?? row.row?.input ?? ''), @@ -55,19 +53,17 @@ function mapV2RowScores(rows: V2RowScore[]): EvaluationResultItem[] { sample: { output_text: String(row.row?.output ?? row.row?.output_text ?? ''), }, - metrics: mapV2Scores(row.scores), + metrics: mapScores(row.scores), })); } -function mapV2Scores(scores?: Record): EvaluationMetrics { +function mapScores(scores?: Record): EvaluationMetrics { if (!scores) return {}; const result: EvaluationMetrics = {}; - // Map V2 flat scores to the nested metrics structure the UI expects for (const [key, value] of Object.entries(scores)) { if (key === 'sentence' || key === 'corpus') { - // BLEU sub-scores if (!result.bleu) { result.bleu = { scores: {} }; } @@ -83,27 +79,31 @@ function mapV2Scores(scores?: Record): EvaluationMetrics { return result; } -async function fetchEvaluationResultsV2( +async function fetchEvaluationResults( workspace: string, - jobName: string + jobName: string, + name: string ): Promise { - const blob = await evaluationDownloadMetricJobResultRowScores(workspace, jobName); + const blob = await evaluatorDownloadEvaluateJobResult(workspace, jobName, name); const text = await blob.text(); const parsed = JSON.parse(text); - // The response is either an array of row scores or wrapped in a container - const rows: V2RowScore[] = Array.isArray(parsed) + const rows: RowScore[] = Array.isArray(parsed) ? parsed : (parsed.data ?? parsed.row_scores ?? []); - return mapV2RowScores(rows); + return mapRowScores(rows); } -export const useEvaluationDownloadResultsById = (workspace: string, jobName: string) => { +export const useEvaluationDownloadResultsById = ( + workspace: string, + jobName: string, + name = 'row_scores' +) => { return useQuery({ - queryKey: ['evaluationDownloadResults', workspace, jobName], - queryFn: () => fetchEvaluationResultsV2(workspace, jobName), + queryKey: ['evaluationDownloadResults', workspace, jobName, name], + queryFn: () => fetchEvaluationResults(workspace, jobName, name), enabled: !!workspace && !!jobName, }); }; diff --git a/web/packages/studio/src/api/evaluation/index.ts b/web/packages/studio/src/api/evaluation/index.ts index 30f3d69d7b0..baa153d146e 100644 --- a/web/packages/studio/src/api/evaluation/index.ts +++ b/web/packages/studio/src/api/evaluation/index.ts @@ -1,12 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { evaluationListMetricJobs } from '@nemo/sdk/generated/platform/api'; -import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/schema'; -import { EvaluationJobWithTaskMetrics } from '@studio/api/evaluation/useEvaluationsWithMetrics'; +import { evaluatorListEvaluateJobs } from '@nemo/sdk/generated/evaluator/api'; +import type { EvaluatorListEvaluateJobsParams } from '@nemo/sdk/generated/evaluator/schema'; +import type { EvaluationJobWithTaskMetrics } from '@studio/api/evaluation/useEvaluationsWithMetrics'; export interface FetchEvaluationsWithMetricsOptions { workspace: string; - query?: EvaluationListMetricJobsParams; + query?: EvaluatorListEvaluateJobsParams; signal?: AbortSignal; } @@ -15,7 +15,7 @@ export const fetchEvaluationsWithMetrics = async ({ query, signal, }: FetchEvaluationsWithMetricsOptions) => { - const evaluations = await evaluationListMetricJobs(workspace, query, signal); + const evaluations = await evaluatorListEvaluateJobs(workspace, query, signal); if (evaluations.data?.length > 0) { const evaluationsWithMetrics = await Promise.all( evaluations.data.map(async (evaluation) => { @@ -28,19 +28,8 @@ export const fetchEvaluationsWithMetrics = async ({ } const evaluationWithMetrics: EvaluationJobWithTaskMetrics = { ...evaluation, tasks: {} }; - // TODO: Download results - // const evaluationResult = await getEvaluationResult( - // evaluation.workspace, - // evaluation.id!, - // evaluation.name, - // signal - // ); - // if (evaluationResult.download_url) { - // } return evaluationWithMetrics; } catch { - // if an eval job failed, there will be a 404 returned by fetchEvaluationResults - // return the original evaluation if there's an error return evaluation; } }) diff --git a/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts b/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts index 202a861ff90..c64dec2db47 100644 --- a/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts +++ b/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { NamedEntity } from '@nemo/common/src/namedEntity'; -import { useEvaluationListMetricJobs } from '@nemo/sdk/generated/platform/api'; -import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/schema'; +import { useEvaluatorListEvaluateJobs } from '@nemo/sdk/generated/evaluator/api'; +import type { EvaluatorListEvaluateJobsParams } from '@nemo/sdk/generated/evaluator/schema'; /** * Custom hook to fetch evaluation jobs filtered by configuration. @@ -19,15 +18,13 @@ import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/sch */ export const useEvaluationsByConfig = ( workspace: string, - config: NamedEntity & Partial + config: Partial ) => { - const params: EvaluationListMetricJobsParams = { + const params: EvaluatorListEvaluateJobsParams = { page: config.page, page_size: config.page_size, sort: 'created_at', - // TODO: Support filtering by config namespace and name - // config: config.namespace && config.name ? `${config.namespace}/${config.name}` : undefined, }; - return useEvaluationListMetricJobs(workspace, params); + return useEvaluatorListEvaluateJobs(workspace, params); }; diff --git a/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts b/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts index 60ee6a16f46..7ff3c4eb6aa 100644 --- a/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts +++ b/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts @@ -10,11 +10,11 @@ * its affiliates is strictly prohibited. */ import { - MetricEvaluationJob, - MetricEvaluationJobsPage, + EvaluateJob, + EvaluateJobsPage, PlatformJobStatus as JobStatus, - EvaluationListMetricJobsParams, -} from '@nemo/sdk/generated/platform/schema'; + EvaluatorListEvaluateJobsParams, +} from '@nemo/sdk/generated/evaluator/schema'; import { EvaluationApiError } from '@studio/api/evaluation/EvaluationApiError'; import { fetchEvaluationsWithMetrics } from '@studio/api/evaluation/index'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; @@ -23,32 +23,20 @@ import { UseQueryOptions, queryOptions, useQuery } from '@tanstack/react-query'; /** Task result shape appended by fetchEvaluationsWithMetrics. */ export type TaskResultOutput = Record; -export interface EvaluationJobWithTaskMetrics extends MetricEvaluationJob { +export interface EvaluationJobWithTaskMetrics extends EvaluateJob { tasks?: Record; } -export interface EvaluationJobsWithTaskMetricsPage extends Omit { +export interface EvaluationJobsWithTaskMetricsPage extends Omit { data: EvaluationJobWithTaskMetrics[]; } -/** - * these types are here to correctly overload our API types with a custom shape for this query - * GET /evaluations currently returns: - * { - * result: { // EvaluationJobSummary - * evaluations: [] // EvaluationConfig[] - * } - * } - * - * This custom query appends an array metrics to each EvaluationConfig object - */ - export interface UseEvaluationsWithMetricsOptions { queryOptions?: Omit< UseQueryOptions, 'queryFn' | 'queryKey' >; - query?: EvaluationListMetricJobsParams; + query?: EvaluatorListEvaluateJobsParams; filters?: { tags?: { projectId?: string; @@ -60,7 +48,7 @@ export interface UseEvaluationsWithMetricsOptions { export const getEvaluationsWithMetricsQueryOptions = ( workspace: string, - query?: EvaluationListMetricJobsParams + query?: EvaluatorListEvaluateJobsParams ) => { return queryOptions({ queryKey: ['evaluationsWithMetrics', query], @@ -75,7 +63,6 @@ export const useEvaluationsWithMetrics = (options?: UseEvaluationsWithMetricsOpt const { tags, status } = options.filters; let filter = ''; - // Add any tags to the filter query string const tagFilters = []; if (tags?.projectId) { tagFilters.push(tags.projectId); @@ -88,7 +75,6 @@ export const useEvaluationsWithMetrics = (options?: UseEvaluationsWithMetricsOpt filter = `tag:${tagFilterString}`; } - // Add status to the filter if (status) { const statusFilterString = `status:${status}`; filter = filter ? `${filter},${statusFilterString}` : statusFilterString; diff --git a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx b/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx deleted file mode 100644 index 86ea5de23a4..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { EvaluationBenchmarksDataView } from '@studio/components/dataViews/EvaluationBenchmarksDataView'; -import { PLATFORM_BASE_URL } from '@studio/constants/environment'; -import { server } from '@studio/mocks/node'; -import { renderRoute } from '@studio/tests/util/render'; -import { waitFor } from '@testing-library/react'; -import { http, HttpResponse } from 'msw'; - -vi.mock('use-debounce', () => ({ - useDebounce: (value: unknown) => [value, { cancel: () => {}, flush: () => {} }], -})); - -const BENCHMARKS_URL = `${PLATFORM_BASE_URL}/apis/evaluation/v2/workspaces/:workspace/benchmarks`; - -const emptyBenchmarksPage = { - data: [], - pagination: { - page: 1, - page_size: 50, - current_page_size: 0, - total_pages: 0, - total_results: 0, - }, -}; - -const captureRequests = () => { - const requestUrls: string[] = []; - server.use( - http.get(BENCHMARKS_URL, ({ request }) => { - requestUrls.push(request.url); - return HttpResponse.json(emptyBenchmarksPage); - }) - ); - return requestUrls; -}; - -const renderDataView = (initialEntry: string) => - renderRoute(, { - history: initialEntry, - routes: [ - { - path: '/workspaces/:workspace/evaluation/benchmarks', - element: , - }, - ], - }); - -describe('EvaluationBenchmarksDataView', () => { - afterEach(() => { - server.resetHandlers(); - }); - - it('sends no filter[ or search params on initial render', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/benchmarks'); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - const url = new URL(requestUrls.at(-1)!); - const keys = Array.from(url.searchParams.keys()); - expect(keys.some((k) => k.startsWith('filter['))).toBe(false); - expect(keys.some((k) => k === 'search' || k.startsWith('search['))).toBe(false); - }); - - it('sends filter[name][$like] when name search is active', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/benchmarks?s=foo'); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[name][$like]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[name][$like]')).toBe('foo'); - }); - - it('sends filter[created_at][$gte]/$lte when column filter is active', async () => { - const requestUrls = captureRequests(); - const gte = '2026-01-01T00:00:00.000Z'; - const lte = '2026-02-01T00:00:00.000Z'; - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: gte, $lte: lte } }]) - ); - renderDataView(`/workspaces/default/evaluation/benchmarks?filters=${filters}`); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[created_at][$gte]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[created_at][$gte]')).toBe(gte); - expect(url.searchParams.get('filter[created_at][$lte]')).toBe(lte); - }); - - it('never emits a top-level search or search[ key', async () => { - const requestUrls = captureRequests(); - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: '2026-01-01T00:00:00.000Z' } }]) - ); - renderDataView(`/workspaces/default/evaluation/benchmarks?s=foo&filters=${filters}`); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - for (const raw of requestUrls) { - const params = new URL(raw).searchParams; - expect(params.has('search')).toBe(false); - expect(Array.from(params.keys()).some((k) => k.startsWith('search['))).toBe(false); - } - }); -}); diff --git a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx deleted file mode 100644 index 7517991cc34..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { withOperators, type WithFilterOperators } from '@nemo/common/src/api/filterOperators'; -import { dateTimeFilter } from '@nemo/common/src/components/DataView/dateTimeFilter'; -import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; -import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; -import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; -import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; -import { getSortParam } from '@nemo/common/src/utils/query'; -import { useEvaluationListBenchmarks } from '@nemo/sdk/generated/platform/api'; -import type { - EvaluationListBenchmarksParams, - EvaluationListBenchmarksSort, -} from '@nemo/sdk/generated/platform/schema'; -import { Button, Stack, Text } from '@nvidia/foundations-react-core'; -import type { - BenchmarkItemWithId, - EvaluationBenchmarksDataViewProps, -} from '@studio/components/dataViews/EvaluationBenchmarksDataView/types'; -import { DocumentationButton } from '@studio/components/DocumentationButton'; -import { EMPTY_FIELD_EMDASH_VALUE } from '@studio/constants/constants'; -import { LINK_EVAL_DOCS_BENCHMARKS } from '@studio/constants/links'; -import { keepPreviousData } from '@tanstack/react-query'; -import { X } from 'lucide-react'; -import { type ComponentProps, type FC, useCallback, useMemo } from 'react'; - -export const EvaluationBenchmarksDataView: FC = ({ - workspace, - onRowClick, -}) => { - const dataViewState = useStudioDataViewState({ - defaultSort: { id: 'created_at', desc: true }, - }); - - const benchmarksFilter = useMemo(() => { - type BenchmarksFilter = NonNullable; - const filterObj: WithFilterOperators = {}; - - const nameFilter = dataViewState.apiFilter.searchText; - if (nameFilter) { - filterObj.name = { $like: nameFilter }; - } - - const createdAtFilter = dataViewState.apiFilter.filter?.created_at; - if ( - createdAtFilter && - typeof createdAtFilter === 'object' && - Object.keys(createdAtFilter).length > 0 - ) { - filterObj.created_at = createdAtFilter as BenchmarksFilter['created_at']; - } - - if (Object.keys(filterObj).length === 0) return undefined; - return withOperators(filterObj); - }, [dataViewState.apiFilter]); - - const { - data: benchmarksResponse, - isFetching, - refetch, - error, - } = useEvaluationListBenchmarks( - workspace, - { - page: dataViewState.pagination.state.pageIndex + 1, - page_size: dataViewState.pagination.state.pageSize, - sort: getSortParam(dataViewState.sorting.state) as EvaluationListBenchmarksSort, - filter: benchmarksFilter, - }, - { - query: { - placeholderData: keepPreviousData, - }, - } - ); - - const benchmarks = useMemo( - () => - (benchmarksResponse?.data || []).map((b) => ({ - ...b, - id: b.id ?? b.name ?? '', - })), - [benchmarksResponse?.data] - ); - - const hasActiveFilters = !!( - dataViewState.debouncedSearchBar || dataViewState.debouncedColumnFilters.length > 0 - ); - - const makeColumns: ComponentProps>['makeColumns'] = - useCallback( - ({ accessor }) => [ - accessor('name', { - header: 'Name', - enableSorting: true, - cell: ({ row }) => row.original.name ?? EMPTY_FIELD_EMDASH_VALUE, - }), - accessor( - (row) => - 'description' in row - ? (row.description ?? EMPTY_FIELD_EMDASH_VALUE) - : EMPTY_FIELD_EMDASH_VALUE, - { - id: 'description', - header: 'Description', - enableSorting: false, - size: 280, - } - ), - accessor( - (row) => ('metrics' in row && Array.isArray(row.metrics) ? row.metrics.length : null), - { - id: 'metrics_count', - header: 'Metrics', - enableSorting: false, - size: 100, - cell: ({ getValue }) => { - const count = getValue(); - return {count !== null ? count : EMPTY_FIELD_EMDASH_VALUE}; - }, - } - ), - accessor( - (row) => { - if (!('dataset' in row)) return '—'; - const { dataset } = row; - if (typeof dataset === 'string') return dataset; - if (dataset !== null && typeof dataset === 'object' && 'name' in dataset) { - return String((dataset as { name: string }).name); - } - return EMPTY_FIELD_EMDASH_VALUE; - }, - { - id: 'dataset', - header: 'Dataset', - enableSorting: false, - } - ), - accessor('created_at', { - id: 'created_at', - header: 'Created', - enableSorting: true, - size: 150, - meta: { - filter: dateTimeFilter('Created At'), - }, - cell: ({ row }) => { - const createdAt = 'created_at' in row.original ? row.original.created_at : undefined; - return createdAt ? : EMPTY_FIELD_EMDASH_VALUE; - }, - }), - ], - [] - ); - - return ( - - - dataViewState={dataViewState} - searchField="name" - makeColumns={makeColumns} - onRowClick={onRowClick} - attributes={{ - DataViewSearchBar: { - placeholder: 'Search benchmark name', - }, - DataViewRoot: { - data: benchmarks, - totalCount: benchmarksResponse?.pagination?.total_results, - requestStatus: error ? 'error' : isFetching ? 'loading' : undefined, - }, - DataViewTableContent: { - renderEmptyState: () => - hasActiveFilters ? ( - dataViewState.resetFilters()}> - Clear Filters - - } - /> - ) : ( - } - /> - ), - renderErrorState: () => ( - refetch()}> - Retry - - } - /> - ), - }, - }} - /> - - ); -}; diff --git a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts b/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts deleted file mode 100644 index 9a6dab046aa..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { BenchmarksListResponse } from '@nemo/sdk/generated/platform/schema'; - -export interface EvaluationBenchmarksDataViewProps { - workspace: string; - onRowClick?: (benchmark: BenchmarkItemWithId) => void; -} - -export type BenchmarkItem = BenchmarksListResponse['data'][number]; -export type BenchmarkItemWithId = BenchmarkItem & { id: string }; diff --git a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx deleted file mode 100644 index 1a93b50a4ab..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -import { EvaluationMetricsDataView } from '@studio/components/dataViews/EvaluationMetricsDataView'; -import { PLATFORM_BASE_URL } from '@studio/constants/environment'; -import { server } from '@studio/mocks/node'; -import { renderRoute } from '@studio/tests/util/render'; -import { waitFor } from '@testing-library/react'; -import { http, HttpResponse } from 'msw'; - -vi.mock('use-debounce', () => ({ - useDebounce: (value: unknown) => [value, { cancel: () => {}, flush: () => {} }], -})); - -const METRICS_URL = `${PLATFORM_BASE_URL}/apis/evaluation/v2/workspaces/:workspace/metrics`; - -const emptyMetricsPage = { - data: [], - pagination: { - page: 1, - page_size: 50, - current_page_size: 0, - total_pages: 0, - total_results: 0, - }, -}; - -const captureRequests = () => { - const requestUrls: string[] = []; - server.use( - http.get(METRICS_URL, ({ request }) => { - requestUrls.push(request.url); - return HttpResponse.json(emptyMetricsPage); - }) - ); - return requestUrls; -}; - -const renderDataView = (initialEntry: string) => - renderRoute(, { - history: initialEntry, - routes: [ - { - path: '/workspaces/:workspace/evaluation/metrics', - element: , - }, - ], - }); - -describe('EvaluationMetricsDataView', () => { - afterEach(() => { - server.resetHandlers(); - }); - - it('sends no filter[ or search params on initial render', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/metrics'); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - const url = new URL(requestUrls.at(-1)!); - const keys = Array.from(url.searchParams.keys()); - expect(keys.some((k) => k.startsWith('filter['))).toBe(false); - expect(keys.some((k) => k === 'search' || k.startsWith('search['))).toBe(false); - }); - - it('sends filter[name][$like] when name search is active', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/metrics?s=foo'); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[name][$like]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[name][$like]')).toBe('foo'); - }); - - it('sends filter[created_at][$gte]/$lte when column filter is active', async () => { - const requestUrls = captureRequests(); - const gte = '2026-01-01T00:00:00.000Z'; - const lte = '2026-02-01T00:00:00.000Z'; - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: gte, $lte: lte } }]) - ); - renderDataView(`/workspaces/default/evaluation/metrics?filters=${filters}`); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[created_at][$gte]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[created_at][$gte]')).toBe(gte); - expect(url.searchParams.get('filter[created_at][$lte]')).toBe(lte); - }); - - it('never emits a top-level search or search[ key', async () => { - const requestUrls = captureRequests(); - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: '2026-01-01T00:00:00.000Z' } }]) - ); - renderDataView(`/workspaces/default/evaluation/metrics?s=foo&filters=${filters}`); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - for (const raw of requestUrls) { - const params = new URL(raw).searchParams; - expect(params.has('search')).toBe(false); - expect(Array.from(params.keys()).some((k) => k.startsWith('search['))).toBe(false); - } - }); -}); diff --git a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx deleted file mode 100644 index 61716d05f53..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { withOperators, type WithFilterOperators } from '@nemo/common/src/api/filterOperators'; -import { dateTimeFilter } from '@nemo/common/src/components/DataView/dateTimeFilter'; -import { - ROW_ACTIONS_COLUMN_SIZE, - ROW_SELECTION_COLUMN_SIZE, - StudioDataView, -} from '@nemo/common/src/components/DataView/StudioDataView'; -import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; -import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; -import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; -import { getSortParam } from '@nemo/common/src/utils/query'; -import { - getEvaluationListMetricsQueryKey, - useEvaluationDeleteMetric, - useEvaluationListMetrics, -} from '@nemo/sdk/generated/platform/api'; -import type { - EvaluationListMetricsParams, - EvaluationListMetricsSort, -} from '@nemo/sdk/generated/platform/schema'; -import { Badge, Button, Flex, Stack, Text } from '@nvidia/foundations-react-core'; -import type { - EvaluationMetricsDataViewProps, - MetricItemWithId, -} from '@studio/components/dataViews/EvaluationMetricsDataView/types'; -import { DeleteConfirmationModal } from '@studio/components/DeleteConfirmationModal'; -import { EvaluationMetricBulkDeleteModal } from '@studio/components/evaluation/Metrics/EvaluationMetricBulkDeleteModal'; -import { - getEvaluationMetricDetailsRoute, - getEvaluationMetricRunRoute, - getNewEvaluationMetricRoute, -} from '@studio/routes/utils'; -import { keepPreviousData, useQueryClient } from '@tanstack/react-query'; -import { Play, Scale, X } from 'lucide-react'; -import { - type ComponentProps, - type FC, - type ReactNode, - useCallback, - useMemo, - useState, -} from 'react'; -import { useNavigate } from 'react-router-dom'; - -interface MetricTypeMeta { - label: string; - icon?: ReactNode; -} - -const METRIC_TYPE_META: Record = { - 'llm-judge': { label: 'LLM Judge', icon: }, -}; - -const getMetricTypeMeta = (type: string): MetricTypeMeta => - METRIC_TYPE_META[type] ?? { - label: type - .split('-') - .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) - .join(' '), - }; - -export const EvaluationMetricsDataView: FC = ({ - workspace, - onRowClick, -}) => { - const queryClient = useQueryClient(); - const navigate = useNavigate(); - const [metricToDelete, setMetricToDelete] = useState(null); - const { mutateAsync: deleteMetric } = useEvaluationDeleteMetric(); - const dataViewState = useStudioDataViewState({ - defaultSort: { id: 'created_at', desc: true }, - }); - - // Build filter object from the dataview filter state - const metricsFilter = useMemo(() => { - type MetricsFilter = NonNullable; - const filterObj: WithFilterOperators = {}; - - const nameFilter = dataViewState.apiFilter.searchText; - if (nameFilter) { - filterObj.name = { $like: nameFilter }; - } - - const createdAtFilter = dataViewState.apiFilter.filter?.created_at; - if ( - createdAtFilter && - typeof createdAtFilter === 'object' && - Object.keys(createdAtFilter).length > 0 - ) { - filterObj.created_at = createdAtFilter as MetricsFilter['created_at']; - } - - if (Object.keys(filterObj).length === 0) return undefined; - return withOperators(filterObj); - }, [dataViewState.apiFilter]); - - const { - data: evaluationsData, - isLoading, - refetch, - error, - } = useEvaluationListMetrics( - workspace, - { - page: dataViewState.pagination.state.pageIndex + 1, - page_size: dataViewState.pagination.state.pageSize, - sort: getSortParam(dataViewState.sorting.state) as EvaluationListMetricsSort, - filter: metricsFilter, - }, - { - query: { - refetchOnMount: 'always', - placeholderData: keepPreviousData, - }, - } - ); - - const evaluationMetrics = useMemo( - () => - (evaluationsData?.data || []).map((metric) => ({ - ...metric, - id: metric.id ?? metric.name ?? '', - })), - [evaluationsData?.data] - ); - - const invalidateMetrics = useCallback(() => { - queryClient.invalidateQueries({ - queryKey: getEvaluationListMetricsQueryKey(workspace), - }); - }, [queryClient, workspace]); - - const hasActiveFilters = !!( - dataViewState.debouncedSearchBar || dataViewState.debouncedColumnFilters.length > 0 - ); - - const makeColumns: ComponentProps>['makeColumns'] = - useCallback( - ({ accessor }, { rowSelectionColumn, rowActionsColumn }) => [ - rowSelectionColumn({ size: ROW_SELECTION_COLUMN_SIZE }), - accessor('name', { - header: 'Name', - enableSorting: false, - cell: ({ row }) => row.original.name ?? '-', - }), - accessor((row) => ('description' in row ? (row.description ?? '') : ''), { - id: 'description', - header: 'Description', - enableSorting: false, - cell: ({ row }) => { - const description = - 'description' in row.original ? row.original.description : undefined; - return description ? <>{description} : <>-; - }, - }), - accessor((row) => ('type' in row ? (row.type ?? '') : ''), { - id: 'type', - header: 'Type', - enableSorting: false, - size: 160, - cell: ({ row }) => { - const type = 'type' in row.original ? (row.original.type ?? null) : null; - if (!type) return <>-; - const meta = getMetricTypeMeta(type); - return ( - - {meta.icon} - {meta.label} - - ); - }, - }), - accessor('created_at', { - id: 'created_at', - header: 'Created', - enableSorting: true, - size: 150, - meta: { - filter: dateTimeFilter('Created At'), - }, - cell: ({ row }) => { - const createdAt = 'created_at' in row.original ? row.original.created_at : undefined; - return createdAt ? : -; - }, - }), - accessor(() => '', { - id: 'run', - header: '', - size: 152, - enableSorting: false, - cell: ({ row }) => ( - - - - ), - }), - rowActionsColumn({ - size: ROW_ACTIONS_COLUMN_SIZE, - cellProps: { - attributes: { - DropdownContent: { className: 'min-w-[140px]' }, - }, - }, - rowActions: (metric) => [ - { - children: 'View', - onSelect: () => { - if (metric.name) navigate(getEvaluationMetricDetailsRoute(workspace, metric.name)); - }, - }, - { - children: 'Delete', - danger: true, - onSelect: () => setMetricToDelete(metric), - }, - ], - }), - ], - [navigate, workspace] - ); - - const handleDeleteConfirm = async () => { - if (!metricToDelete?.name) return false; - try { - await deleteMetric({ workspace, name: metricToDelete.name }); - invalidateMetrics(); - return true; - } catch { - return false; - } - }; - - return ( - - - dataViewState={dataViewState} - searchField="name" - makeColumns={makeColumns} - onRowClick={onRowClick} - renderBulkActions={({ selectedRows }) => ( - { - invalidateMetrics(); - dataViewState.rowSelection.set({}); - }} - /> - )} - attributes={{ - DataViewSearchBar: { - placeholder: 'Search Metric Name', - }, - DataViewRoot: { - data: evaluationMetrics, - totalCount: evaluationsData?.pagination?.total_results, - requestStatus: error ? 'error' : isLoading ? 'loading' : undefined, - }, - DataViewTableContent: { - renderEmptyState: () => - hasActiveFilters ? ( - dataViewState.resetFilters()}> - Clear Filters - - } - /> - ) : ( - navigate(getNewEvaluationMetricRoute(workspace))} - > - New Metric - - } - /> - ), - renderErrorState: () => ( - refetch()}> - Retry - - } - /> - ), - }, - }} - /> - setMetricToDelete(null)} - onDelete={handleDeleteConfirm} - title="Delete Metric" - description={`Are you sure you want to delete "${metricToDelete?.name}"? This action cannot be undone.`} - simpleConfirm - successText="Successfully deleted evaluation metric" - errorText="Failed to delete evaluation metric. Please try again." - /> - - ); -}; diff --git a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts index 297e17ccd31..76fd02ca11c 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts +++ b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts @@ -1,22 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { DatetimeFilter, MetricsListResponse } from '@nemo/sdk/generated/platform/schema'; - -/** Filter fields for the metrics list API. */ -export interface EvaluationMetricsFilterFields { +/** A metric list item augmented with a stable `id` field for table row keying. */ +export interface MetricItemWithId { + id: string; name?: string; description?: string; type?: string; - project?: string; - created_at?: DatetimeFilter; - updated_at?: DatetimeFilter; + created_at?: string; + updated_at?: string; + workspace?: string; + [key: string]: unknown; } export interface EvaluationMetricsDataViewProps { workspace: string; onRowClick?: (metric: MetricItemWithId) => void; } - -export type MetricItem = MetricsListResponse['data'][number]; -export type MetricItemWithId = MetricItem & { id: string }; diff --git a/web/packages/studio/src/components/evaluation/Configurations/LLMJudgeDisplay.tsx b/web/packages/studio/src/components/evaluation/Configurations/LLMJudgeDisplay.tsx index b6f6294c210..8614e6e4d6c 100644 --- a/web/packages/studio/src/components/evaluation/Configurations/LLMJudgeDisplay.tsx +++ b/web/packages/studio/src/components/evaluation/Configurations/LLMJudgeDisplay.tsx @@ -4,15 +4,24 @@ import { Stack, Text } from '@nvidia/foundations-react-core'; import { Pre } from '@studio/components/common/Pre'; import { ReadOnlyField } from '@studio/components/common/ReadOnlyField'; -import { - getModelNameFromLLMJudgeParams, - getParserPatternFromLLMJudgeParams, - getScoreTypeFromLLMJudgeParams, - getSystemMessageFromLLMJudgeParams, - getUserMessageFromLLMJudgeParams, -} from '@studio/selectors/evaluationConfigMetrics'; import { FC } from 'react'; +const getStr = (params: Record | undefined, key: string): string | undefined => { + const val = params?.[key]; + return typeof val === 'string' ? val : undefined; +}; + +const getModelNameFromLLMJudgeParams = (params: Record | undefined) => + getStr(params, 'model'); +const getSystemMessageFromLLMJudgeParams = (params: Record | undefined) => + getStr(params, 'system_message'); +const getUserMessageFromLLMJudgeParams = (params: Record | undefined) => + getStr(params, 'user_message'); +const getScoreTypeFromLLMJudgeParams = (params: Record | undefined) => + getStr(params, 'score_type'); +const getParserPatternFromLLMJudgeParams = (params: Record | undefined) => + getStr(params, 'parser_pattern'); + export interface LLMJudgeDisplayProps { metricName: string; metricConfig: { type: string; params?: Record }; diff --git a/web/packages/studio/src/components/evaluation/Jobs/ActionMenu.spec.tsx b/web/packages/studio/src/components/evaluation/Jobs/ActionMenu.spec.tsx index 1a46e040709..dcb553d720f 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/ActionMenu.spec.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/ActionMenu.spec.tsx @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { MetricEvaluationJob, PlatformJobStatus } from '@nemo/sdk/generated/platform/schema'; +import { EvaluateJob, PlatformJobStatus } from '@nemo/sdk/generated/evaluator/schema'; import { ActionMenu } from '@studio/components/evaluation/Jobs/ActionMenu'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { mockUseParams } from '@studio/tests/util/mockUseParams'; @@ -10,37 +10,26 @@ import userEvent from '@testing-library/user-event'; const TEST_WORKSPACE = 'test-workspace'; -// Mock the delete APIs -const mockDeleteMetricJob = vi.fn(); -const mockDeleteBenchmarkJob = vi.fn(); -vi.mock('@nemo/sdk/generated/platform/api', async (importOriginal) => { +// Mock the delete API +const mockDeleteJob = vi.fn(); +vi.mock('@nemo/sdk/generated/evaluator/api', async (importOriginal) => { const original = await importOriginal(); return { // @ts-expect-error expect issue here with spread ...original, - useEvaluationDeleteMetricJob: vi.fn(() => ({ - mutateAsync: mockDeleteMetricJob, - })), - useEvaluationDeleteBenchmarkJob: vi.fn(() => ({ - mutateAsync: mockDeleteBenchmarkJob, + useEvaluatorDeleteEvaluateJob: vi.fn(() => ({ + mutateAsync: mockDeleteJob, })), }; }); describe('ActionMenu', () => { - const mockJob: MetricEvaluationJob = { + const mockJob: EvaluateJob = { id: 'test-job-1', name: 'test-job-1', status: PlatformJobStatus.completed, created_at: '2024-01-01T00:00:00Z', - spec: { - metric: 'test-namespace/test-metric', - dataset: 'default/test-dataset', - params: { - ignore_request_failure: false, - parallelism: 8, - }, - }, + spec: { metrics: [], dataset: [] }, }; const mockOnNavigateToDetails = vi.fn(); @@ -49,7 +38,7 @@ describe('ActionMenu', () => { beforeEach(() => { vi.clearAllMocks(); - mockDeleteMetricJob.mockResolvedValue(undefined); + mockDeleteJob.mockResolvedValue(undefined); mockUseParams({ [ROUTE_PARAMS.workspace]: TEST_WORKSPACE, }); @@ -168,7 +157,7 @@ describe('ActionMenu', () => { const confirmButton = await screen.findByRole('button', { name: 'Delete' }); await user.click(confirmButton); - expect(mockDeleteMetricJob).toHaveBeenCalledWith({ + expect(mockDeleteJob).toHaveBeenCalledWith({ workspace: TEST_WORKSPACE, name: 'test-job-1', }); @@ -204,7 +193,7 @@ describe('ActionMenu', () => { it('should handle delete API errors gracefully', async () => { const user = userEvent.setup(); const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - mockDeleteMetricJob.mockRejectedValue(new Error('Delete failed')); + mockDeleteJob.mockRejectedValue(new Error('Delete failed')); render( { it('should not show modal when job ID is undefined', async () => { const user = userEvent.setup(); - const jobWithoutId = { ...mockJob, name: undefined } as unknown as MetricEvaluationJob; + const jobWithoutId = { ...mockJob, name: undefined } as unknown as EvaluateJob; render( { - return 'benchmark' in job.spec; -}; - interface ActionMenuProps { - job: MetricEvaluationJob | BenchmarkEvaluationJob; - onNavigateToDetails: (job: MetricEvaluationJob | BenchmarkEvaluationJob) => void; - onJobDeleted?: (job: MetricEvaluationJob | BenchmarkEvaluationJob) => void; + job: EvaluateJob; + onNavigateToDetails: (job: EvaluateJob) => void; + onJobDeleted?: (job: EvaluateJob) => void; } export const ActionMenu: FC = ({ job, onNavigateToDetails, onJobDeleted }) => { const [modalOpen, setModalOpen] = useState<'edit' | 'delete' | undefined>(undefined); const [isDeleting, setIsDeleting] = useState(false); const workspace = useWorkspaceFromPath(); - const { mutateAsync: deleteMetricJob } = useEvaluationDeleteMetricJob(); - const { mutateAsync: deleteBenchmarkJob } = useEvaluationDeleteBenchmarkJob(); + const { mutateAsync: deleteEvaluateJob } = useEvaluatorDeleteEvaluateJob(); const toast = useToast(); const jobName = getEvaluationJobName(job); @@ -53,11 +40,7 @@ export const ActionMenu: FC = ({ job, onNavigateToDetails, onJo if (!jobName) { throw new Error('Evaluation job name is undefined'); } - if (isBenchmarkJob(job)) { - await deleteBenchmarkJob({ workspace, name: jobName }); - } else { - await deleteMetricJob({ workspace, name: jobName }); - } + await deleteEvaluateJob({ workspace, name: jobName }); toast.success('Evaluation job deleted successfully'); onJobDeleted?.(job); handleModalClose(); diff --git a/web/packages/studio/src/components/evaluation/Jobs/ComparisonPanel.tsx b/web/packages/studio/src/components/evaluation/Jobs/ComparisonPanel.tsx index ae53c0d0005..20cb4ac02eb 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/ComparisonPanel.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/ComparisonPanel.tsx @@ -3,6 +3,9 @@ import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; import { renderMultipleSelectedValues } from '@nemo/common/src/utils/form'; +import { getColorsFromLength } from '@nemo/common/src/utils/formatters'; +import { useEvaluatorListEvaluateJobResults } from '@nemo/sdk/generated/evaluator/api'; +import type { EvaluateJob } from '@nemo/sdk/generated/evaluator/schema'; import { Flex, Panel, @@ -12,38 +15,92 @@ import { StatusMessage, Text, } from '@nvidia/foundations-react-core'; -import { useEvaluationJobResultV2 } from '@studio/hooks/evaluation/useEvaluationJobResultV2'; -import { useJobsAndResultsV2 } from '@studio/hooks/evaluation/useJobsAndResultsV2'; -import { type EvaluationJobV2 } from '@studio/selectors/evaluationJob'; import { prettifyName } from '@studio/util/evaluations'; +import { useQuery } from '@tanstack/react-query'; import { ChartLine, ChartNetwork, TriangleAlert } from 'lucide-react'; import { useState, ReactNode } from 'react'; -interface ComparisonPanelV2Props { - job: EvaluationJobV2; +interface AggregateScores { + scores: Record>; +} + +interface ComparisonPanelProps { + job: EvaluateJob; workspace: string; jobName: string; } -/** - * V2 Comparison panel that works with artifact-based results - */ -export const ComparisonPanel = ({ job, workspace, jobName }: ComparisonPanelV2Props) => { +export const ComparisonPanel = ({ job, workspace, jobName }: ComparisonPanelProps) => { const [filterMetrics, setFilterMetrics] = useState([]); const { status } = job ?? {}; const isPendingStatus = status === 'pending' || status === 'active'; - const { scores, isLoading: isPendingResult } = useEvaluationJobResultV2(workspace, jobName); + const { data: resultsPage, isLoading: isLoadingMetadata } = useEvaluatorListEvaluateJobResults( + workspace, + jobName, + { + query: { + enabled: !!workspace && !!jobName, + }, + } + ); + + const aggregateScoresResult = resultsPage?.data?.find((r) => r.name === 'aggregate-scores'); + + const { data: scores, isLoading: isLoadingScores } = useQuery({ + queryKey: [ + 'evaluation-job-result-scores', + workspace, + jobName, + aggregateScoresResult?.download_url, + ], + queryFn: async () => { + if (!aggregateScoresResult?.download_url) { + throw new Error('No download URL available'); + } + const response = await fetch(aggregateScoresResult.download_url); + if (!response.ok) { + throw new Error(`Failed to download results: ${response.statusText}`); + } + return response.json() as Promise; + }, + enabled: !!aggregateScoresResult?.download_url, + }); + + const isPendingResult = isLoadingMetadata || isLoadingScores; + + // Inline useJobsAndResultsV2 logic typed for EvaluateJob + const combined = [{ job, result: scores }]; + const colorList = getColorsFromLength(combined.length); + const allRowsWithColors = combined.map((row, idx) => ({ ...row, color: colorList[idx] })); + + const metricOpts = Array.from( + new Set( + combined.flatMap((row) => { + if (!row.result?.scores) return []; + return Object.keys(row.result.scores).map((metricName) => prettifyName(metricName)); + }) + ) + ); + + const metricsToShow = metricOpts.filter((opt) => !filterMetrics.includes(opt)); - // For now, just show single job - can add comparison features later - const { rows, allMetrics, activeMetrics } = useJobsAndResultsV2({ - jobs: [job], - results: [scores], - filter: { metrics: filterMetrics }, - prettifyName, + const rows = allRowsWithColors.map((row) => { + if (!row.result?.scores) return row; + const filteredScores: Record> = {}; + Object.entries(row.result.scores).forEach(([metricName, metricScores]) => { + const prettified = prettifyName(metricName); + if (metricsToShow.includes(prettified)) { + filteredScores[prettified] = metricScores; + } + }); + return { ...row, result: { scores: filteredScores } }; }); + const allMetrics = metricOpts; + const activeMetrics = metricsToShow; + const handleFilterMetrics = (remaining: string[]) => { setFilterMetrics(allMetrics.filter((opt) => !remaining.includes(opt))); }; @@ -90,7 +147,6 @@ export const ComparisonPanel = ({ job, workspace, jobName }: ComparisonPanelV2Pr /> ); } else { - // Display metrics in a simple table format const row = rows[0]; content = ( diff --git a/web/packages/studio/src/components/evaluation/Jobs/DetailsPanel.tsx b/web/packages/studio/src/components/evaluation/Jobs/DetailsPanel.tsx index 13d374db525..1acc20bfbee 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/DetailsPanel.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/DetailsPanel.tsx @@ -16,35 +16,31 @@ import { utcToLocalDate, } from '@nemo/common/src/utils/date'; import { - getEvaluationGetMetricJobQueryKey, - useEvaluationCancelMetricJob, -} from '@nemo/sdk/generated/platform/api'; + getEvaluatorGetEvaluateJobQueryKey, + useEvaluatorCancelEvaluateJob, +} from '@nemo/sdk/generated/evaluator/api'; +import type { EvaluateJob } from '@nemo/sdk/generated/evaluator/schema'; import { Banner, Button, Flex, Modal, Panel, Stack, Text } from '@nvidia/foundations-react-core'; import { ButtonLaunchEvaluation } from '@studio/components/evaluation/ButtonLaunchEvaluation'; import { StatusLogsContent } from '@studio/components/evaluation/Jobs/StatusLogsContent'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { getEvaluationJobModel, type EvaluationJobV2 } from '@studio/selectors/evaluationJob'; import { useQueryClient } from '@tanstack/react-query'; import { ChartBar, LayoutList, CircleX } from 'lucide-react'; import { useState } from 'react'; import { useNavigate } from 'react-router-dom'; interface DetailsPanelProps { - evaluationJob?: EvaluationJobV2; + evaluationJob?: EvaluateJob; error?: boolean; } -/** - * Component to display the details of an evaluation job. - * Shows job ID, model, created time, configuration details, and status logs. - */ export const DetailsPanel = ({ evaluationJob, error }: DetailsPanelProps) => { const navigate = useNavigate(); const workspace = useWorkspaceFromPath(); const toast = useToast(); const queryClient = useQueryClient(); const [cancelModalOpen, setCancelModalOpen] = useState(false); - const { mutateAsync: cancelJob, isPending: isCancelling } = useEvaluationCancelMetricJob(); + const { mutateAsync: cancelJob, isPending: isCancelling } = useEvaluatorCancelEvaluateJob(); const handleRefreshClick = () => { navigate(0); @@ -57,9 +53,8 @@ export const DetailsPanel = ({ evaluationJob, error }: DetailsPanelProps) => { await cancelJob({ workspace, name: evaluationJob.name }); toast.success('Job cancellation requested'); - // Invalidate queries to refetch job status await queryClient.invalidateQueries({ - queryKey: getEvaluationGetMetricJobQueryKey(workspace, evaluationJob.name), + queryKey: getEvaluatorGetEvaluateJobQueryKey(workspace, evaluationJob.name), }); setCancelModalOpen(false); @@ -68,6 +63,7 @@ export const DetailsPanel = ({ evaluationJob, error }: DetailsPanelProps) => { toast.error('Failed to cancel job. Please try again.'); } }; + const differenceInMilliseconds = getDifferenceInMilliseconds( evaluationJob?.created_at, evaluationJob?.updated_at @@ -111,8 +107,7 @@ export const DetailsPanel = ({ evaluationJob, error }: DetailsPanelProps) => { | { message?: string; stage?: string; progress?: number } | undefined; - // Extract model from V2 spec - const model = getEvaluationJobModel(evaluationJob); + const model = evaluationJob.spec.target?.name; return ( <> diff --git a/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.spec.tsx b/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.spec.tsx index 34acc6a09b4..b6069ba6e1e 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.spec.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.spec.tsx @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { MetricEvaluationJob, PlatformJobStatus } from '@nemo/sdk/generated/platform/schema'; +import { EvaluateJob, PlatformJobStatus } from '@nemo/sdk/generated/evaluator/schema'; import { EvaluationJobBulkDeleteModal } from '@studio/components/evaluation/Jobs/EvaluationJobBulkDeleteModal'; import { ROUTE_PARAMS } from '@studio/constants/routes'; import { mockUseParams } from '@studio/tests/util/mockUseParams'; @@ -11,32 +11,28 @@ import { fireEvent, screen, waitFor, within } from '@testing-library/react'; const TEST_WORKSPACE = 'test-workspace'; // Mock the delete API -const mockDeleteMetricJob = vi.fn(); -vi.mock('@nemo/sdk/generated/platform/api', async (importOriginal) => { +const mockDeleteEvaluateJob = vi.fn(); +vi.mock('@nemo/sdk/generated/evaluator/api', async (importOriginal) => { const original = await importOriginal(); return { // @ts-expect-error expect issue here with spread ...original, - useEvaluationDeleteMetricJob: vi.fn(() => ({ - mutateAsync: mockDeleteMetricJob, + useEvaluatorDeleteEvaluateJob: vi.fn(() => ({ + mutateAsync: mockDeleteEvaluateJob, })), }; }); describe('EvaluationJobBulkDeleteModal', () => { - const mockJobs: MetricEvaluationJob[] = [ + const mockJobs: EvaluateJob[] = [ { id: 'job-1', name: 'job-1', status: PlatformJobStatus.completed, created_at: '2024-01-01T00:00:00Z', spec: { - metric: 'test-namespace/test-config-1', - dataset: 'default/test-dataset', - params: { - ignore_request_failure: false, - parallelism: 8, - }, + metrics: [], + dataset: [], }, custom_fields: {}, }, @@ -46,12 +42,8 @@ describe('EvaluationJobBulkDeleteModal', () => { status: PlatformJobStatus.active, created_at: '2024-01-02T00:00:00Z', spec: { - metric: 'test-namespace/test-config-2', - dataset: 'default/test-dataset', - params: { - ignore_request_failure: false, - parallelism: 8, - }, + metrics: [], + dataset: [], }, custom_fields: {}, }, @@ -61,7 +53,7 @@ describe('EvaluationJobBulkDeleteModal', () => { beforeEach(() => { vi.clearAllMocks(); - mockDeleteMetricJob.mockResolvedValue(undefined); + mockDeleteEvaluateJob.mockResolvedValue(undefined); mockUseParams({ [ROUTE_PARAMS.workspace]: TEST_WORKSPACE, }); @@ -157,16 +149,16 @@ describe('EvaluationJobBulkDeleteModal', () => { fireEvent.click(deleteButton); await waitFor(() => { - expect(mockDeleteMetricJob).toHaveBeenCalledTimes(2); + expect(mockDeleteEvaluateJob).toHaveBeenCalledTimes(2); }); await waitFor(() => { - expect(mockDeleteMetricJob).toHaveBeenCalledWith({ + expect(mockDeleteEvaluateJob).toHaveBeenCalledWith({ workspace: TEST_WORKSPACE, name: 'job-1', }); }); await waitFor(() => { - expect(mockDeleteMetricJob).toHaveBeenCalledWith({ + expect(mockDeleteEvaluateJob).toHaveBeenCalledWith({ workspace: TEST_WORKSPACE, name: 'job-2', }); @@ -183,7 +175,7 @@ describe('EvaluationJobBulkDeleteModal', () => { it('should handle deletion errors gracefully', async () => { const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - mockDeleteMetricJob.mockRejectedValue(new Error('Delete failed')); + mockDeleteEvaluateJob.mockRejectedValue(new Error('Delete failed')); render( { await waitFor(() => { // Should only call delete for jobs with valid names - expect(mockDeleteMetricJob).toHaveBeenCalledTimes(2); + expect(mockDeleteEvaluateJob).toHaveBeenCalledTimes(2); }); await waitFor(() => { - expect(mockDeleteMetricJob).toHaveBeenCalledWith({ + expect(mockDeleteEvaluateJob).toHaveBeenCalledWith({ workspace: TEST_WORKSPACE, name: 'job-2', }); }); await waitFor(() => { - expect(mockDeleteMetricJob).toHaveBeenCalledWith({ + expect(mockDeleteEvaluateJob).toHaveBeenCalledWith({ workspace: TEST_WORKSPACE, name: 'job-3', }); diff --git a/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.tsx b/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.tsx index 3ecc16a5b87..b37771abc27 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/EvaluationJobBulkDeleteModal.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { useEvaluationDeleteMetricJob } from '@nemo/sdk/generated/platform/api'; +import { useEvaluatorDeleteEvaluateJob } from '@nemo/sdk/generated/evaluator/api'; import { Button, Flex, Modal } from '@nvidia/foundations-react-core'; import { useMutateMany } from '@studio/api/common/useMutateMany'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; @@ -21,7 +21,7 @@ export const EvaluationJobBulkDeleteModal: FC const [open, setOpen] = useState(false); const workspace = useWorkspaceFromPath(); - const { mutateAsync: deleteJob } = useEvaluationDeleteMetricJob(); + const { mutateAsync: deleteJob } = useEvaluatorDeleteEvaluateJob(); const { mutateAsync: deleteJobs, isPending } = useMutateMany(deleteJob); const toast = useToast(); diff --git a/web/packages/studio/src/components/evaluation/Jobs/ResultsPanel.tsx b/web/packages/studio/src/components/evaluation/Jobs/ResultsPanel.tsx index 4c6bf5734f5..2c270bd70e1 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/ResultsPanel.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/ResultsPanel.tsx @@ -2,12 +2,17 @@ // SPDX-License-Identifier: Apache-2.0 import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { useEvaluationDownloadMetricJobResultAggregateScores } from '@nemo/sdk/generated/platform/api'; +import { useEvaluatorGetEvaluateJobResult } from '@nemo/sdk/generated/evaluator/api'; import { Panel, Spinner, Stack, Text, StatusMessage } from '@nvidia/foundations-react-core'; import { prettifyName } from '@studio/util/evaluations'; +import { useQuery } from '@tanstack/react-query'; import { ChartBar, TriangleAlert } from 'lucide-react'; import { type FC } from 'react'; +interface AggregateScores { + scores: Record>; +} + interface ResultsPanelProps { workspace: string; jobName: string; @@ -20,17 +25,43 @@ export const ResultsPanel: FC = ({ workspace, jobName, status const isPendingStatus = status === 'pending' || status === 'active'; const failedStatus = status === 'error' || status === 'cancelled'; + const enabled = !!workspace && !!jobName && !isPendingStatus && !failedStatus; + const { - data: scores, - isLoading, - error, - } = useEvaluationDownloadMetricJobResultAggregateScores(workspace, jobName, { + data: resultMetadata, + isLoading: isLoadingMetadata, + error: metadataError, + } = useEvaluatorGetEvaluateJobResult(workspace, jobName, 'aggregate-scores', { query: { - enabled: !!workspace && !!jobName && !isPendingStatus && !failedStatus, + enabled, retry: 3, }, }); + const { + data: scores, + isLoading: isLoadingScores, + error: scoresError, + } = useQuery({ + queryKey: ['results-panel-aggregate-scores', workspace, jobName, resultMetadata?.download_url], + queryFn: async () => { + if (!resultMetadata?.download_url) { + throw new Error('No download URL available'); + } + const response = await fetch(resultMetadata.download_url); + if (!response.ok) { + throw new Error(`Failed to download results: ${response.statusText}`); + } + const data: AggregateScores = await response.json(); + return data; + }, + enabled: !!resultMetadata?.download_url, + retry: 3, + }); + + const isLoading = isLoadingMetadata || isLoadingScores; + const error = metadataError || scoresError; + if (isPendingStatus) { return ( } slotHeading={EVALUATIONS_PANEL_HEADING}> @@ -77,7 +108,9 @@ export const ResultsPanel: FC = ({ workspace, jobName, status ); } - if (!scores?.scores?.length) { + const scoreEntries = scores?.scores ? Object.entries(scores.scores) : []; + + if (!scoreEntries.length) { return ( } slotHeading={EVALUATIONS_PANEL_HEADING}> = ({ workspace, jobName, status Aggregate Scores - {scores.scores.map((score) => ( -
- {score.name} + {scoreEntries.map(([scoreName, scoreValues]) => ( +
+ {scoreName} - {Object.entries(score) - .filter( - ([key, value]) => - key !== 'name' && - key !== 'score_type' && - (typeof value === 'number' || typeof value === 'string') - ) - .map(([key, value]) => ( -
- {prettifyName(key)}: - - {typeof value === 'number' - ? Number.isInteger(value) - ? value - : value.toFixed(4) - : value} - -
- ))} + {Object.entries(scoreValues).map(([key, value]) => ( +
+ {prettifyName(key)}: + + {typeof value === 'number' + ? Number.isInteger(value) + ? value + : value.toFixed(4) + : value} + +
+ ))}
))} diff --git a/web/packages/studio/src/components/evaluation/Jobs/TestMetric/ResultsLog.tsx b/web/packages/studio/src/components/evaluation/Jobs/TestMetric/ResultsLog.tsx index 49bcf93a51f..59f4f75450e 100644 --- a/web/packages/studio/src/components/evaluation/Jobs/TestMetric/ResultsLog.tsx +++ b/web/packages/studio/src/components/evaluation/Jobs/TestMetric/ResultsLog.tsx @@ -2,18 +2,33 @@ // SPDX-License-Identifier: Apache-2.0 import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import type { MetricEvaluationResponse } from '@nemo/sdk/generated/platform/schema'; import { Badge, CodeSnippet, Flex, Stack, Text } from '@nvidia/foundations-react-core'; import type { FC } from 'react'; +interface AggregateScore { + name: string; + count: number; + mean?: number | null; + min?: number | null; + max?: number | null; +} + +interface RowScore { + index: number; + scores?: Record; + error?: string | null; +} + +interface EvaluateJobResults { + aggregate_scores: AggregateScore[]; + row_scores: RowScore[]; +} + function formatScoreValue(value: number): string { return String(value); } -function buildRowResultLines( - rowScores: MetricEvaluationResponse['row_scores'], - scoreOrder: string[] -): string[] { +function buildRowResultLines(rowScores: RowScore[], scoreOrder: string[]): string[] { const lines: string[] = []; const sorted = [...rowScores].sort((a, b) => a.index - b.index); @@ -38,7 +53,7 @@ function buildRowResultLines( return lines; } -export const ResultsLog: FC<{ results: MetricEvaluationResponse }> = ({ results }) => { +export const ResultsLog: FC<{ results: EvaluateJobResults }> = ({ results }) => { const toast = useToast(); const { aggregate_scores, row_scores } = results; const scoreNames = aggregate_scores.map((s) => s.name); @@ -52,9 +67,9 @@ export const ResultsLog: FC<{ results: MetricEvaluationResponse }> = ({ results {aggregate_scores.map((agg) => ( - {agg.name}: mean={'mean' in agg && agg.mean != null ? agg.mean.toFixed(2) : 'N/A'}{' '} - min={'min' in agg && agg.min != null ? agg.min : 'N/A'} max= - {'max' in agg && agg.max != null ? agg.max : 'N/A'} (n={agg.count}) + {agg.name}: mean={agg.mean != null ? agg.mean.toFixed(2) : 'N/A'} min= + {agg.min != null ? agg.min : 'N/A'} max= + {agg.max != null ? agg.max : 'N/A'} (n={agg.count}) ))} diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsPanel.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsPanel.tsx deleted file mode 100644 index ae3a783a048..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsPanel.tsx +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { Divider, Stack } from '@nvidia/foundations-react-core'; -import { MetricDetailsSection } from '@studio/components/evaluation/Jobs/form/MetricDetailsSection'; -import { MetricScoreSection } from '@studio/components/evaluation/Jobs/form/MetricScoreSection'; -import { MetricTypeSection } from '@studio/components/evaluation/Jobs/form/MetricTypeSection'; -import type { FC } from 'react'; - -export const MetricDetailsPanel: FC = () => ( - - - - - - - -); diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.spec.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.spec.tsx deleted file mode 100644 index 79629c0d240..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.spec.tsx +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { MetricDetailsSection } from '@studio/components/evaluation/Jobs/form/MetricDetailsSection'; -import { renderRoute } from '@studio/tests/util/render'; -import { screen } from '@testing-library/react'; -import { FormProvider, useForm } from 'react-hook-form'; - -function renderSection() { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { name: '', body: { description: '' } }, - }); - return ( - - - - ); - }; - - return renderRoute(); -} - -describe('MetricDetailsSection', () => { - it('renders metric name and description fields', async () => { - renderSection(); - - expect(await screen.findByText('Metric Name')).toBeInTheDocument(); - expect(screen.getByText('Description (optional)')).toBeInTheDocument(); - }); - - it('renders an input for the metric name', async () => { - renderSection(); - - await screen.findByText('Metric Name'); - // KUI TextInput renders as a textbox role - expect(screen.getAllByRole('textbox').length).toBeGreaterThan(0); - }); -}); diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.tsx deleted file mode 100644 index 84d27210441..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricDetailsSection.tsx +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { ControlledTextArea } from '@nemo/common/src/components/form/ControlledTextArea'; -import { ControlledTextInput } from '@nemo/common/src/components/form/ControlledTextInput'; -import { Stack } from '@nvidia/foundations-react-core'; -import type { MetricPanelFormData } from '@studio/hooks/evaluation/useMetricPanelForm'; -import { formatWhitespaceHyphens } from '@studio/util/forms/transforms'; -import type { FC } from 'react'; -import { useFormContext } from 'react-hook-form'; - -export const MetricDetailsSection: FC = () => { - const { control, setValue } = useFormContext(); - return ( - - { - setValue('name', formatWhitespaceHyphens(e), { - shouldValidate: true, - }); - }} - /> - - - ); -}; diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.spec.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.spec.tsx deleted file mode 100644 index ac95f2fd2fc..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.spec.tsx +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { ModelEntity } from '@nemo/sdk/generated/platform/schema'; -import { MetricFormPanels } from '@studio/components/evaluation/Jobs/form/MetricFormPanels'; -import { DEFAULT_BUILD_MODEL_NAME } from '@studio/constants/constants'; -import { ROUTE_PARAMS } from '@studio/constants/routes'; -import { workspace1 } from '@studio/mocks/entity-store/projects'; -import { mockUseParams } from '@studio/tests/util/mockUseParams'; -import { renderRoute } from '@studio/tests/util/render'; -import { screen, waitFor } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; -import { useFormContext, useWatch } from 'react-hook-form'; - -const mockUseJudgeModels = vi.hoisted(() => - vi.fn(() => ({ - data: [] as ModelEntity[], - isLoading: false, - error: undefined as Error | undefined, - })) -); - -vi.mock('@studio/components/evaluation/JudgeModelSelect', () => ({ - JudgeModelSelect: ({ formFieldName }: { formFieldName: string }) => { - const { control, setValue } = useFormContext(); - const value = useWatch({ control, name: formFieldName as never }) as unknown as - | string - | undefined; - return ( -
- - setValue(formFieldName as never, e.target.value as never)} - /> -
- ); - }, -})); - -vi.mock('@studio/hooks/evaluation/useJudgeModels', () => ({ - useJudgeModels: mockUseJudgeModels, -})); - -vi.mock('@nemo/sdk/generated/platform/api', async (importOriginal) => { - const original = await importOriginal(); - return { - ...original, - useEvaluationEvaluateMetric: vi.fn(() => ({ - mutateAsync: vi.fn(), - isPending: false, - })), - }; -}); - -const TEST_FORM_ID = 'metric-form'; - -const renderPanel = (onSubmit = vi.fn()) => - renderRoute( - <> - - - - ); - -describe('MetricFormPanels', () => { - beforeEach(() => { - mockUseParams({ [ROUTE_PARAMS.workspace]: workspace1.name }); - mockUseJudgeModels.mockReturnValue({ - data: [], - isLoading: false, - error: undefined, - }); - }); - - describe('Left panel', () => { - it('renders metric name, description, metric type, and score sections', async () => { - renderPanel(); - - // KUI renders the label text in multiple elements; check at least one exists - expect((await screen.findAllByText('Metric Name')).length).toBeGreaterThan(0); - expect(screen.getByText('Metric Type')).toBeInTheDocument(); - expect(screen.getByText('Score Definitions')).toBeInTheDocument(); - expect(screen.getByText('LLM-as-a-Judge')).toBeInTheDocument(); - }); - }); - - describe('Default judge model', () => { - it('prefills judge model when the workspace lists the default build model', async () => { - const expectedUrn = `${workspace1.name}/${DEFAULT_BUILD_MODEL_NAME}`; - mockUseJudgeModels.mockReturnValue({ - data: [ - { - id: 'model-default-build', - workspace: workspace1.name, - name: DEFAULT_BUILD_MODEL_NAME, - created_at: '2024-01-01T00:00:00.000Z', - updated_at: '2024-01-01T00:00:00.000Z', - api_endpoint: { url: 'https://example.com/v1/chat/completions', format: 'nim' }, - }, - ], - isLoading: false, - error: undefined, - }); - - renderPanel(); - - await waitFor(() => { - expect(screen.getByTestId('judge-model-input')).toHaveValue(expectedUrn); - }); - }); - - it('does not prefill judge model when the default build model is not in the list', async () => { - mockUseJudgeModels.mockReturnValue({ - data: [ - { - id: 'model-other', - workspace: workspace1.name, - name: 'some-other-model', - created_at: '2024-01-01T00:00:00.000Z', - updated_at: '2024-01-01T00:00:00.000Z', - api_endpoint: { url: 'https://example.com/v1/chat/completions', format: 'nim' }, - }, - ], - isLoading: false, - error: undefined, - }); - - renderPanel(); - - await screen.findByTestId('judge-model-input'); - expect(screen.getByTestId('judge-model-input')).toHaveValue(''); - }); - }); - - describe('Right panel tabs', () => { - it('shows Configure tab content by default', async () => { - renderPanel(); - - // KUI SegmentedControl renders items as role="radio" inside a radiogroup - expect(await screen.findByRole('radio', { name: 'Configure' })).toBeInTheDocument(); - expect(screen.getByRole('radio', { name: 'Test' })).toBeInTheDocument(); - expect(screen.getByText('Model & Prompt')).toBeInTheDocument(); - }); - - it('switches to Test tab and shows test panel', async () => { - const user = userEvent.setup(); - renderPanel(); - - await screen.findByRole('radio', { name: 'Test' }); - await user.click(screen.getByRole('radio', { name: 'Test' })); - - await waitFor(() => { - expect(screen.getByText('Run Test')).toBeInTheDocument(); - }); - }); - }); - - it('requires confirmation again after editing a successfully tested metric', async () => { - const user = userEvent.setup(); - const onSubmit = vi.fn(); - renderPanel(onSubmit); - - await user.type(screen.getByRole('textbox', { name: 'Metric Name' }), 'response-quality'); - await user.type(await screen.findByTestId('judge-model-input'), 'test-model'); - await user.click(screen.getByRole('radio', { name: 'Test' })); - await user.click(await screen.findByRole('button', { name: 'Run Test' })); - - await user.click(screen.getByRole('button', { name: 'Save Metric' })); - - await waitFor(() => { - expect(onSubmit).toHaveBeenCalledTimes(1); - }); - expect(screen.queryByText('Save without Testing?')).not.toBeInTheDocument(); - - await user.click(screen.getByRole('radio', { name: 'Configure' })); - await user.type(screen.getByTestId('judge-model-input'), '-edited'); - await user.click(screen.getByRole('button', { name: 'Save Metric' })); - - expect(await screen.findByText('Save without Testing?')).toBeInTheDocument(); - expect(onSubmit).toHaveBeenCalledTimes(1); - }); -}); diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx deleted file mode 100644 index 78195840699..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { getEntityReference, getURNFromNamedEntityRef } from '@nemo/common/src/namedEntity'; -import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { ResizeablePanel } from '@studio/components/common/ResizeablePanel'; -import { MetricDetailsPanel } from '@studio/components/evaluation/Jobs/form/MetricDetailsPanel'; -import { MetricLLMJudgePanel } from '@studio/components/evaluation/Jobs/form/MetricLLMJudgePanel'; -import { ConfirmationModal } from '@studio/components/modals/ConfirmationModal'; -import { DEFAULT_BUILD_MODEL_NAME } from '@studio/constants/constants'; -import { useJudgeModels } from '@studio/hooks/evaluation/useJudgeModels'; -import { - type MetricPanelFormData, - useMetricPanelForm, -} from '@studio/hooks/evaluation/useMetricPanelForm'; -import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { handleFormErrorsGeneric } from '@studio/util/forms/error'; -import { type FC, useEffect, useMemo, useRef, useState } from 'react'; -import { type FieldErrors, FormProvider } from 'react-hook-form'; - -async function resolveSubmitSuccess( - result: boolean | Promise | void | Promise -): Promise { - const settled = await result; - return settled !== false; -} - -interface MetricFormPanelsProps { - /** Return false when the save failed after showing your own error UI; otherwise return true or void. */ - onSubmit: (data: MetricPanelFormData) => boolean | Promise | void | Promise; - formId?: string; -} - -export const MetricFormPanels: FC = ({ onSubmit, formId }) => { - const form = useMetricPanelForm(); - const workspace = useWorkspaceFromPath(); - const { data: judgeModels = [], isLoading: judgeModelsLoading } = useJudgeModels({ - enabled: !!workspace, - }); - const defaultJudgeModelUrn = useMemo(() => { - if (!workspace) return ''; - const candidate = getEntityReference({ workspace, name: DEFAULT_BUILD_MODEL_NAME }); - return judgeModels.some((m) => getURNFromNamedEntityRef(m) === candidate) ? candidate : ''; - }, [workspace, judgeModels]); - - const defaultJudgeModelAppliedRef = useRef(false); - const toast = useToast(); - const [hasSuccessfulTestRun, setHasSuccessfulTestRun] = useState(false); - const [saveWithoutTestOpen, setSaveWithoutTestOpen] = useState(false); - const [pendingSubmitData, setPendingSubmitData] = useState(null); - - const { handleSubmit, watch, getValues, setValue } = form; - - useEffect(() => { - if (judgeModelsLoading || !defaultJudgeModelUrn || defaultJudgeModelAppliedRef.current) { - return; - } - const currentName = getValues('body.model.name'); - if (!currentName) { - setValue('body.model.name', defaultJudgeModelUrn, { - shouldDirty: false, - shouldValidate: true, - }); - defaultJudgeModelAppliedRef.current = true; - } - }, [judgeModelsLoading, defaultJudgeModelUrn, getValues, setValue]); - - useEffect(() => { - const subscription = watch(() => { - setHasSuccessfulTestRun(false); - }); - return () => subscription.unsubscribe(); - }, [watch]); - - const onError = (errors: FieldErrors) => { - toast.error( - 'There was an error with the form submission. Please fix any errors and try again.' - ); - handleFormErrorsGeneric({ title: 'Evaluation Metric' })(errors); - }; - - const flushSubmit = async (data: MetricPanelFormData) => resolveSubmitSuccess(onSubmit(data)); - - const trySubmit = (data: MetricPanelFormData) => { - if (!hasSuccessfulTestRun) { - setPendingSubmitData(data); - setSaveWithoutTestOpen(true); - return; - } - void flushSubmit(data); - }; - - return ( - <> - -
- } - slotRight={ - setHasSuccessfulTestRun(true)} /> - } - /> - -
- { - setSaveWithoutTestOpen(false); - setPendingSubmitData(null); - }} - title="Save without Testing?" - description="You have not run a successful test from the Test tab. You can still save this metric; run a test first if you want to preview results with your sample data." - submitButtonText="Save" - onConfirm={async () => { - if (!pendingSubmitData) return false; - return flushSubmit(pendingSubmitData); - }} - suppressResultToasts - /> - - ); -}; diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx deleted file mode 100644 index 96f57f3424c..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { ChatCompletionInput } from '@nemo/common/src/components/ChatCompletionInput'; -import { SegmentedControl, Stack, Text } from '@nvidia/foundations-react-core'; -import { MetricTestPanel } from '@studio/components/evaluation/Jobs/form/MetricTestPanel'; -import { JudgeModelSelect } from '@studio/components/evaluation/JudgeModelSelect'; -import { mergeInferenceParamsFormValues } from '@studio/hooks/evaluation/useCreateConfigurationForm'; -import { type MetricPanelFormData } from '@studio/hooks/evaluation/useMetricPanelForm'; -import { type FC, useState } from 'react'; -import { useFieldArray, useFormContext } from 'react-hook-form'; - -type ActivePanel = 'configure' | 'test'; - -const SEGMENTED_CONTROL_ITEMS = [ - { value: 'configure', children: 'Configure' }, - { value: 'test', children: 'Test' }, -]; - -interface MetricRightPanelProps { - onSuccessfulTestRun: () => void; -} - -export const MetricLLMJudgePanel: FC = ({ onSuccessfulTestRun }) => { - const { control, watch, setValue } = useFormContext(); - const [activePanel, setActivePanel] = useState('configure'); - const { fields, append, remove, move } = useFieldArray({ control, name: 'body.messages' }); - const inferenceParams = watch('body.inference'); - - return ( - <> - setActivePanel(v as ActivePanel)} - items={SEGMENTED_CONTROL_ITEMS} - className="w-full shrink-0" - /> - - {/* - Keep both panels mounted so MetricTestPanel local state (sample data, file selection, - results) survives switching Configure ↔ Test. Use `hidden` instead of conditional render. - */} - - - Model & Prompt - - Provide instructions for the judge, with variables that get filled from your dataset. - Customize the judge prompt to match your evaluation criteria. - - - - - - formFieldName="body.model.name" - required - showParams - inferenceParams={inferenceParams} - onInferenceParamsChange={(params) => - setValue('body.inference', mergeInferenceParamsFormValues(inferenceParams, params)) - } - /> - - - {fields.map((field, index) => ( - - key={field.id} - control={control} - name={`body.messages.${index}`} - fieldArrayLength={fields.length} - onMoveUp={index > 0 ? () => move(index, index - 1) : undefined} - onMoveDown={index < fields.length - 1 ? () => move(index, index + 1) : undefined} - onDuplicate={() => - append( - { role: field.role, content: field.content, expanded: true }, - { shouldFocus: false } - ) - } - onRemove={() => remove(index)} - allowRemove={fields.length > 1} - /> - ))} - - - - -
- -
- - ); -}; diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx deleted file mode 100644 index ca58280ad08..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { MetricSelector } from '@studio/components/evaluation/Jobs/form/MetricSelector'; -import { render, screen } from '@testing-library/react'; -import { FormProvider, useForm } from 'react-hook-form'; - -function TestWrapper({ children }: { children: React.ReactNode }) { - const methods = useForm(); - return {children}; -} - -describe('MetricSelector', () => { - it('should render without crashing when no metric type is selected', () => { - const { container } = render( - - - - ); - - expect(container).toBeInTheDocument(); - }); - - it('should show LLM-as-a-Judge form fields when llm-judge type is selected', () => { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { - metricType: 'llm-judge', - metricConfig: {}, - }, - }); - return ( - - - - ); - }; - - render(); - - // Verify required form fields are displayed - expect(screen.getByLabelText(/Score Name/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Minimum Value/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Maximum Value/)).toBeInTheDocument(); - - // Verify optional fields - expect(screen.getByLabelText(/Score Description/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Prompt Template/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Parser Pattern/)).toBeInTheDocument(); - }); - - it('should display required field markers for LLM-as-a-Judge', () => { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { - metricType: 'llm-judge', - metricConfig: {}, - }, - }); - return ( - - - - ); - }; - - render(); - - // Verify required fields are marked via KUI's required class - const requiredFields = screen - .getAllByTestId('nv-form-field-root') - .filter((el) => el.classList.contains('nv-form-field-root--required')); - - // Should have at least 3 required fields: Score Name, Minimum, Maximum - expect(requiredFields.length).toBeGreaterThanOrEqual(3); - }); - - it('should show correct placeholder text for LLM judge fields', () => { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { - metricType: 'llm-judge', - metricConfig: {}, - }, - }); - return ( - - - - ); - }; - - render(); - - // Verify placeholder text - expect(screen.getByPlaceholderText('quality')).toBeInTheDocument(); - expect(screen.getByPlaceholderText('Overall quality of the response')).toBeInTheDocument(); - }); -}); diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx deleted file mode 100644 index a235800d9b6..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx +++ /dev/null @@ -1,204 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { - LLMJudgeMetric, - MetricType as SDKMetricType, -} from '@nemo/sdk/generated/platform/schema'; -import { FormField, Stack, TextArea, TextInput } from '@nvidia/foundations-react-core'; -import { FC } from 'react'; -import { Controller, useFormContext } from 'react-hook-form'; - -// Subset of metrics currently supported in the UI -export type MetricType = Extract; - -export type MetricConfig = LLMJudgeMetric; - -interface MetricSelectorProps { - workspace: string; -} - -/** - * Component for selecting and configuring evaluation metrics - */ -export const MetricSelector: FC = () => { - const { control, watch } = useFormContext(); - const metricType = watch('metricType') as MetricType | undefined; - - return ( - - {metricType === 'llm-judge' && ( - - ( - - {(props) => ( - - )} - - )} - /> - - ( - - {(props) => ( - - )} - - )} - /> - -
- ( - - {(props) => ( - field.onChange(v === '' ? undefined : Number(v))} - attributes={{ Input: { placeholder: '1' } }} - /> - )} - - )} - /> - - ( - - {(props) => ( - field.onChange(v === '' ? undefined : Number(v))} - attributes={{ Input: { placeholder: '5' } }} - /> - )} - - )} - /> -
- - ( - - Custom prompt for the judge. Use template variables like{' '} - {'{{input}}'} and {'{{output}}'}. Leave empty for - auto-generated prompt. - - } - > - {(props) => ( -