diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 95fd0b26815..67691a7f3c8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -531,12 +531,7 @@ jobs: web-studio-e2e: name: Studio UI E2E tests - needs: [changes] - if: > - !cancelled() && ( - github.event_name == 'workflow_dispatch' || - needs.changes.outputs.web-studio == 'true' - ) + if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest defaults: run: diff --git a/plugins/nemo-evaluator/openapi/openapi.yaml b/plugins/nemo-evaluator/openapi/openapi.yaml new file mode 100644 index 00000000000..c1d4e98da5d --- /dev/null +++ b/plugins/nemo-evaluator/openapi/openapi.yaml @@ -0,0 +1,1446 @@ +openapi: 3.1.0 +info: + title: evaluator (plugin) + version: 0.1.3 +paths: + /apis/evaluator/v1/healthz: + get: + tags: + - Evaluator Plugin + summary: Healthz + operationId: healthz_apis_evaluator_v1_healthz_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + additionalProperties: true + type: object + title: Response Healthz Apis Evaluator V1 Healthz Get + /apis/evaluator/v1/hello/{name}: + get: + tags: + - Evaluator Plugin Hello Routes + summary: Hello + description: "Greet a name.\n\nThe greeting style is controlled by ``EvaluatorConfig.greeting_style``:\n\ + \n- ``\"formal\"`` (default) \u2192 ``\"Hello, {name}!\"``\n- ``\"casual\"\ + `` \u2192 ``\"Hey, {name}!\"``\n\nOverride at runtime: ``NMP_EVALUATOR_GREETING_STYLE=casual``." + operationId: hello_apis_evaluator_v1_hello__name__get + parameters: + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/HelloResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs: + post: + tags: + - Evaluator Plugin Jobs Routes + summary: Create Job + operationId: create_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJobRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Evaluator Plugin Jobs Routes + summary: List Jobs + operationId: list_jobs_apis_evaluator_v2_workspaces__workspace__evaluate_jobs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + exclusiveMinimum: 0 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + exclusiveMinimum: 0 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/EvaluateJobsSortField' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: -created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/EvaluateJobsListFilter' + description: Filter jobs on various criteria. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJobsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{job}/results/{name}: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job Result + operationId: get_job_result_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__job__results__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: job + in: path + required: true + schema: + type: string + title: Job + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{job}/results/{name}/download: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Download Job Result + operationId: download_job_result_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__job__results__name__download_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: job + in: path + required: true + schema: + type: string + title: Job + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/octet-stream: + schema: + type: string + format: binary + '404': + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job + operationId: get_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Evaluator Plugin Jobs Routes + summary: Delete Job + operationId: delete_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/cancel: + post: + tags: + - Evaluator Plugin Jobs Routes + summary: Cancel Job + operationId: cancel_job_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__cancel_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/logs: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job Logs + operationId: get_job_logs_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__logs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: limit + in: query + required: false + schema: + title: Limit + type: integer + - name: page_cursor + in: query + required: false + schema: + title: Page Cursor + type: string + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobLogPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/results: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: List Job Results + operationId: list_job_results_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobListResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/evaluator/v2/workspaces/{workspace}/evaluate/jobs/{name}/status: + get: + tags: + - Evaluator Plugin Jobs Routes + summary: Get Job Status + operationId: get_job_status_apis_evaluator_v2_workspaces__workspace__evaluate_jobs__name__status_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobStatusResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' +components: + schemas: + Agent: + properties: + url: + type: string + title: Url + description: Base URL of the agent endpoint. + name: + type: string + title: Name + description: Agent name / identifier. + format: + type: string + enum: + - generic + - nemo_agent_toolkit + title: Format + description: Agent format that determines the execution path. + default: generic + api_key_secret: + allOf: + - $ref: '#/components/schemas/SecretRef' + description: 'API key secret reference for the agent. Format: workspace/secret_name + or secret_name within the job workspace.' + body: + title: Body + description: Jinja template for the request payload. Required for generic + agents. + additionalProperties: true + type: object + response_path: + title: Response Path + description: JSONPath expression to extract the response text from the agent's + response body. Required for generic agents. + type: string + trajectory_path: + title: Trajectory Path + description: JSONPath expression to extract the trajectory from the agent's + response body. Optional. + type: string + additionalProperties: false + type: object + required: + - url + - name + title: Agent + description: "Agent definition for inference in online evaluation jobs.\n\n\ + An agent is an endpoint that accepts a request and returns a response,\npotentially\ + \ with a trajectory. Two formats are supported:\n\n- ``generic``: configurable\ + \ HTTP POST with Jinja-templated body and\n JSONPath extraction for response\ + \ and trajectory.\n- ``nemo_agent_toolkit``: NeMo Agent Toolkit SSE streaming\ + \ protocol\n (``/generate/full?filter_steps=none``)." + BundledMetricOutputSpec: + properties: + name: + type: string + title: Name + description: + title: Description + type: string + value_json_schema: + additionalProperties: true + type: object + title: Value Json Schema + additionalProperties: false + type: object + required: + - name + - value_json_schema + title: BundledMetricOutputSpec + description: JSON-safe projection of a runtime metric output spec. + DatetimeFilter: + additionalProperties: false + properties: + $gte: + description: Filter for results greater than or equal to this datetime. + title: $Gte + format: date-time + type: string + $lte: + description: Filter for results less than or equal to this datetime. + title: $Lte + format: date-time + type: string + title: DatetimeFilter + type: object + EvaluateInputSpec: + properties: + metrics: + items: + $ref: '#/components/schemas/MetricBundleInput' + type: array + minItems: 1 + title: Metrics + description: Bundled metric entities to evaluate. + dataset: + anyOf: + - items: + additionalProperties: true + type: object + type: array + minItems: 1 + - $ref: '#/components/schemas/FilesetRef' + title: Dataset + description: Inline dataset rows or a persisted FilesetRef dataset source + to evaluate. + params: + anyOf: + - $ref: '#/components/schemas/RunConfig' + - $ref: '#/components/schemas/RunConfigOnline' + - $ref: '#/components/schemas/RunConfigOnlineModel' + title: Params + description: Optional evaluator SDK execution parameters. + target: + anyOf: + - $ref: '#/components/schemas/Model' + - $ref: '#/components/schemas/Agent' + title: Target + description: Optional model or agent target for online evaluation. + prompt_template: + anyOf: + - type: string + - additionalProperties: true + type: object + title: Prompt Template + description: Optional prompt template for online target generation. + field_mapping: + allOf: + - $ref: '#/components/schemas/FieldMapping' + description: Optional mapping from canonical evaluator fields to dataset + columns. + additionalProperties: false + type: object + required: + - metrics + - dataset + title: EvaluateInputSpec + description: Submitter-facing SDK evaluation input for the evaluator plugin + job. + EvaluateJob: + properties: + id: + title: Id + type: string + name: + type: string + title: Name + description: + title: Description + type: string + project: + title: Project + type: string + workspace: + title: Workspace + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + spec: + $ref: '#/components/schemas/EvaluateSpec' + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + title: Status Details + additionalProperties: true + type: object + error_details: + title: Error Details + additionalProperties: true + type: object + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object + type: object + required: + - name + - spec + title: EvaluateJob + EvaluateJobRequest: + properties: + name: + title: Name + type: string + description: + title: Description + type: string + project: + title: Project + type: string + spec: + $ref: '#/components/schemas/EvaluateInputSpec' + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object + type: object + required: + - spec + title: EvaluateJobRequest + EvaluateJobsListFilter: + additionalProperties: false + properties: + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs created at 'gte' datetime or 'lte' datetime. + name: + description: Name of the job. + title: Name + type: string + workspace: + description: Workspace of the job. + title: Workspace + type: string + project: + description: Project containing the job. + title: Project + type: string + status: + allOf: + - $ref: '#/components/schemas/PlatformJobStatus' + description: The current status. + updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs updated at 'gte' datetime or 'lte' datetime. + title: EvaluateJobsListFilter + type: object + EvaluateJobsPage: + properties: + data: + items: + $ref: '#/components/schemas/EvaluateJob' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: EvaluateJobsPage + EvaluateJobsSortField: + type: string + enum: + - created_at + - -created_at + - updated_at + - -updated_at + title: EvaluateJobsSortField + EvaluateSpec: + properties: + metrics: + items: + $ref: '#/components/schemas/MetricBundleOutput' + type: array + minItems: 1 + title: Metrics + description: Bundled metric entities to evaluate. + dataset: + anyOf: + - items: + additionalProperties: true + type: object + type: array + minItems: 1 + - $ref: '#/components/schemas/FilesetRef' + title: Dataset + description: Inline dataset rows or a persisted FilesetRef dataset source + to evaluate. + params: + anyOf: + - $ref: '#/components/schemas/RunConfig' + - $ref: '#/components/schemas/RunConfigOnline' + - $ref: '#/components/schemas/RunConfigOnlineModel' + title: Params + description: Optional evaluator SDK execution parameters. + target: + anyOf: + - $ref: '#/components/schemas/Model' + - $ref: '#/components/schemas/Agent' + title: Target + description: Optional model or agent target for online evaluation. + prompt_template: + anyOf: + - type: string + - additionalProperties: true + type: object + title: Prompt Template + description: Optional prompt template for online target generation. + field_mapping: + allOf: + - $ref: '#/components/schemas/FieldMapping' + description: Optional mapping from canonical evaluator fields to dataset + columns. + additionalProperties: false + type: object + required: + - metrics + - dataset + title: EvaluateSpec + description: Canonical SDK evaluation spec with platform model references resolved. + FieldMapping: + properties: + input: + title: Input + description: Binding for the canonical 'input' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + output: + title: Output + description: Binding for the canonical 'output' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + context: + title: Context + description: Binding for the canonical 'context' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + reference: + title: Reference + description: Binding for the canonical 'reference' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + trajectory: + title: Trajectory + description: Binding for the canonical 'trajectory' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + messages: + title: Messages + description: Binding for the canonical 'messages' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + tool_calls: + title: Tool Calls + description: Binding for the canonical 'tool_calls' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + tools: + title: Tools + description: Binding for the canonical 'tools' evaluator field. + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + custom: + additionalProperties: + type: string + minLength: 1 + pattern: ^[^\[\]]*$ + type: object + title: Custom + description: Additional evaluator field bindings keyed by canonical field + name. + additionalProperties: false + type: object + title: FieldMapping + description: 'Maps canonical evaluator fields to raw dataset column paths. + + Example: {''input'': ''question'', ''output'': ''answer'', ''reference'': + ''gold'', ''trajectory'': ''steps''}' + FileStorageType: + type: string + enum: + - fileset + title: FileStorageType + FilesetRef: + type: string + title: FilesetRef + description: Reference to a persisted platform fileset, optionally with a file + fragment. + HTTPValidationError: + properties: + detail: + items: + $ref: '#/components/schemas/ValidationError' + type: array + title: Detail + type: object + title: HTTPValidationError + HelloResponse: + properties: + message: + type: string + title: Message + type: object + required: + - message + title: HelloResponse + InferenceParams: + properties: + temperature: + title: Temperature + description: Float value between 0 and 1. temp of 0 indicates greedy decoding, + where the token with highest prob is chosen. Temperature can't be set + to 0.0 currently + type: number + maximum: 2.0 + minimum: 0.0 + max_tokens: + title: Max Tokens + description: Max tokens to generate + type: integer + minimum: 1.0 + max_completion_tokens: + title: Max Completion Tokens + description: Max tokens to generate + type: integer + minimum: 1.0 + top_p: + title: Top P + description: Float value between 0 and 1; limits to the top tokens within + a certain probability. top_p=0 means the model will only consider the + single most likely token for the next prediction + type: number + maximum: 1.0 + minimum: 0.0 + stop: + title: Stop + items: + type: string + type: array + additionalProperties: true + type: object + title: InferenceParams + description: Parameters for model inference. Extra fields can be supplied for + additional options applied to the inference request directly. Fields not supported + by the model may cause inference errors during evaluation. + MetricBundleInput: + properties: + bundle_kind: + type: string + const: metric-bundle + title: Bundle Kind + default: metric-bundle + bundle_format_version: + type: string + const: v1 + title: Bundle Format Version + default: v1 + metric_type: + type: string + minLength: 1 + title: Metric Type + metadata: + $ref: '#/components/schemas/MetricMetadata' + outputs: + items: + $ref: '#/components/schemas/BundledMetricOutputSpec' + type: array + minItems: 1 + title: Outputs + secrets: + additionalProperties: + $ref: '#/components/schemas/SecretRef' + type: object + title: Secrets + payload: + $ref: '#/components/schemas/MetricBundlePayload' + additionalProperties: false + type: object + required: + - metric_type + - outputs + - payload + title: MetricBundleInput + description: Standalone executable metric bundle entity used by backend execution. + MetricBundleOutput: + properties: + bundle_kind: + type: string + const: metric-bundle + title: Bundle Kind + default: metric-bundle + bundle_format_version: + type: string + const: v1 + title: Bundle Format Version + default: v1 + metric_type: + type: string + minLength: 1 + title: Metric Type + metadata: + $ref: '#/components/schemas/MetricMetadata' + outputs: + items: + $ref: '#/components/schemas/BundledMetricOutputSpec' + type: array + minItems: 1 + title: Outputs + secrets: + additionalProperties: + $ref: '#/components/schemas/SecretRef' + type: object + title: Secrets + payload: + additionalProperties: true + type: object + additionalProperties: false + type: object + required: + - metric_type + - outputs + - payload + title: MetricBundleOutput + description: Standalone executable metric bundle entity used by backend execution. + MetricBundlePayload: + properties: {} + type: object + title: MetricBundlePayload + description: Base class for concrete Pydantic metric bundle payloads. + MetricMetadata: + properties: + description: + title: Description + type: string + labels: + additionalProperties: + type: string + type: object + title: Labels + additionalProperties: true + type: object + title: MetricMetadata + description: User-facing metadata captured with a bundled metric. + Model: + properties: + url: + type: string + title: Url + description: URL of the model. + name: + type: string + title: Name + description: Name of the model. + api_key_secret: + allOf: + - $ref: '#/components/schemas/SecretRef' + description: 'API key secret reference for the model. Format: workspace/secret_name + or secret_name within the job workspace.' + format: + type: string + enum: + - nim + - openai + - llama_stack + title: Format + description: API format of the model. + default: nim + additionalProperties: false + type: object + required: + - url + - name + title: Model + description: Model definition for use without persisting to the Models API. + PaginationData: + properties: + page: + type: integer + title: Page + description: The current page number. + page_size: + type: integer + title: Page Size + description: The page size used for the query. + current_page_size: + type: integer + title: Current Page Size + description: The size for the current page. + total_pages: + type: integer + title: Total Pages + description: The total number of pages. + total_results: + type: integer + title: Total Results + description: The total number of results. + type: object + required: + - page + - page_size + - current_page_size + - total_pages + - total_results + title: PaginationData + PlatformJobListResultResponse: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobResultResponse' + type: array + title: Data + type: object + required: + - data + title: PlatformJobListResultResponse + PlatformJobLog: + properties: + timestamp: + type: string + format: date-time + title: Timestamp + job: + type: string + title: Job + job_step: + type: string + title: Job Step + job_task: + type: string + title: Job Task + message: + type: string + title: Message + type: object + required: + - timestamp + - job + - job_step + - job_task + - message + title: PlatformJobLog + PlatformJobLogPage: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobLog' + type: array + title: Data + total: + type: integer + title: Total + next_page: + title: Next Page + type: string + prev_page: + title: Prev Page + type: string + type: object + required: + - data + - total + - next_page + - prev_page + title: PlatformJobLogPage + PlatformJobResultResponse: + properties: + name: + type: string + title: Name + job: + type: string + title: Job + workspace: + type: string + title: Workspace + project: + title: Project + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + artifact_url: + type: string + title: Artifact Url + artifact_storage_type: + $ref: '#/components/schemas/FileStorageType' + download_url: + title: Download Url + type: string + type: object + required: + - name + - job + - workspace + - artifact_url + - artifact_storage_type + title: PlatformJobResultResponse + PlatformJobStatus: + type: string + enum: + - created + - pending + - active + - cancelled + - cancelling + - error + - completed + - paused + - pausing + - resuming + title: PlatformJobStatus + description: 'Enumeration of possible job statuses. + + + This enum represents the various states a job can be in during its lifecycle, + + from creation to a terminal state.' + PlatformJobStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + steps: + items: + $ref: '#/components/schemas/PlatformJobStepStatusResponse' + type: array + title: Steps + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - steps + - created_at + - updated_at + title: PlatformJobStatusResponse + PlatformJobStepStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + tasks: + items: + $ref: '#/components/schemas/PlatformJobTaskStatusResponse' + type: array + title: Tasks + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - tasks + - created_at + - updated_at + title: PlatformJobStepStatusResponse + PlatformJobTaskStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + error_stack: + title: Error Stack + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - error_stack + - created_at + - updated_at + title: PlatformJobTaskStatusResponse + ReasoningParams: + properties: + end_token: + title: End Token + description: 'Configure the end token to trim reasoning context based on + the model''s reasoning API. Example for Nemotron models: ''''' + type: string + include_if_not_finished: + title: Include If Not Finished + description: Configure whether to include reasoning context if the model + has not finished reasoning. + type: boolean + effort: + title: Effort + description: Option for OpenAI models to specify low, medium, or high reasoning + effort. + type: string + type: object + title: ReasoningParams + description: Custom settings that control the model's reasoning behavior. + RunConfig: + properties: + parallelism: + type: integer + minimum: 1.0 + title: Parallelism + description: Parallelism to be used for the evaluation job. Typically, this + represents the maximum number of concurrent requests made to the model. + default: 8 + limit_samples: + title: Limit Samples + description: Limit number of evaluation samples, taking the first `limit` + samples from the dataset. + type: integer + minimum: 1.0 + additionalProperties: false + type: object + title: RunConfig + description: Job parameters. + RunConfigOnline: + properties: + parallelism: + type: integer + minimum: 1.0 + title: Parallelism + description: Parallelism to be used for the evaluation job. Typically, this + represents the maximum number of concurrent requests made to the model. + default: 8 + limit_samples: + title: Limit Samples + description: Limit number of evaluation samples, taking the first `limit` + samples from the dataset. + type: integer + minimum: 1.0 + ignore_request_failure: + type: boolean + title: Ignore Request Failure + description: If True, request failures will be ignored and the result will + be marked as NaN. If False (default), request failures will raise an exception. + default: false + request_timeout: + title: Request Timeout + description: The timeout to be used for requests made to the model. + type: integer + max_retries: + type: integer + minimum: 0.0 + title: Max Retries + description: Maximum number of retries for failed requests. + default: 3 + additionalProperties: false + type: object + title: RunConfigOnline + description: Job parameters for online evaluation. + RunConfigOnlineModel: + properties: + parallelism: + type: integer + minimum: 1.0 + title: Parallelism + description: Parallelism to be used for the evaluation job. Typically, this + represents the maximum number of concurrent requests made to the model. + default: 8 + limit_samples: + title: Limit Samples + description: Limit number of evaluation samples, taking the first `limit` + samples from the dataset. + type: integer + minimum: 1.0 + ignore_request_failure: + type: boolean + title: Ignore Request Failure + description: If True, request failures will be ignored and the result will + be marked as NaN. If False (default), request failures will raise an exception. + default: false + request_timeout: + title: Request Timeout + description: The timeout to be used for requests made to the model. + type: integer + max_retries: + type: integer + minimum: 0.0 + title: Max Retries + description: Maximum number of retries for failed requests. + default: 3 + inference: + allOf: + - $ref: '#/components/schemas/InferenceParams' + description: Custom settings that control the model's text generation behavior. + system_prompt: + title: System Prompt + description: Initial instructions that define the model's role and behavior + for the conversation. + type: string + reasoning: + allOf: + - $ref: '#/components/schemas/ReasoningParams' + description: Custom settings that control the model's reasoning behavior. + structured_output: + title: Structured Output + description: JSON schema to apply structured output for the model. + additionalProperties: true + type: object + additionalProperties: false + type: object + title: RunConfigOnlineModel + description: Job parameters for model online evaluation. + SecretRef: + type: string + pattern: ^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)?$ + title: SecretRef + description: 'Reference to a platform secret or local environment variable. + Format: ''secret_name'' (uses request workspace) or ''workspace/secret_name'' + (explicit workspace).' + ValidationError: + properties: + loc: + items: + anyOf: + - type: string + - type: integer + type: array + title: Location + msg: + type: string + title: Message + type: + type: string + title: Error Type + input: + title: Input + ctx: + type: object + title: Context + additionalProperties: true + type: object + required: + - loc + - msg + - type + title: ValidationError diff --git a/plugins/nemo-evaluator/pyproject.toml b/plugins/nemo-evaluator/pyproject.toml index 4449858dfa0..be206e01ff7 100644 --- a/plugins/nemo-evaluator/pyproject.toml +++ b/plugins/nemo-evaluator/pyproject.toml @@ -51,5 +51,8 @@ asyncio_mode = "auto" pythonpath = ["src"] testpaths = ["tests"] +# Opt this plugin into OpenAPI spec generation +[tool.nemo.openapi] + [tool.pyright] extraPaths = ["src"] diff --git a/web/packages/sdk/orval/constants.ts b/web/packages/sdk/orval/constants.ts index 348799fb899..99d84f57bab 100644 --- a/web/packages/sdk/orval/constants.ts +++ b/web/packages/sdk/orval/constants.ts @@ -43,6 +43,12 @@ export const serviceConfigs: Record = { apiEnvKeys: ['VITE_PLATFORM_BASE_URL'], zod: true, }, + evaluator: { + path: 'evaluator', + url: `../../../../plugins/nemo-evaluator/openapi/openapi.yaml`, + apiEnvKeys: ['VITE_PLATFORM_BASE_URL'], + zod: true, + }, }; export const serviceToConfig = { diff --git a/web/packages/sdk/package.json b/web/packages/sdk/package.json index c035c62cb6d..245c02168f5 100644 --- a/web/packages/sdk/package.json +++ b/web/packages/sdk/package.json @@ -14,9 +14,10 @@ "gen:customizer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts customizer", "gen:data-designer": "tsx ./orval/generate.ts data-designer", "gen:data-designer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts data-designer", + "gen:evaluator": "tsx ./orval/generate.ts evaluator", + "gen:evaluator-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts evaluator", "gen:deployment-management": "tsx ./orval/generate.ts deployment-management", "gen:entity-store": "tsx ./orval/generate.ts entity-store", - "gen:evaluator": "tsx ./orval/generate.ts evaluator", "gen:guardrails": "tsx ./orval/generate.ts guardrails", "gen:inference-gateway": "tsx ./orval/generate.ts inference-gateway", "gen:jobs": "tsx ./orval/generate.ts jobs", @@ -25,7 +26,9 @@ "gen:platform-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts platform", "gen:safe-synthesizer": "tsx ./orval/generate.ts safe-synthesizer", "gen:safe-synthesizer-zod": "ORVAL_CLIENT=zod tsx ./orval/generate.ts safe-synthesizer", - "test": "vitest run" + "test": "vitest run", + "test:ci": "vitest run", + "typecheck": "tsc --noEmit" }, "dependencies": { "@tanstack/react-query": "catalog:", @@ -39,7 +42,10 @@ "concurrently": "catalog:", "orval": "catalog:", "prettier": "catalog:", + "@types/node": "catalog:", "tsx": "catalog:", + "typescript": "catalog:", + "vite": "catalog:", "vitest": "catalog:" } } diff --git a/web/packages/sdk/tsconfig.json b/web/packages/sdk/tsconfig.json new file mode 100644 index 00000000000..b7945bf6a1b --- /dev/null +++ b/web/packages/sdk/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "strict": true, + "skipLibCheck": true, + "types": ["vite/client", "node"] + }, + "include": ["src/**/*.ts", "generated/*/api.ts", "generated/*/schema/**/*.ts"] +} diff --git a/web/packages/studio/src/api/evaluation/EvaluationApiError.ts b/web/packages/studio/src/api/evaluation/EvaluationApiError.ts index c178aa68e5e..a93d9db4abe 100644 --- a/web/packages/studio/src/api/evaluation/EvaluationApiError.ts +++ b/web/packages/studio/src/api/evaluation/EvaluationApiError.ts @@ -1,23 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { ErrorResponse, HTTPValidationError } from '@nemo/sdk/generated/platform/schema'; - -const isHttpValidationError = ( - response: HTTPValidationError | ErrorResponse -): response is HTTPValidationError => { - return 'detail' in response && Array.isArray(response.detail); -}; +import type { HTTPValidationError } from '@nemo/sdk/generated/evaluator/schema'; export class EvaluationApiError extends Error { - constructor(public response: ErrorResponse | HTTPValidationError) { - let message; - if (isHttpValidationError(response)) { - message = response.detail?.map((err: { msg?: string }) => err.msg).join(''); - } else { - // ErrorResponse has {detail: string} - message = response.detail; - } + constructor(public response: HTTPValidationError) { + const message = Array.isArray(response.detail) + ? response.detail.map((err: { msg?: string }) => err.msg).join('') + : response.detail; super(message || 'Unknown error'); } diff --git a/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx b/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx index e87f98833e7..a5f92135472 100644 --- a/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx +++ b/web/packages/studio/src/api/evaluation/details/useAggregatedEvaluations.tsx @@ -3,15 +3,15 @@ import { NamedEntity } from '@nemo/common/src/namedEntity'; import { - evaluationGetMetricJob, - evaluationListMetricJobsResults, - evaluationListMetricJobs, -} from '@nemo/sdk/generated/platform/api'; + evaluatorGetEvaluateJob, + evaluatorListEvaluateJobResults, + evaluatorListEvaluateJobs, +} from '@nemo/sdk/generated/evaluator/api'; import { - MetricEvaluationJob, - MetricEvaluationJobsPage, + EvaluateJob, + EvaluateJobsPage, PlatformJobListResultResponse, -} from '@nemo/sdk/generated/platform/schema'; +} from '@nemo/sdk/generated/evaluator/schema'; import { useDetailsChartsStore } from '@studio/api/evaluation/details/useDetailsChartsStore'; import { getEvaluationJobModel, isEvaluationJobSucceeded } from '@studio/selectors/evaluationJob'; import { useQuery } from '@tanstack/react-query'; @@ -23,24 +23,24 @@ import { useQuery } from '@tanstack/react-query'; * Example usage in a component: * fetchAllEvaluationsByConfig(workspace); * - * @returns {Object} Promise + * @returns {Object} Promise */ const fetchAllEvaluationsByConfig = async ( workspace: string | undefined -): Promise => { +): Promise => { if (!workspace) { return { data: [], }; } - let allEvaluations: MetricEvaluationJobsPage['data'] = []; + let allEvaluations: EvaluateJobsPage['data'] = []; let currentPage = 1; let totalPages = 1; // initialize to avoid infinite loop const pageSize = 100; while (currentPage <= totalPages) { - const response = await evaluationListMetricJobs(workspace, { + const response = await evaluatorListEvaluateJobs(workspace, { page_size: pageSize, page: currentPage, sort: '-created_at', @@ -77,11 +77,11 @@ const fetchAllEvaluationsByConfig = async ( * Example usage in a component: * useUniqueModelsByConfig(workspace, id); * - * @returns {Object} + * @returns {Object} */ export const useUniqueModelsByConfig = (workspace: string | undefined, evaluationId?: string) => { - return useQuery, Error>({ + return useQuery, Error>({ queryKey: ['uniqueModelsByConfig', workspace, evaluationId], queryFn: async () => { const evaluationsByConfig = await fetchAllEvaluationsByConfig(workspace); @@ -97,13 +97,13 @@ export const useUniqueModelsByConfig = (workspace: string | undefined, evaluatio if (modelName) { if (!acc[modelName]) { - acc[modelName] = [] as MetricEvaluationJob[]; + acc[modelName] = [] as EvaluateJob[]; } acc[modelName].push(job); } return acc; }, - {} as Record + {} as Record ); }, placeholderData: (prevData) => prevData ?? {}, @@ -130,7 +130,7 @@ export const useAggregatedEvaluationResults = (config: NamedEntity) => { } return await Promise.all( selectedEvaluations.map((evaluationId) => - evaluationListMetricJobsResults(config.workspace!, evaluationId) + evaluatorListEvaluateJobResults(config.workspace!, evaluationId) ) ); }, @@ -146,12 +146,12 @@ export const useAggregatedEvaluationResults = (config: NamedEntity) => { * Example usage in a component: * useAggregatedEvaluations(config); * - * @returns {Array} MetricEvaluationJob[] + * @returns {Array} EvaluateJob[] */ export const useAggregatedEvaluations = (config: NamedEntity) => { const { selectedEvaluations } = useDetailsChartsStore(); - return useQuery({ + return useQuery({ queryKey: ['aggregatedEvaluations', config, selectedEvaluations], queryFn: async () => { if (!config.workspace) { @@ -159,7 +159,7 @@ export const useAggregatedEvaluations = (config: NamedEntity) => { } return await Promise.all( selectedEvaluations.map((evaluationId) => - evaluationGetMetricJob(config.workspace!, evaluationId) + evaluatorGetEvaluateJob(config.workspace!, evaluationId) ) ); }, diff --git a/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx b/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx index 2c6e2d30a60..9129ac79d77 100644 --- a/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx +++ b/web/packages/studio/src/api/evaluation/details/useEvaluationDownloadResultsById.tsx @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { ExcludedChatCompletionMessageParam } from '@nemo/common/src/types/chat'; -import { evaluationDownloadMetricJobResultRowScores } from '@nemo/sdk/generated/platform/api'; +import { evaluatorDownloadEvaluateJobResult } from '@nemo/sdk/generated/evaluator/api'; import { useQuery } from '@tanstack/react-query'; export interface EvaluationMetrics { @@ -34,16 +34,14 @@ export interface EvaluationResultItem { metrics: EvaluationMetrics; } -// V2 row score shape from the platform API -interface V2RowScore { +interface RowScore { index: number; row: Record; scores?: Record; error?: string; } -// Map V2 row scores to the EvaluationResultItem format used by the UI -function mapV2RowScores(rows: V2RowScore[]): EvaluationResultItem[] { +function mapRowScores(rows: RowScore[]): EvaluationResultItem[] { return rows.map((row) => ({ item: { prompt: String(row.row?.prompt ?? row.row?.input ?? ''), @@ -55,19 +53,17 @@ function mapV2RowScores(rows: V2RowScore[]): EvaluationResultItem[] { sample: { output_text: String(row.row?.output ?? row.row?.output_text ?? ''), }, - metrics: mapV2Scores(row.scores), + metrics: mapScores(row.scores), })); } -function mapV2Scores(scores?: Record): EvaluationMetrics { +function mapScores(scores?: Record): EvaluationMetrics { if (!scores) return {}; const result: EvaluationMetrics = {}; - // Map V2 flat scores to the nested metrics structure the UI expects for (const [key, value] of Object.entries(scores)) { if (key === 'sentence' || key === 'corpus') { - // BLEU sub-scores if (!result.bleu) { result.bleu = { scores: {} }; } @@ -83,27 +79,31 @@ function mapV2Scores(scores?: Record): EvaluationMetrics { return result; } -async function fetchEvaluationResultsV2( +async function fetchEvaluationResults( workspace: string, - jobName: string + jobName: string, + name: string ): Promise { - const blob = await evaluationDownloadMetricJobResultRowScores(workspace, jobName); + const blob = await evaluatorDownloadEvaluateJobResult(workspace, jobName, name); const text = await blob.text(); const parsed = JSON.parse(text); - // The response is either an array of row scores or wrapped in a container - const rows: V2RowScore[] = Array.isArray(parsed) + const rows: RowScore[] = Array.isArray(parsed) ? parsed : (parsed.data ?? parsed.row_scores ?? []); - return mapV2RowScores(rows); + return mapRowScores(rows); } -export const useEvaluationDownloadResultsById = (workspace: string, jobName: string) => { +export const useEvaluationDownloadResultsById = ( + workspace: string, + jobName: string, + name = 'row_scores' +) => { return useQuery({ - queryKey: ['evaluationDownloadResults', workspace, jobName], - queryFn: () => fetchEvaluationResultsV2(workspace, jobName), + queryKey: ['evaluationDownloadResults', workspace, jobName, name], + queryFn: () => fetchEvaluationResults(workspace, jobName, name), enabled: !!workspace && !!jobName, }); }; diff --git a/web/packages/studio/src/api/evaluation/index.ts b/web/packages/studio/src/api/evaluation/index.ts index 30f3d69d7b0..baa153d146e 100644 --- a/web/packages/studio/src/api/evaluation/index.ts +++ b/web/packages/studio/src/api/evaluation/index.ts @@ -1,12 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { evaluationListMetricJobs } from '@nemo/sdk/generated/platform/api'; -import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/schema'; -import { EvaluationJobWithTaskMetrics } from '@studio/api/evaluation/useEvaluationsWithMetrics'; +import { evaluatorListEvaluateJobs } from '@nemo/sdk/generated/evaluator/api'; +import type { EvaluatorListEvaluateJobsParams } from '@nemo/sdk/generated/evaluator/schema'; +import type { EvaluationJobWithTaskMetrics } from '@studio/api/evaluation/useEvaluationsWithMetrics'; export interface FetchEvaluationsWithMetricsOptions { workspace: string; - query?: EvaluationListMetricJobsParams; + query?: EvaluatorListEvaluateJobsParams; signal?: AbortSignal; } @@ -15,7 +15,7 @@ export const fetchEvaluationsWithMetrics = async ({ query, signal, }: FetchEvaluationsWithMetricsOptions) => { - const evaluations = await evaluationListMetricJobs(workspace, query, signal); + const evaluations = await evaluatorListEvaluateJobs(workspace, query, signal); if (evaluations.data?.length > 0) { const evaluationsWithMetrics = await Promise.all( evaluations.data.map(async (evaluation) => { @@ -28,19 +28,8 @@ export const fetchEvaluationsWithMetrics = async ({ } const evaluationWithMetrics: EvaluationJobWithTaskMetrics = { ...evaluation, tasks: {} }; - // TODO: Download results - // const evaluationResult = await getEvaluationResult( - // evaluation.workspace, - // evaluation.id!, - // evaluation.name, - // signal - // ); - // if (evaluationResult.download_url) { - // } return evaluationWithMetrics; } catch { - // if an eval job failed, there will be a 404 returned by fetchEvaluationResults - // return the original evaluation if there's an error return evaluation; } }) diff --git a/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts b/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts index 202a861ff90..c64dec2db47 100644 --- a/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts +++ b/web/packages/studio/src/api/evaluation/useEvaluationsByConfig.ts @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { NamedEntity } from '@nemo/common/src/namedEntity'; -import { useEvaluationListMetricJobs } from '@nemo/sdk/generated/platform/api'; -import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/schema'; +import { useEvaluatorListEvaluateJobs } from '@nemo/sdk/generated/evaluator/api'; +import type { EvaluatorListEvaluateJobsParams } from '@nemo/sdk/generated/evaluator/schema'; /** * Custom hook to fetch evaluation jobs filtered by configuration. @@ -19,15 +18,13 @@ import { EvaluationListMetricJobsParams } from '@nemo/sdk/generated/platform/sch */ export const useEvaluationsByConfig = ( workspace: string, - config: NamedEntity & Partial + config: Partial ) => { - const params: EvaluationListMetricJobsParams = { + const params: EvaluatorListEvaluateJobsParams = { page: config.page, page_size: config.page_size, sort: 'created_at', - // TODO: Support filtering by config namespace and name - // config: config.namespace && config.name ? `${config.namespace}/${config.name}` : undefined, }; - return useEvaluationListMetricJobs(workspace, params); + return useEvaluatorListEvaluateJobs(workspace, params); }; diff --git a/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts b/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts index 60ee6a16f46..7ff3c4eb6aa 100644 --- a/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts +++ b/web/packages/studio/src/api/evaluation/useEvaluationsWithMetrics.ts @@ -10,11 +10,11 @@ * its affiliates is strictly prohibited. */ import { - MetricEvaluationJob, - MetricEvaluationJobsPage, + EvaluateJob, + EvaluateJobsPage, PlatformJobStatus as JobStatus, - EvaluationListMetricJobsParams, -} from '@nemo/sdk/generated/platform/schema'; + EvaluatorListEvaluateJobsParams, +} from '@nemo/sdk/generated/evaluator/schema'; import { EvaluationApiError } from '@studio/api/evaluation/EvaluationApiError'; import { fetchEvaluationsWithMetrics } from '@studio/api/evaluation/index'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; @@ -23,32 +23,20 @@ import { UseQueryOptions, queryOptions, useQuery } from '@tanstack/react-query'; /** Task result shape appended by fetchEvaluationsWithMetrics. */ export type TaskResultOutput = Record; -export interface EvaluationJobWithTaskMetrics extends MetricEvaluationJob { +export interface EvaluationJobWithTaskMetrics extends EvaluateJob { tasks?: Record; } -export interface EvaluationJobsWithTaskMetricsPage extends Omit { +export interface EvaluationJobsWithTaskMetricsPage extends Omit { data: EvaluationJobWithTaskMetrics[]; } -/** - * these types are here to correctly overload our API types with a custom shape for this query - * GET /evaluations currently returns: - * { - * result: { // EvaluationJobSummary - * evaluations: [] // EvaluationConfig[] - * } - * } - * - * This custom query appends an array metrics to each EvaluationConfig object - */ - export interface UseEvaluationsWithMetricsOptions { queryOptions?: Omit< UseQueryOptions, 'queryFn' | 'queryKey' >; - query?: EvaluationListMetricJobsParams; + query?: EvaluatorListEvaluateJobsParams; filters?: { tags?: { projectId?: string; @@ -60,7 +48,7 @@ export interface UseEvaluationsWithMetricsOptions { export const getEvaluationsWithMetricsQueryOptions = ( workspace: string, - query?: EvaluationListMetricJobsParams + query?: EvaluatorListEvaluateJobsParams ) => { return queryOptions({ queryKey: ['evaluationsWithMetrics', query], @@ -75,7 +63,6 @@ export const useEvaluationsWithMetrics = (options?: UseEvaluationsWithMetricsOpt const { tags, status } = options.filters; let filter = ''; - // Add any tags to the filter query string const tagFilters = []; if (tags?.projectId) { tagFilters.push(tags.projectId); @@ -88,7 +75,6 @@ export const useEvaluationsWithMetrics = (options?: UseEvaluationsWithMetricsOpt filter = `tag:${tagFilterString}`; } - // Add status to the filter if (status) { const statusFilterString = `status:${status}`; filter = filter ? `${filter},${statusFilterString}` : statusFilterString; diff --git a/web/packages/studio/src/components/dataViews/CustomModelsDataView/index.tsx b/web/packages/studio/src/components/dataViews/CustomModelsDataView/index.tsx index 2ff8e1d05af..bc04e78e0bc 100644 --- a/web/packages/studio/src/components/dataViews/CustomModelsDataView/index.tsx +++ b/web/packages/studio/src/components/dataViews/CustomModelsDataView/index.tsx @@ -39,7 +39,7 @@ import { BaseModelSearchFilterField } from '@studio/components/FilterFields'; import type { ModelPanelTab } from '@studio/components/sidePanels/ModelPanels/ModelPanel'; import { INTAKE_ENABLED } from '@studio/constants/environment'; import { LINK_DOCS_STUDIO_CUSTOMIZATION } from '@studio/constants/links'; -import { getEvaluationMetricsRunRoute, getIntakeTracesRoute } from '@studio/routes/utils'; +import { getIntakeTracesRoute } from '@studio/routes/utils'; import { keepPreviousData } from '@tanstack/react-query'; import { BrainCircuit, X, Trash } from 'lucide-react'; import { ComponentProps, FC, useCallback, useMemo, useState } from 'react'; @@ -365,13 +365,6 @@ export const CustomModelsDataView: FC = ({ children: 'Chat Playground', onSelect: () => onRowClick?.(row, 'chat-playground'), }, - { kind: 'divider' as const }, - { - children: 'Evaluate Model', - onSelect: () => { - navigate(getEvaluationMetricsRunRoute(workspace, { model: row.name })); - }, - }, ...(INTAKE_ENABLED ? [ { diff --git a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx b/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx deleted file mode 100644 index 86ea5de23a4..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.spec.tsx +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { EvaluationBenchmarksDataView } from '@studio/components/dataViews/EvaluationBenchmarksDataView'; -import { PLATFORM_BASE_URL } from '@studio/constants/environment'; -import { server } from '@studio/mocks/node'; -import { renderRoute } from '@studio/tests/util/render'; -import { waitFor } from '@testing-library/react'; -import { http, HttpResponse } from 'msw'; - -vi.mock('use-debounce', () => ({ - useDebounce: (value: unknown) => [value, { cancel: () => {}, flush: () => {} }], -})); - -const BENCHMARKS_URL = `${PLATFORM_BASE_URL}/apis/evaluation/v2/workspaces/:workspace/benchmarks`; - -const emptyBenchmarksPage = { - data: [], - pagination: { - page: 1, - page_size: 50, - current_page_size: 0, - total_pages: 0, - total_results: 0, - }, -}; - -const captureRequests = () => { - const requestUrls: string[] = []; - server.use( - http.get(BENCHMARKS_URL, ({ request }) => { - requestUrls.push(request.url); - return HttpResponse.json(emptyBenchmarksPage); - }) - ); - return requestUrls; -}; - -const renderDataView = (initialEntry: string) => - renderRoute(, { - history: initialEntry, - routes: [ - { - path: '/workspaces/:workspace/evaluation/benchmarks', - element: , - }, - ], - }); - -describe('EvaluationBenchmarksDataView', () => { - afterEach(() => { - server.resetHandlers(); - }); - - it('sends no filter[ or search params on initial render', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/benchmarks'); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - const url = new URL(requestUrls.at(-1)!); - const keys = Array.from(url.searchParams.keys()); - expect(keys.some((k) => k.startsWith('filter['))).toBe(false); - expect(keys.some((k) => k === 'search' || k.startsWith('search['))).toBe(false); - }); - - it('sends filter[name][$like] when name search is active', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/benchmarks?s=foo'); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[name][$like]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[name][$like]')).toBe('foo'); - }); - - it('sends filter[created_at][$gte]/$lte when column filter is active', async () => { - const requestUrls = captureRequests(); - const gte = '2026-01-01T00:00:00.000Z'; - const lte = '2026-02-01T00:00:00.000Z'; - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: gte, $lte: lte } }]) - ); - renderDataView(`/workspaces/default/evaluation/benchmarks?filters=${filters}`); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[created_at][$gte]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[created_at][$gte]')).toBe(gte); - expect(url.searchParams.get('filter[created_at][$lte]')).toBe(lte); - }); - - it('never emits a top-level search or search[ key', async () => { - const requestUrls = captureRequests(); - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: '2026-01-01T00:00:00.000Z' } }]) - ); - renderDataView(`/workspaces/default/evaluation/benchmarks?s=foo&filters=${filters}`); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - for (const raw of requestUrls) { - const params = new URL(raw).searchParams; - expect(params.has('search')).toBe(false); - expect(Array.from(params.keys()).some((k) => k.startsWith('search['))).toBe(false); - } - }); -}); diff --git a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx deleted file mode 100644 index 7517991cc34..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/index.tsx +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { withOperators, type WithFilterOperators } from '@nemo/common/src/api/filterOperators'; -import { dateTimeFilter } from '@nemo/common/src/components/DataView/dateTimeFilter'; -import { StudioDataView } from '@nemo/common/src/components/DataView/StudioDataView'; -import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; -import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; -import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; -import { getSortParam } from '@nemo/common/src/utils/query'; -import { useEvaluationListBenchmarks } from '@nemo/sdk/generated/platform/api'; -import type { - EvaluationListBenchmarksParams, - EvaluationListBenchmarksSort, -} from '@nemo/sdk/generated/platform/schema'; -import { Button, Stack, Text } from '@nvidia/foundations-react-core'; -import type { - BenchmarkItemWithId, - EvaluationBenchmarksDataViewProps, -} from '@studio/components/dataViews/EvaluationBenchmarksDataView/types'; -import { DocumentationButton } from '@studio/components/DocumentationButton'; -import { EMPTY_FIELD_EMDASH_VALUE } from '@studio/constants/constants'; -import { LINK_EVAL_DOCS_BENCHMARKS } from '@studio/constants/links'; -import { keepPreviousData } from '@tanstack/react-query'; -import { X } from 'lucide-react'; -import { type ComponentProps, type FC, useCallback, useMemo } from 'react'; - -export const EvaluationBenchmarksDataView: FC = ({ - workspace, - onRowClick, -}) => { - const dataViewState = useStudioDataViewState({ - defaultSort: { id: 'created_at', desc: true }, - }); - - const benchmarksFilter = useMemo(() => { - type BenchmarksFilter = NonNullable; - const filterObj: WithFilterOperators = {}; - - const nameFilter = dataViewState.apiFilter.searchText; - if (nameFilter) { - filterObj.name = { $like: nameFilter }; - } - - const createdAtFilter = dataViewState.apiFilter.filter?.created_at; - if ( - createdAtFilter && - typeof createdAtFilter === 'object' && - Object.keys(createdAtFilter).length > 0 - ) { - filterObj.created_at = createdAtFilter as BenchmarksFilter['created_at']; - } - - if (Object.keys(filterObj).length === 0) return undefined; - return withOperators(filterObj); - }, [dataViewState.apiFilter]); - - const { - data: benchmarksResponse, - isFetching, - refetch, - error, - } = useEvaluationListBenchmarks( - workspace, - { - page: dataViewState.pagination.state.pageIndex + 1, - page_size: dataViewState.pagination.state.pageSize, - sort: getSortParam(dataViewState.sorting.state) as EvaluationListBenchmarksSort, - filter: benchmarksFilter, - }, - { - query: { - placeholderData: keepPreviousData, - }, - } - ); - - const benchmarks = useMemo( - () => - (benchmarksResponse?.data || []).map((b) => ({ - ...b, - id: b.id ?? b.name ?? '', - })), - [benchmarksResponse?.data] - ); - - const hasActiveFilters = !!( - dataViewState.debouncedSearchBar || dataViewState.debouncedColumnFilters.length > 0 - ); - - const makeColumns: ComponentProps>['makeColumns'] = - useCallback( - ({ accessor }) => [ - accessor('name', { - header: 'Name', - enableSorting: true, - cell: ({ row }) => row.original.name ?? EMPTY_FIELD_EMDASH_VALUE, - }), - accessor( - (row) => - 'description' in row - ? (row.description ?? EMPTY_FIELD_EMDASH_VALUE) - : EMPTY_FIELD_EMDASH_VALUE, - { - id: 'description', - header: 'Description', - enableSorting: false, - size: 280, - } - ), - accessor( - (row) => ('metrics' in row && Array.isArray(row.metrics) ? row.metrics.length : null), - { - id: 'metrics_count', - header: 'Metrics', - enableSorting: false, - size: 100, - cell: ({ getValue }) => { - const count = getValue(); - return {count !== null ? count : EMPTY_FIELD_EMDASH_VALUE}; - }, - } - ), - accessor( - (row) => { - if (!('dataset' in row)) return '—'; - const { dataset } = row; - if (typeof dataset === 'string') return dataset; - if (dataset !== null && typeof dataset === 'object' && 'name' in dataset) { - return String((dataset as { name: string }).name); - } - return EMPTY_FIELD_EMDASH_VALUE; - }, - { - id: 'dataset', - header: 'Dataset', - enableSorting: false, - } - ), - accessor('created_at', { - id: 'created_at', - header: 'Created', - enableSorting: true, - size: 150, - meta: { - filter: dateTimeFilter('Created At'), - }, - cell: ({ row }) => { - const createdAt = 'created_at' in row.original ? row.original.created_at : undefined; - return createdAt ? : EMPTY_FIELD_EMDASH_VALUE; - }, - }), - ], - [] - ); - - return ( - - - dataViewState={dataViewState} - searchField="name" - makeColumns={makeColumns} - onRowClick={onRowClick} - attributes={{ - DataViewSearchBar: { - placeholder: 'Search benchmark name', - }, - DataViewRoot: { - data: benchmarks, - totalCount: benchmarksResponse?.pagination?.total_results, - requestStatus: error ? 'error' : isFetching ? 'loading' : undefined, - }, - DataViewTableContent: { - renderEmptyState: () => - hasActiveFilters ? ( - dataViewState.resetFilters()}> - Clear Filters - - } - /> - ) : ( - } - /> - ), - renderErrorState: () => ( - refetch()}> - Retry - - } - /> - ), - }, - }} - /> - - ); -}; diff --git a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts b/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts deleted file mode 100644 index 9a6dab046aa..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationBenchmarksDataView/types.ts +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { BenchmarksListResponse } from '@nemo/sdk/generated/platform/schema'; - -export interface EvaluationBenchmarksDataViewProps { - workspace: string; - onRowClick?: (benchmark: BenchmarkItemWithId) => void; -} - -export type BenchmarkItem = BenchmarksListResponse['data'][number]; -export type BenchmarkItemWithId = BenchmarkItem & { id: string }; diff --git a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx deleted file mode 100644 index 1a93b50a4ab..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.spec.tsx +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -import { EvaluationMetricsDataView } from '@studio/components/dataViews/EvaluationMetricsDataView'; -import { PLATFORM_BASE_URL } from '@studio/constants/environment'; -import { server } from '@studio/mocks/node'; -import { renderRoute } from '@studio/tests/util/render'; -import { waitFor } from '@testing-library/react'; -import { http, HttpResponse } from 'msw'; - -vi.mock('use-debounce', () => ({ - useDebounce: (value: unknown) => [value, { cancel: () => {}, flush: () => {} }], -})); - -const METRICS_URL = `${PLATFORM_BASE_URL}/apis/evaluation/v2/workspaces/:workspace/metrics`; - -const emptyMetricsPage = { - data: [], - pagination: { - page: 1, - page_size: 50, - current_page_size: 0, - total_pages: 0, - total_results: 0, - }, -}; - -const captureRequests = () => { - const requestUrls: string[] = []; - server.use( - http.get(METRICS_URL, ({ request }) => { - requestUrls.push(request.url); - return HttpResponse.json(emptyMetricsPage); - }) - ); - return requestUrls; -}; - -const renderDataView = (initialEntry: string) => - renderRoute(, { - history: initialEntry, - routes: [ - { - path: '/workspaces/:workspace/evaluation/metrics', - element: , - }, - ], - }); - -describe('EvaluationMetricsDataView', () => { - afterEach(() => { - server.resetHandlers(); - }); - - it('sends no filter[ or search params on initial render', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/metrics'); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - const url = new URL(requestUrls.at(-1)!); - const keys = Array.from(url.searchParams.keys()); - expect(keys.some((k) => k.startsWith('filter['))).toBe(false); - expect(keys.some((k) => k === 'search' || k.startsWith('search['))).toBe(false); - }); - - it('sends filter[name][$like] when name search is active', async () => { - const requestUrls = captureRequests(); - renderDataView('/workspaces/default/evaluation/metrics?s=foo'); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[name][$like]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[name][$like]')).toBe('foo'); - }); - - it('sends filter[created_at][$gte]/$lte when column filter is active', async () => { - const requestUrls = captureRequests(); - const gte = '2026-01-01T00:00:00.000Z'; - const lte = '2026-02-01T00:00:00.000Z'; - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: gte, $lte: lte } }]) - ); - renderDataView(`/workspaces/default/evaluation/metrics?filters=${filters}`); - - await waitFor(() => { - expect(requestUrls.some((u) => new URL(u).searchParams.has('filter[created_at][$gte]'))).toBe( - true - ); - }); - - const url = new URL(requestUrls.at(-1)!); - expect(url.searchParams.get('filter[created_at][$gte]')).toBe(gte); - expect(url.searchParams.get('filter[created_at][$lte]')).toBe(lte); - }); - - it('never emits a top-level search or search[ key', async () => { - const requestUrls = captureRequests(); - const filters = encodeURIComponent( - JSON.stringify([{ id: 'created_at', value: { $gte: '2026-01-01T00:00:00.000Z' } }]) - ); - renderDataView(`/workspaces/default/evaluation/metrics?s=foo&filters=${filters}`); - - await waitFor(() => expect(requestUrls.length).toBeGreaterThan(0)); - - for (const raw of requestUrls) { - const params = new URL(raw).searchParams; - expect(params.has('search')).toBe(false); - expect(Array.from(params.keys()).some((k) => k.startsWith('search['))).toBe(false); - } - }); -}); diff --git a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx deleted file mode 100644 index 61716d05f53..00000000000 --- a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/index.tsx +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { withOperators, type WithFilterOperators } from '@nemo/common/src/api/filterOperators'; -import { dateTimeFilter } from '@nemo/common/src/components/DataView/dateTimeFilter'; -import { - ROW_ACTIONS_COLUMN_SIZE, - ROW_SELECTION_COLUMN_SIZE, - StudioDataView, -} from '@nemo/common/src/components/DataView/StudioDataView'; -import { ErrorMessage } from '@nemo/common/src/components/ErrorMessage'; -import { RelativeTime } from '@nemo/common/src/components/RelativeTime'; -import { TableEmptyState } from '@nemo/common/src/components/TableEmptyState'; -import { useStudioDataViewState } from '@nemo/common/src/hooks/useStudioDataViewState'; -import { getSortParam } from '@nemo/common/src/utils/query'; -import { - getEvaluationListMetricsQueryKey, - useEvaluationDeleteMetric, - useEvaluationListMetrics, -} from '@nemo/sdk/generated/platform/api'; -import type { - EvaluationListMetricsParams, - EvaluationListMetricsSort, -} from '@nemo/sdk/generated/platform/schema'; -import { Badge, Button, Flex, Stack, Text } from '@nvidia/foundations-react-core'; -import type { - EvaluationMetricsDataViewProps, - MetricItemWithId, -} from '@studio/components/dataViews/EvaluationMetricsDataView/types'; -import { DeleteConfirmationModal } from '@studio/components/DeleteConfirmationModal'; -import { EvaluationMetricBulkDeleteModal } from '@studio/components/evaluation/Metrics/EvaluationMetricBulkDeleteModal'; -import { - getEvaluationMetricDetailsRoute, - getEvaluationMetricRunRoute, - getNewEvaluationMetricRoute, -} from '@studio/routes/utils'; -import { keepPreviousData, useQueryClient } from '@tanstack/react-query'; -import { Play, Scale, X } from 'lucide-react'; -import { - type ComponentProps, - type FC, - type ReactNode, - useCallback, - useMemo, - useState, -} from 'react'; -import { useNavigate } from 'react-router-dom'; - -interface MetricTypeMeta { - label: string; - icon?: ReactNode; -} - -const METRIC_TYPE_META: Record = { - 'llm-judge': { label: 'LLM Judge', icon: }, -}; - -const getMetricTypeMeta = (type: string): MetricTypeMeta => - METRIC_TYPE_META[type] ?? { - label: type - .split('-') - .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) - .join(' '), - }; - -export const EvaluationMetricsDataView: FC = ({ - workspace, - onRowClick, -}) => { - const queryClient = useQueryClient(); - const navigate = useNavigate(); - const [metricToDelete, setMetricToDelete] = useState(null); - const { mutateAsync: deleteMetric } = useEvaluationDeleteMetric(); - const dataViewState = useStudioDataViewState({ - defaultSort: { id: 'created_at', desc: true }, - }); - - // Build filter object from the dataview filter state - const metricsFilter = useMemo(() => { - type MetricsFilter = NonNullable; - const filterObj: WithFilterOperators = {}; - - const nameFilter = dataViewState.apiFilter.searchText; - if (nameFilter) { - filterObj.name = { $like: nameFilter }; - } - - const createdAtFilter = dataViewState.apiFilter.filter?.created_at; - if ( - createdAtFilter && - typeof createdAtFilter === 'object' && - Object.keys(createdAtFilter).length > 0 - ) { - filterObj.created_at = createdAtFilter as MetricsFilter['created_at']; - } - - if (Object.keys(filterObj).length === 0) return undefined; - return withOperators(filterObj); - }, [dataViewState.apiFilter]); - - const { - data: evaluationsData, - isLoading, - refetch, - error, - } = useEvaluationListMetrics( - workspace, - { - page: dataViewState.pagination.state.pageIndex + 1, - page_size: dataViewState.pagination.state.pageSize, - sort: getSortParam(dataViewState.sorting.state) as EvaluationListMetricsSort, - filter: metricsFilter, - }, - { - query: { - refetchOnMount: 'always', - placeholderData: keepPreviousData, - }, - } - ); - - const evaluationMetrics = useMemo( - () => - (evaluationsData?.data || []).map((metric) => ({ - ...metric, - id: metric.id ?? metric.name ?? '', - })), - [evaluationsData?.data] - ); - - const invalidateMetrics = useCallback(() => { - queryClient.invalidateQueries({ - queryKey: getEvaluationListMetricsQueryKey(workspace), - }); - }, [queryClient, workspace]); - - const hasActiveFilters = !!( - dataViewState.debouncedSearchBar || dataViewState.debouncedColumnFilters.length > 0 - ); - - const makeColumns: ComponentProps>['makeColumns'] = - useCallback( - ({ accessor }, { rowSelectionColumn, rowActionsColumn }) => [ - rowSelectionColumn({ size: ROW_SELECTION_COLUMN_SIZE }), - accessor('name', { - header: 'Name', - enableSorting: false, - cell: ({ row }) => row.original.name ?? '-', - }), - accessor((row) => ('description' in row ? (row.description ?? '') : ''), { - id: 'description', - header: 'Description', - enableSorting: false, - cell: ({ row }) => { - const description = - 'description' in row.original ? row.original.description : undefined; - return description ? <>{description} : <>-; - }, - }), - accessor((row) => ('type' in row ? (row.type ?? '') : ''), { - id: 'type', - header: 'Type', - enableSorting: false, - size: 160, - cell: ({ row }) => { - const type = 'type' in row.original ? (row.original.type ?? null) : null; - if (!type) return <>-; - const meta = getMetricTypeMeta(type); - return ( - - {meta.icon} - {meta.label} - - ); - }, - }), - accessor('created_at', { - id: 'created_at', - header: 'Created', - enableSorting: true, - size: 150, - meta: { - filter: dateTimeFilter('Created At'), - }, - cell: ({ row }) => { - const createdAt = 'created_at' in row.original ? row.original.created_at : undefined; - return createdAt ? : -; - }, - }), - accessor(() => '', { - id: 'run', - header: '', - size: 152, - enableSorting: false, - cell: ({ row }) => ( - - - - ), - }), - rowActionsColumn({ - size: ROW_ACTIONS_COLUMN_SIZE, - cellProps: { - attributes: { - DropdownContent: { className: 'min-w-[140px]' }, - }, - }, - rowActions: (metric) => [ - { - children: 'View', - onSelect: () => { - if (metric.name) navigate(getEvaluationMetricDetailsRoute(workspace, metric.name)); - }, - }, - { - children: 'Delete', - danger: true, - onSelect: () => setMetricToDelete(metric), - }, - ], - }), - ], - [navigate, workspace] - ); - - const handleDeleteConfirm = async () => { - if (!metricToDelete?.name) return false; - try { - await deleteMetric({ workspace, name: metricToDelete.name }); - invalidateMetrics(); - return true; - } catch { - return false; - } - }; - - return ( - - - dataViewState={dataViewState} - searchField="name" - makeColumns={makeColumns} - onRowClick={onRowClick} - renderBulkActions={({ selectedRows }) => ( - { - invalidateMetrics(); - dataViewState.rowSelection.set({}); - }} - /> - )} - attributes={{ - DataViewSearchBar: { - placeholder: 'Search Metric Name', - }, - DataViewRoot: { - data: evaluationMetrics, - totalCount: evaluationsData?.pagination?.total_results, - requestStatus: error ? 'error' : isLoading ? 'loading' : undefined, - }, - DataViewTableContent: { - renderEmptyState: () => - hasActiveFilters ? ( - dataViewState.resetFilters()}> - Clear Filters - - } - /> - ) : ( - navigate(getNewEvaluationMetricRoute(workspace))} - > - New Metric - - } - /> - ), - renderErrorState: () => ( - refetch()}> - Retry - - } - /> - ), - }, - }} - /> - setMetricToDelete(null)} - onDelete={handleDeleteConfirm} - title="Delete Metric" - description={`Are you sure you want to delete "${metricToDelete?.name}"? This action cannot be undone.`} - simpleConfirm - successText="Successfully deleted evaluation metric" - errorText="Failed to delete evaluation metric. Please try again." - /> - - ); -}; diff --git a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts index 297e17ccd31..76fd02ca11c 100644 --- a/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts +++ b/web/packages/studio/src/components/dataViews/EvaluationMetricsDataView/types.ts @@ -1,22 +1,19 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { DatetimeFilter, MetricsListResponse } from '@nemo/sdk/generated/platform/schema'; - -/** Filter fields for the metrics list API. */ -export interface EvaluationMetricsFilterFields { +/** A metric list item augmented with a stable `id` field for table row keying. */ +export interface MetricItemWithId { + id: string; name?: string; description?: string; type?: string; - project?: string; - created_at?: DatetimeFilter; - updated_at?: DatetimeFilter; + created_at?: string; + updated_at?: string; + workspace?: string; + [key: string]: unknown; } export interface EvaluationMetricsDataViewProps { workspace: string; onRowClick?: (metric: MetricItemWithId) => void; } - -export type MetricItem = MetricsListResponse['data'][number]; -export type MetricItemWithId = MetricItem & { id: string }; diff --git a/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.spec.tsx b/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.spec.tsx index 1297dc8632b..ba7ea9b6a84 100644 --- a/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.spec.tsx +++ b/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.spec.tsx @@ -4,31 +4,22 @@ import { DEFAULT_WORKSPACE } from '@nemo/common/src/models/constants'; import { ButtonLaunchEvaluation } from '@studio/components/evaluation/ButtonLaunchEvaluation'; import { ROUTE_PARAMS, ROUTES } from '@studio/constants/routes'; -import { getEvaluationMetricRunRoute, getEvaluationMetricsRunRoute } from '@studio/routes/utils'; +import { getEvaluationResultsRoute } from '@studio/routes/utils'; import { LOCATION_DISPLAY_TEST_ID } from '@studio/tests/util/constants'; import { LocationDisplay } from '@studio/tests/util/LocationDisplay'; import { mockUseParams } from '@studio/tests/util/mockUseParams'; import { TestProviders } from '@studio/tests/util/TestProviders'; import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; -import { ComponentProps } from 'react'; import { createMemoryRouter, RouterProvider } from 'react-router-dom'; -const renderRoute = ({ - buttonProps, -}: { buttonProps?: ComponentProps } = {}) => { +const renderRoute = () => { const router = createMemoryRouter( [ - { path: ROUTES.workspace.index, element: }, - { - path: getEvaluationMetricRunRoute(DEFAULT_WORKSPACE, 'metric-name'), - element: , - }, - { path: getEvaluationMetricsRunRoute(DEFAULT_WORKSPACE), element: }, + { path: ROUTES.workspace.index, element: }, + { path: getEvaluationResultsRoute(DEFAULT_WORKSPACE), element: }, ], - { - initialEntries: [ROUTES.workspace.index], - } + { initialEntries: [ROUTES.workspace.index] } ); return render( @@ -39,76 +30,35 @@ const renderRoute = ({ describe('ButtonLaunchEvaluation', () => { beforeEach(() => { - mockUseParams({ - [ROUTE_PARAMS.workspace]: DEFAULT_WORKSPACE, - }); + mockUseParams({ [ROUTE_PARAMS.workspace]: DEFAULT_WORKSPACE }); }); + it('should render the button with correct text', async () => { renderRoute(); - expect(await screen.findByText('Launch Evaluation')).toBeInTheDocument(); expect(screen.getByRole('button')).toBeInTheDocument(); }); - it('should navigate to correct evaluation launch route when clicked', async () => { + it('should navigate to evaluation results when clicked', async () => { const user = userEvent.setup(); - const metricRef = 'metric-name'; - renderRoute({ buttonProps: { metricRef } }); + renderRoute(); - const launchButton = await screen.findByText('Launch Evaluation'); - await user.click(launchButton); + await user.click(await screen.findByText('Launch Evaluation')); - // Verify we navigated to the correct URL const locationElement = await screen.findByTestId(LOCATION_DISPLAY_TEST_ID); - expect(locationElement).toHaveTextContent( - getEvaluationMetricRunRoute(DEFAULT_WORKSPACE, metricRef) - ); + expect(locationElement).toHaveTextContent(getEvaluationResultsRoute(DEFAULT_WORKSPACE)); }); - it('should navigate to the generic launch route with model deep linking', async () => { - const user = userEvent.setup(); - const modelRef = `${DEFAULT_WORKSPACE}/model-name`; - renderRoute({ buttonProps: { modelRef } }); - - const launchButton = await screen.findByText('Launch Evaluation'); - await user.click(launchButton); - - const locationElement = await screen.findByTestId(LOCATION_DISPLAY_TEST_ID); - expect(locationElement).toHaveTextContent( - getEvaluationMetricsRunRoute(DEFAULT_WORKSPACE, { model: modelRef }) + it('should pass through additional props', async () => { + const router = createMemoryRouter( + [{ path: ROUTES.workspace.index, element: }], + { initialEntries: [ROUTES.workspace.index] } ); - }); - - it('should navigate to the metric launch route with model deep linking', async () => { - const user = userEvent.setup(); - const metricRef = 'metric-name'; - const modelRef = `${DEFAULT_WORKSPACE}/model-name`; - renderRoute({ buttonProps: { metricRef, modelRef } }); - - const launchButton = await screen.findByText('Launch Evaluation'); - await user.click(launchButton); - - const locationElement = await screen.findByTestId(LOCATION_DISPLAY_TEST_ID); - expect(locationElement).toHaveTextContent( - getEvaluationMetricRunRoute(DEFAULT_WORKSPACE, metricRef, { model: modelRef }) + render( + + + ); - }); - - it('should pass through additional props', async () => { - renderRoute({ buttonProps: { disabled: true } }); - - const button = await screen.findByRole('button'); - expect(button).toBeDisabled(); - }); - - it('should navigate to the correct evaluation launch route when metricRef is not provided', async () => { - const user = userEvent.setup(); - renderRoute(); - - const launchButton = await screen.findByText('Launch Evaluation'); - await user.click(launchButton); - - const locationElement = await screen.findByTestId(LOCATION_DISPLAY_TEST_ID); - expect(locationElement).toHaveTextContent(getEvaluationMetricsRunRoute(DEFAULT_WORKSPACE)); + expect(await screen.findByRole('button')).toBeDisabled(); }); }); diff --git a/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.tsx b/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.tsx index 2ccbe79d535..0c099496a6f 100644 --- a/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.tsx +++ b/web/packages/studio/src/components/evaluation/ButtonLaunchEvaluation.tsx @@ -3,42 +3,24 @@ import { Button, ButtonProps } from '@nvidia/foundations-react-core'; import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { getEvaluationMetricRunRoute, getEvaluationMetricsRunRoute } from '@studio/routes/utils'; +import { getEvaluationResultsRoute } from '@studio/routes/utils'; import { useNavigate } from 'react-router-dom'; -type ButtonLaunchEvaluationProps = ButtonProps & { - /** Optional metric reference (namespace/name) to pre-fill in the form */ - metricRef?: string; - /** Optional model reference to pre-fill in the metric run side panel */ - modelRef?: string; -}; - /** - * Button that navigates to the evaluation launch form. - * Optionally accepts a metricRef and modelRef to pre-fill the run side panel. + * Button that navigates to the evaluation results page. */ export const ButtonLaunchEvaluation = ({ - metricRef, - modelRef, children = 'Launch Evaluation', ...buttonProps -}: ButtonLaunchEvaluationProps) => { +}: ButtonProps) => { const navigate = useNavigate(); const workspace = useWorkspaceFromPath(); - const handleLaunchEvaluationsClick = () => { - if (metricRef) { - navigate(getEvaluationMetricRunRoute(workspace, metricRef, { model: modelRef })); - } else { - navigate(getEvaluationMetricsRunRoute(workspace, { model: modelRef })); - } - }; - return ( - - ); - -describe('MetricFormPanels', () => { - beforeEach(() => { - mockUseParams({ [ROUTE_PARAMS.workspace]: workspace1.name }); - mockUseJudgeModels.mockReturnValue({ - data: [], - isLoading: false, - error: undefined, - }); - }); - - describe('Left panel', () => { - it('renders metric name, description, metric type, and score sections', async () => { - renderPanel(); - - // KUI renders the label text in multiple elements; check at least one exists - expect((await screen.findAllByText('Metric Name')).length).toBeGreaterThan(0); - expect(screen.getByText('Metric Type')).toBeInTheDocument(); - expect(screen.getByText('Score Definitions')).toBeInTheDocument(); - expect(screen.getByText('LLM-as-a-Judge')).toBeInTheDocument(); - }); - }); - - describe('Default judge model', () => { - it('prefills judge model when the workspace lists the default build model', async () => { - const expectedUrn = `${workspace1.name}/${DEFAULT_BUILD_MODEL_NAME}`; - mockUseJudgeModels.mockReturnValue({ - data: [ - { - id: 'model-default-build', - workspace: workspace1.name, - name: DEFAULT_BUILD_MODEL_NAME, - created_at: '2024-01-01T00:00:00.000Z', - updated_at: '2024-01-01T00:00:00.000Z', - api_endpoint: { url: 'https://example.com/v1/chat/completions', format: 'nim' }, - }, - ], - isLoading: false, - error: undefined, - }); - - renderPanel(); - - await waitFor(() => { - expect(screen.getByTestId('judge-model-input')).toHaveValue(expectedUrn); - }); - }); - - it('does not prefill judge model when the default build model is not in the list', async () => { - mockUseJudgeModels.mockReturnValue({ - data: [ - { - id: 'model-other', - workspace: workspace1.name, - name: 'some-other-model', - created_at: '2024-01-01T00:00:00.000Z', - updated_at: '2024-01-01T00:00:00.000Z', - api_endpoint: { url: 'https://example.com/v1/chat/completions', format: 'nim' }, - }, - ], - isLoading: false, - error: undefined, - }); - - renderPanel(); - - await screen.findByTestId('judge-model-input'); - expect(screen.getByTestId('judge-model-input')).toHaveValue(''); - }); - }); - - describe('Right panel tabs', () => { - it('shows Configure tab content by default', async () => { - renderPanel(); - - // KUI SegmentedControl renders items as role="radio" inside a radiogroup - expect(await screen.findByRole('radio', { name: 'Configure' })).toBeInTheDocument(); - expect(screen.getByRole('radio', { name: 'Test' })).toBeInTheDocument(); - expect(screen.getByText('Model & Prompt')).toBeInTheDocument(); - }); - - it('switches to Test tab and shows test panel', async () => { - const user = userEvent.setup(); - renderPanel(); - - await screen.findByRole('radio', { name: 'Test' }); - await user.click(screen.getByRole('radio', { name: 'Test' })); - - await waitFor(() => { - expect(screen.getByText('Run Test')).toBeInTheDocument(); - }); - }); - }); - - it('requires confirmation again after editing a successfully tested metric', async () => { - const user = userEvent.setup(); - const onSubmit = vi.fn(); - renderPanel(onSubmit); - - await user.type(screen.getByRole('textbox', { name: 'Metric Name' }), 'response-quality'); - await user.type(await screen.findByTestId('judge-model-input'), 'test-model'); - await user.click(screen.getByRole('radio', { name: 'Test' })); - await user.click(await screen.findByRole('button', { name: 'Run Test' })); - - await user.click(screen.getByRole('button', { name: 'Save Metric' })); - - await waitFor(() => { - expect(onSubmit).toHaveBeenCalledTimes(1); - }); - expect(screen.queryByText('Save without Testing?')).not.toBeInTheDocument(); - - await user.click(screen.getByRole('radio', { name: 'Configure' })); - await user.type(screen.getByTestId('judge-model-input'), '-edited'); - await user.click(screen.getByRole('button', { name: 'Save Metric' })); - - expect(await screen.findByText('Save without Testing?')).toBeInTheDocument(); - expect(onSubmit).toHaveBeenCalledTimes(1); - }); -}); diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx deleted file mode 100644 index 78195840699..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricFormPanels.tsx +++ /dev/null @@ -1,129 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { getEntityReference, getURNFromNamedEntityRef } from '@nemo/common/src/namedEntity'; -import { useToast } from '@nemo/common/src/providers/toast/useToast'; -import { ResizeablePanel } from '@studio/components/common/ResizeablePanel'; -import { MetricDetailsPanel } from '@studio/components/evaluation/Jobs/form/MetricDetailsPanel'; -import { MetricLLMJudgePanel } from '@studio/components/evaluation/Jobs/form/MetricLLMJudgePanel'; -import { ConfirmationModal } from '@studio/components/modals/ConfirmationModal'; -import { DEFAULT_BUILD_MODEL_NAME } from '@studio/constants/constants'; -import { useJudgeModels } from '@studio/hooks/evaluation/useJudgeModels'; -import { - type MetricPanelFormData, - useMetricPanelForm, -} from '@studio/hooks/evaluation/useMetricPanelForm'; -import { useWorkspaceFromPath } from '@studio/hooks/useWorkspaceFromPath'; -import { handleFormErrorsGeneric } from '@studio/util/forms/error'; -import { type FC, useEffect, useMemo, useRef, useState } from 'react'; -import { type FieldErrors, FormProvider } from 'react-hook-form'; - -async function resolveSubmitSuccess( - result: boolean | Promise | void | Promise -): Promise { - const settled = await result; - return settled !== false; -} - -interface MetricFormPanelsProps { - /** Return false when the save failed after showing your own error UI; otherwise return true or void. */ - onSubmit: (data: MetricPanelFormData) => boolean | Promise | void | Promise; - formId?: string; -} - -export const MetricFormPanels: FC = ({ onSubmit, formId }) => { - const form = useMetricPanelForm(); - const workspace = useWorkspaceFromPath(); - const { data: judgeModels = [], isLoading: judgeModelsLoading } = useJudgeModels({ - enabled: !!workspace, - }); - const defaultJudgeModelUrn = useMemo(() => { - if (!workspace) return ''; - const candidate = getEntityReference({ workspace, name: DEFAULT_BUILD_MODEL_NAME }); - return judgeModels.some((m) => getURNFromNamedEntityRef(m) === candidate) ? candidate : ''; - }, [workspace, judgeModels]); - - const defaultJudgeModelAppliedRef = useRef(false); - const toast = useToast(); - const [hasSuccessfulTestRun, setHasSuccessfulTestRun] = useState(false); - const [saveWithoutTestOpen, setSaveWithoutTestOpen] = useState(false); - const [pendingSubmitData, setPendingSubmitData] = useState(null); - - const { handleSubmit, watch, getValues, setValue } = form; - - useEffect(() => { - if (judgeModelsLoading || !defaultJudgeModelUrn || defaultJudgeModelAppliedRef.current) { - return; - } - const currentName = getValues('body.model.name'); - if (!currentName) { - setValue('body.model.name', defaultJudgeModelUrn, { - shouldDirty: false, - shouldValidate: true, - }); - defaultJudgeModelAppliedRef.current = true; - } - }, [judgeModelsLoading, defaultJudgeModelUrn, getValues, setValue]); - - useEffect(() => { - const subscription = watch(() => { - setHasSuccessfulTestRun(false); - }); - return () => subscription.unsubscribe(); - }, [watch]); - - const onError = (errors: FieldErrors) => { - toast.error( - 'There was an error with the form submission. Please fix any errors and try again.' - ); - handleFormErrorsGeneric({ title: 'Evaluation Metric' })(errors); - }; - - const flushSubmit = async (data: MetricPanelFormData) => resolveSubmitSuccess(onSubmit(data)); - - const trySubmit = (data: MetricPanelFormData) => { - if (!hasSuccessfulTestRun) { - setPendingSubmitData(data); - setSaveWithoutTestOpen(true); - return; - } - void flushSubmit(data); - }; - - return ( - <> - -
- } - slotRight={ - setHasSuccessfulTestRun(true)} /> - } - /> - -
- { - setSaveWithoutTestOpen(false); - setPendingSubmitData(null); - }} - title="Save without Testing?" - description="You have not run a successful test from the Test tab. You can still save this metric; run a test first if you want to preview results with your sample data." - submitButtonText="Save" - onConfirm={async () => { - if (!pendingSubmitData) return false; - return flushSubmit(pendingSubmitData); - }} - suppressResultToasts - /> - - ); -}; diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx deleted file mode 100644 index 96f57f3424c..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricLLMJudgePanel.tsx +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { ChatCompletionInput } from '@nemo/common/src/components/ChatCompletionInput'; -import { SegmentedControl, Stack, Text } from '@nvidia/foundations-react-core'; -import { MetricTestPanel } from '@studio/components/evaluation/Jobs/form/MetricTestPanel'; -import { JudgeModelSelect } from '@studio/components/evaluation/JudgeModelSelect'; -import { mergeInferenceParamsFormValues } from '@studio/hooks/evaluation/useCreateConfigurationForm'; -import { type MetricPanelFormData } from '@studio/hooks/evaluation/useMetricPanelForm'; -import { type FC, useState } from 'react'; -import { useFieldArray, useFormContext } from 'react-hook-form'; - -type ActivePanel = 'configure' | 'test'; - -const SEGMENTED_CONTROL_ITEMS = [ - { value: 'configure', children: 'Configure' }, - { value: 'test', children: 'Test' }, -]; - -interface MetricRightPanelProps { - onSuccessfulTestRun: () => void; -} - -export const MetricLLMJudgePanel: FC = ({ onSuccessfulTestRun }) => { - const { control, watch, setValue } = useFormContext(); - const [activePanel, setActivePanel] = useState('configure'); - const { fields, append, remove, move } = useFieldArray({ control, name: 'body.messages' }); - const inferenceParams = watch('body.inference'); - - return ( - <> - setActivePanel(v as ActivePanel)} - items={SEGMENTED_CONTROL_ITEMS} - className="w-full shrink-0" - /> - - {/* - Keep both panels mounted so MetricTestPanel local state (sample data, file selection, - results) survives switching Configure ↔ Test. Use `hidden` instead of conditional render. - */} - - - Model & Prompt - - Provide instructions for the judge, with variables that get filled from your dataset. - Customize the judge prompt to match your evaluation criteria. - - - - - - formFieldName="body.model.name" - required - showParams - inferenceParams={inferenceParams} - onInferenceParamsChange={(params) => - setValue('body.inference', mergeInferenceParamsFormValues(inferenceParams, params)) - } - /> - - - {fields.map((field, index) => ( - - key={field.id} - control={control} - name={`body.messages.${index}`} - fieldArrayLength={fields.length} - onMoveUp={index > 0 ? () => move(index, index - 1) : undefined} - onMoveDown={index < fields.length - 1 ? () => move(index, index + 1) : undefined} - onDuplicate={() => - append( - { role: field.role, content: field.content, expanded: true }, - { shouldFocus: false } - ) - } - onRemove={() => remove(index)} - allowRemove={fields.length > 1} - /> - ))} - - - - -
- -
- - ); -}; diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx deleted file mode 100644 index ca58280ad08..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.spec.tsx +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { MetricSelector } from '@studio/components/evaluation/Jobs/form/MetricSelector'; -import { render, screen } from '@testing-library/react'; -import { FormProvider, useForm } from 'react-hook-form'; - -function TestWrapper({ children }: { children: React.ReactNode }) { - const methods = useForm(); - return {children}; -} - -describe('MetricSelector', () => { - it('should render without crashing when no metric type is selected', () => { - const { container } = render( - - - - ); - - expect(container).toBeInTheDocument(); - }); - - it('should show LLM-as-a-Judge form fields when llm-judge type is selected', () => { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { - metricType: 'llm-judge', - metricConfig: {}, - }, - }); - return ( - - - - ); - }; - - render(); - - // Verify required form fields are displayed - expect(screen.getByLabelText(/Score Name/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Minimum Value/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Maximum Value/)).toBeInTheDocument(); - - // Verify optional fields - expect(screen.getByLabelText(/Score Description/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Prompt Template/)).toBeInTheDocument(); - expect(screen.getByLabelText(/Parser Pattern/)).toBeInTheDocument(); - }); - - it('should display required field markers for LLM-as-a-Judge', () => { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { - metricType: 'llm-judge', - metricConfig: {}, - }, - }); - return ( - - - - ); - }; - - render(); - - // Verify required fields are marked via KUI's required class - const requiredFields = screen - .getAllByTestId('nv-form-field-root') - .filter((el) => el.classList.contains('nv-form-field-root--required')); - - // Should have at least 3 required fields: Score Name, Minimum, Maximum - expect(requiredFields.length).toBeGreaterThanOrEqual(3); - }); - - it('should show correct placeholder text for LLM judge fields', () => { - const Wrapper = () => { - const methods = useForm({ - defaultValues: { - metricType: 'llm-judge', - metricConfig: {}, - }, - }); - return ( - - - - ); - }; - - render(); - - // Verify placeholder text - expect(screen.getByPlaceholderText('quality')).toBeInTheDocument(); - expect(screen.getByPlaceholderText('Overall quality of the response')).toBeInTheDocument(); - }); -}); diff --git a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx b/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx deleted file mode 100644 index a235800d9b6..00000000000 --- a/web/packages/studio/src/components/evaluation/Jobs/form/MetricSelector.tsx +++ /dev/null @@ -1,204 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import type { - LLMJudgeMetric, - MetricType as SDKMetricType, -} from '@nemo/sdk/generated/platform/schema'; -import { FormField, Stack, TextArea, TextInput } from '@nvidia/foundations-react-core'; -import { FC } from 'react'; -import { Controller, useFormContext } from 'react-hook-form'; - -// Subset of metrics currently supported in the UI -export type MetricType = Extract; - -export type MetricConfig = LLMJudgeMetric; - -interface MetricSelectorProps { - workspace: string; -} - -/** - * Component for selecting and configuring evaluation metrics - */ -export const MetricSelector: FC = () => { - const { control, watch } = useFormContext(); - const metricType = watch('metricType') as MetricType | undefined; - - return ( - - {metricType === 'llm-judge' && ( - - ( - - {(props) => ( - - )} - - )} - /> - - ( - - {(props) => ( - - )} - - )} - /> - -
- ( - - {(props) => ( - field.onChange(v === '' ? undefined : Number(v))} - attributes={{ Input: { placeholder: '1' } }} - /> - )} - - )} - /> - - ( - - {(props) => ( - field.onChange(v === '' ? undefined : Number(v))} - attributes={{ Input: { placeholder: '5' } }} - /> - )} - - )} - /> -
- - ( - - Custom prompt for the judge. Use template variables like{' '} - {'{{input}}'} and {'{{output}}'}. Leave empty for - auto-generated prompt. - - } - > - {(props) => ( -