diff --git a/specification/ai/Azure.AI.Projects/evaluation-results/models.tsp b/specification/ai/Azure.AI.Projects/evaluation-results/models.tsp deleted file mode 100644 index b4a941c17048..000000000000 --- a/specification/ai/Azure.AI.Projects/evaluation-results/models.tsp +++ /dev/null @@ -1,66 +0,0 @@ -import "@typespec/rest"; -import "@azure-tools/typespec-autorest"; -import "@typespec/versioning"; -import "@azure-tools/typespec-azure-core"; -import "../common/models.tsp"; -import "../main.tsp"; - -using TypeSpec.OpenAPI; -using TypeSpec.Http; -using TypeSpec.Rest; -using TypeSpec.Versioning; -using Azure.Core; -using Azure.Core.Traits; - -namespace Azure.AI.Projects; - -@doc("Evaluation Result resource Definition") -@Rest.resource("evaluationResults") -@added(Versions.v2025_05_15_preview) -@removed(Versions.v_latest) -model EvaluationResult { - @doc("Type of Evaluation result") - ResultType?: ResultType; - - @doc("Model Name") - ModelName?: string; - - @doc("Model Version") - ModelVersion?: string; - - @doc("Model Asset ID") - ModelAssetId?: string; - - @doc("Dataset Family") - DatasetFamily?: string; - - @doc("Dataset Name") - DatasetName?: string; - - @doc("Metrics") - Metrics?: Record; - - @doc("Blob URI") - BlobUri?: string; - - ...AssetBase; -} - -@doc("Type of Evaluation result") -@added(Versions.v2025_05_15_preview) -@removed(Versions.v_latest) -union ResultType { - string, - - @doc("Benchmark result") - Benchmark: "Benchmark", - - @doc("Evaluations Result") - Evaluation: "Evaluation", - - @doc("Red Team Result") - Redteam: "Redteam", - - @doc("Simulation Result") - Simulation: "Simulation", -} diff --git a/specification/ai/Azure.AI.Projects/evaluation-results/routes.tsp b/specification/ai/Azure.AI.Projects/evaluation-results/routes.tsp deleted file mode 100644 index d5a1cd0dd4ca..000000000000 --- a/specification/ai/Azure.AI.Projects/evaluation-results/routes.tsp +++ /dev/null @@ -1,48 +0,0 @@ -import "@typespec/http"; -import "@typespec/rest"; -import "@azure-tools/typespec-autorest"; -import "@typespec/versioning"; -import "@azure-tools/typespec-azure-core"; -import "../common/models.tsp"; -import "./models.tsp"; -import "../servicepatterns.tsp"; - -using TypeSpec.Http; -using TypeSpec.Versioning; - -namespace Azure.AI.Projects; - -alias ListEvaluationResultsParameters = { - @doc("Top count of results, top count cannot be greater than the page size. If topCount > page size, results with be default page size count will be returned") - @query - top?: int32; - - @doc("Continuation token for pagination.") - @query - skip?: string; - - @doc("Comma-separated list of tag names (and optionally values). Example: tag1,tag2=value2") - @query - tags?: string; - - @doc("[ListViewType.ActiveOnly, ListViewType.ArchivedOnly, ListViewType.All] View type for including/excluding (for example) archived entities.") - @query - listViewType?: ListViewType; -}; - -#suppress "@azure-tools/typespec-azure-core/use-standard-operations" "We are using service specific operation templates" -@added(Versions.v2025_05_15_preview) -@removed(Versions.v_latest) -interface EvaluationResults - extends ServicePatterns.VersionedOperations< - EvaluationResult, - ListEvaluationResultsParameters, - ListEvaluationResultsParameters - > { - @doc("Create or start a pending upload of a evaluation results for a specific version.") - startPendingUpload is ServicePatterns.VersionedResourceAction< - EvaluationResult, - PendingUploadRequest, - PendingUploadResponse - >; -} diff --git a/specification/ai/Azure.AI.Projects/evaluations/models.tsp b/specification/ai/Azure.AI.Projects/evaluations/models.tsp index 5cf18801fc14..9a25d2e7d154 100644 --- a/specification/ai/Azure.AI.Projects/evaluations/models.tsp +++ b/specification/ai/Azure.AI.Projects/evaluations/models.tsp @@ -78,7 +78,4 @@ model Evaluation { @doc("Evaluators to be used for the evaluation.") evaluators: Record; - @doc("Read-only result outputs. Example: { 'evaluationResultId': 'azureai://accounts/{AccountName}/projects/{projectName}/evaluationresults/{name}/{version}', 'logId': 'azureai://accounts/{AccountName}/projects/{projectName}/datasets/{dataset-name}/{dataset-version}' }") - @visibility(Lifecycle.Read) - outputs: Record; } diff --git a/specification/ai/Azure.AI.Projects/main.tsp b/specification/ai/Azure.AI.Projects/main.tsp index 8f3f5f61ce58..367cc5b8f00b 100644 --- a/specification/ai/Azure.AI.Projects/main.tsp +++ b/specification/ai/Azure.AI.Projects/main.tsp @@ -8,7 +8,6 @@ import "./evaluations/routes.tsp"; import "./datasets/routes.tsp"; import "./indexes/routes.tsp"; import "./deployments/routes.tsp"; -import "./evaluation-results/routes.tsp"; import "./red-teams/routes.tsp"; using TypeSpec.Http; diff --git a/specification/ai/Azure.AI.Projects/red-teams/models.tsp b/specification/ai/Azure.AI.Projects/red-teams/models.tsp index 1a045ccc66af..06a467b78809 100644 --- a/specification/ai/Azure.AI.Projects/red-teams/models.tsp +++ b/specification/ai/Azure.AI.Projects/red-teams/models.tsp @@ -99,10 +99,6 @@ model RedTeam { @doc("Simulation-only or Simulation + Evaluation. Default false, if true the scan outputs conversation not evaluation result.") simulationOnly: boolean; - @doc("Read-only result outputs. Example: { 'redTeamResultId': 'azureai://accounts/{AccountName}/projects/{projectName}/evaluationresults/{name}/{version}', 'logId': 'azureai://accounts/{AccountName}/projects/{projectName}/datasets/{dataset-name}/{dataset-version}' }") - @visibility(Lifecycle.Read) - outputs: Record; - @doc("List of risk categories to generate attack objectives for.") riskCategories: RiskCategory[]; diff --git a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json index 89e75246a568..ad26e3fac345 100644 --- a/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json +++ b/specification/ai/data-plane/Azure.AI.Projects/preview/2025-05-15-preview/azure-ai-projects-1dp.json @@ -839,473 +839,6 @@ } } }, - "/evaluationResults": { - "get": { - "operationId": "EvaluationResults_ListLatest", - "description": "List the latest version of each EvaluationResult", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "top", - "in": "query", - "description": "Top count of results, top count cannot be greater than the page size. If topCount > page size, results with be default page size count will be returned", - "required": false, - "type": "integer", - "format": "int32" - }, - { - "name": "skip", - "in": "query", - "description": "Continuation token for pagination.", - "required": false, - "type": "string" - }, - { - "name": "tags", - "in": "query", - "description": "Comma-separated list of tag names (and optionally values). Example: tag1,tag2=value2", - "required": false, - "type": "string" - }, - { - "name": "listViewType", - "in": "query", - "description": "[ListViewType.ActiveOnly, ListViewType.ArchivedOnly, ListViewType.All] View type for including/excluding (for example) archived entities.", - "required": false, - "type": "string", - "enum": [ - "ActiveOnly", - "ArchivedOnly", - "All" - ], - "x-ms-enum": { - "name": "ListViewType", - "modelAsString": true, - "values": [ - { - "name": "ActiveOnly", - "value": "ActiveOnly", - "description": "List only active items." - }, - { - "name": "ArchivedOnly", - "value": "ArchivedOnly", - "description": "List only archived items." - }, - { - "name": "All", - "value": "All", - "description": "List all items." - } - ] - } - } - ], - "responses": { - "200": { - "description": "The request has succeeded.", - "schema": { - "$ref": "#/definitions/PagedEvaluationResult" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - }, - "x-ms-pageable": { - "nextLinkName": "nextLink" - } - } - }, - "/evaluationResults/{name}/versions": { - "get": { - "operationId": "EvaluationResults_ListVersions", - "description": "List all versions of the given EvaluationResult", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "name", - "in": "path", - "description": "The name of the resource", - "required": true, - "type": "string" - }, - { - "name": "top", - "in": "query", - "description": "Top count of results, top count cannot be greater than the page size. If topCount > page size, results with be default page size count will be returned", - "required": false, - "type": "integer", - "format": "int32" - }, - { - "name": "skip", - "in": "query", - "description": "Continuation token for pagination.", - "required": false, - "type": "string" - }, - { - "name": "tags", - "in": "query", - "description": "Comma-separated list of tag names (and optionally values). Example: tag1,tag2=value2", - "required": false, - "type": "string" - }, - { - "name": "listViewType", - "in": "query", - "description": "[ListViewType.ActiveOnly, ListViewType.ArchivedOnly, ListViewType.All] View type for including/excluding (for example) archived entities.", - "required": false, - "type": "string", - "enum": [ - "ActiveOnly", - "ArchivedOnly", - "All" - ], - "x-ms-enum": { - "name": "ListViewType", - "modelAsString": true, - "values": [ - { - "name": "ActiveOnly", - "value": "ActiveOnly", - "description": "List only active items." - }, - { - "name": "ArchivedOnly", - "value": "ArchivedOnly", - "description": "List only archived items." - }, - { - "name": "All", - "value": "All", - "description": "List all items." - } - ] - } - } - ], - "responses": { - "200": { - "description": "The request has succeeded.", - "schema": { - "$ref": "#/definitions/PagedEvaluationResult" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - }, - "x-ms-pageable": { - "nextLinkName": "nextLink" - } - }, - "post": { - "operationId": "EvaluationResults_Create", - "description": "Create a new EvaluationResult. The version id will be generated by the service.", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "name", - "in": "path", - "description": "The name of the resource", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/Azure.Core.RepeatabilityRequestHeaders.repeatabilityRequestId" - }, - { - "$ref": "#/parameters/Azure.Core.RepeatabilityRequestHeaders.repeatabilityFirstSent" - }, - { - "$ref": "#/parameters/Azure.Core.ClientRequestIdHeader" - }, - { - "name": "body", - "in": "body", - "description": "The definition of the EvaluationResult to create", - "required": true, - "schema": { - "$ref": "#/definitions/EvaluationResult" - } - } - ], - "responses": { - "200": { - "description": "The request has succeeded.", - "schema": { - "$ref": "#/definitions/EvaluationResult" - }, - "headers": { - "Repeatability-Result": { - "type": "string", - "description": "Indicates whether the repeatable request was accepted or rejected.", - "enum": [ - "accepted", - "rejected" - ], - "x-ms-enum": { - "name": "RepeatabilityResult", - "modelAsString": false, - "values": [ - { - "name": "accepted", - "value": "accepted", - "description": "If the request was accepted and the server guarantees that the server state reflects a single execution of the operation." - }, - { - "name": "rejected", - "value": "rejected", - "description": "If the request was rejected because the combination of Repeatability-First-Sent and Repeatability-Request-ID were invalid\nor because the Repeatability-First-Sent value was outside the range of values held by the server." - } - ] - } - }, - "x-ms-client-request-id": { - "type": "string", - "format": "uuid", - "description": "An opaque, globally-unique, client-generated string identifier for the request." - } - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - } - } - }, - "/evaluationResults/{name}/versions/{version}": { - "get": { - "operationId": "EvaluationResults_GetVersion", - "description": "Get the specific version of the EvaluationResult", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "name", - "in": "path", - "description": "The name of the resource", - "required": true, - "type": "string" - }, - { - "name": "version", - "in": "path", - "description": "The specific version id of the EvaluationResult to retrieve.", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "The request has succeeded.", - "schema": { - "$ref": "#/definitions/EvaluationResult" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - } - }, - "put": { - "operationId": "EvaluationResults_CreateVersion", - "description": "Create a new or replace an existing EvaluationResult with the given version id", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "name", - "in": "path", - "description": "The name of the resource", - "required": true, - "type": "string" - }, - { - "name": "version", - "in": "path", - "description": "The specific version id of the EvaluationResult to create or replace.", - "required": true, - "type": "string" - }, - { - "name": "body", - "in": "body", - "description": "The definition of the EvaluationResult to create", - "required": true, - "schema": { - "$ref": "#/definitions/EvaluationResult" - } - } - ], - "responses": { - "200": { - "description": "The request has succeeded.", - "schema": { - "$ref": "#/definitions/EvaluationResult" - } - }, - "201": { - "description": "The request has succeeded and a new resource has been created as a result.", - "schema": { - "$ref": "#/definitions/EvaluationResult" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - } - }, - "delete": { - "operationId": "EvaluationResults_DeleteVersion", - "description": "Delete the specific version of the EvaluationResult", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "name", - "in": "path", - "description": "The name of the resource", - "required": true, - "type": "string" - }, - { - "name": "version", - "in": "path", - "description": "The version of the EvaluationResult to delete.", - "required": true, - "type": "string" - } - ], - "responses": { - "204": { - "description": "There is no content to send for this request, but the headers may be useful." - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - } - } - }, - "/evaluationResults/{name}/versions/{version}/startPendingUpload": { - "post": { - "operationId": "EvaluationResults_StartPendingUpload", - "description": "Create or start a pending upload of a evaluation results for a specific version.", - "parameters": [ - { - "$ref": "#/parameters/Azure.Core.Foundations.ApiVersionParameter" - }, - { - "name": "name", - "in": "path", - "description": "The name of the resource", - "required": true, - "type": "string" - }, - { - "name": "version", - "in": "path", - "description": "The specific version id of the EvaluationResult to operate on.", - "required": true, - "type": "string" - }, - { - "name": "body", - "in": "body", - "description": "Parameters for the action", - "required": true, - "schema": { - "$ref": "#/definitions/PendingUploadRequest" - } - } - ], - "responses": { - "200": { - "description": "The request has succeeded.", - "schema": { - "$ref": "#/definitions/PendingUploadResponse" - } - }, - "default": { - "description": "An unexpected error response.", - "schema": { - "$ref": "#/definitions/Azure.Core.Foundations.ErrorResponse" - }, - "headers": { - "x-ms-error-code": { - "type": "string", - "description": "String error code indicating what went wrong." - } - } - } - } - } - }, "/evaluations/runs": { "get": { "operationId": "Evaluations_List", @@ -2540,102 +2073,12 @@ "additionalProperties": { "$ref": "#/definitions/EvaluatorConfiguration" } - }, - "outputs": { - "type": "object", - "description": "Read-only result outputs. Example: { 'evaluationResultId': 'azureai://accounts/{AccountName}/projects/{projectName}/evaluationresults/{name}/{version}', 'logId': 'azureai://accounts/{AccountName}/projects/{projectName}/datasets/{dataset-name}/{dataset-version}' }", - "additionalProperties": { - "type": "string" - }, - "readOnly": true } }, "required": [ "id", "data", - "evaluators", - "outputs" - ] - }, - "EvaluationResult": { - "type": "object", - "description": "Evaluation Result resource Definition", - "properties": { - "ResultType": { - "$ref": "#/definitions/ResultType", - "description": "Type of Evaluation result" - }, - "ModelName": { - "type": "string", - "description": "Model Name" - }, - "ModelVersion": { - "type": "string", - "description": "Model Version" - }, - "ModelAssetId": { - "type": "string", - "description": "Model Asset ID" - }, - "DatasetFamily": { - "type": "string", - "description": "Dataset Family" - }, - "DatasetName": { - "type": "string", - "description": "Dataset Name" - }, - "Metrics": { - "type": "object", - "description": "Metrics", - "additionalProperties": { - "format": "double", - "type": "number" - } - }, - "BlobUri": { - "type": "string", - "description": "Blob URI" - }, - "stage": { - "type": "string", - "description": "Asset stage", - "x-ms-mutability": [ - "read", - "update", - "create" - ] - }, - "id": { - "type": "string", - "description": "A unique identifier for the asset, assetId probably?", - "readOnly": true - }, - "name": { - "type": "string", - "description": "The name of the resource", - "readOnly": true - }, - "version": { - "type": "string", - "description": "The version of the resource", - "readOnly": true - }, - "description": { - "type": "string", - "description": "The asset description text." - }, - "tags": { - "type": "object", - "description": "Tag dictionary. Tags can be added, removed, and updated.", - "additionalProperties": { - "type": "string" - } - } - }, - "required": [ - "name", - "version" + "evaluators" ] }, "EvaluatorConfiguration": { @@ -3016,27 +2459,6 @@ "value" ] }, - "PagedEvaluationResult": { - "type": "object", - "description": "Paged collection of EvaluationResult items", - "properties": { - "value": { - "type": "array", - "description": "The EvaluationResult items on this page", - "items": { - "$ref": "#/definitions/EvaluationResult" - } - }, - "nextLink": { - "type": "string", - "format": "uri", - "description": "The link to the next page of items" - } - }, - "required": [ - "value" - ] - }, "PagedIndex": { "type": "object", "description": "Paged collection of Index items", @@ -3210,14 +2632,6 @@ "type": "boolean", "description": "Simulation-only or Simulation + Evaluation. Default false, if true the scan outputs conversation not evaluation result." }, - "outputs": { - "type": "object", - "description": "Read-only result outputs. Example: { 'redTeamResultId': 'azureai://accounts/{AccountName}/projects/{projectName}/evaluationresults/{name}/{version}', 'logId': 'azureai://accounts/{AccountName}/projects/{projectName}/datasets/{dataset-name}/{dataset-version}' }", - "additionalProperties": { - "type": "string" - }, - "readOnly": true - }, "riskCategories": { "type": "array", "description": "List of risk categories to generate attack objectives for.", @@ -3255,46 +2669,9 @@ "numTurns", "attackStrategy", "simulationOnly", - "outputs", "riskCategories" ] }, - "ResultType": { - "type": "string", - "description": "Type of Evaluation result", - "enum": [ - "Benchmark", - "Evaluation", - "Redteam", - "Simulation" - ], - "x-ms-enum": { - "name": "ResultType", - "modelAsString": true, - "values": [ - { - "name": "Benchmark", - "value": "Benchmark", - "description": "Benchmark result" - }, - { - "name": "Evaluation", - "value": "Evaluation", - "description": "Evaluations Result" - }, - { - "name": "Redteam", - "value": "Redteam", - "description": "Red Team Result" - }, - { - "name": "Simulation", - "value": "Simulation", - "description": "Simulation Result" - } - ] - } - }, "RiskCategory": { "type": "string", "description": "Risk category for the attack objective.",