From b988a41e114ef4e0fa3f22d51511f301622df22e Mon Sep 17 00:00:00 2001 From: Brian Newsom Date: Thu, 21 May 2026 12:41:07 -0600 Subject: [PATCH] feat(intake): add traces API to intake Mirrors the traces API work from the original Platform repository and includes generated SDK/CLI/auth updates plus review feedback. Signed-off-by: Brian Newsom --- .../authorization/permissions-reference.md | 1 + openapi/ga/individual/platform.openapi.yaml | 280 ++++++++- openapi/ga/openapi.yaml | 280 ++++++++- openapi/openapi.yaml | 280 ++++++++- .../cli/commands/api/intake/__init__.py | 3 +- .../api/intake/ingest/chat_completions.py | 15 +- .../cli/commands/api/intake/traces.py | 183 ++++++ .../nemo-platform/.nmpcontext/openapi.yaml | 280 ++++++++- .../nemo-platform/.nmpcontext/stainless.yaml | 9 + .../cli/commands/api/intake/__init__.py | 3 +- .../api/intake/ingest/chat_completions.py | 15 +- .../cli/commands/api/intake/traces.py | 183 ++++++ .../resources/intake/__init__.py | 14 + .../src/nemo_platform/resources/intake/api.md | 13 + .../intake/ingest/chat_completions.py | 12 +- .../nemo_platform/resources/intake/intake.py | 32 ++ .../nemo_platform/resources/intake/traces.py | 351 ++++++++++++ .../nemo_platform/types/intake/__init__.py | 6 + .../ingest/chat_completion_create_params.py | 7 +- .../src/nemo_platform/types/intake/trace.py | 65 +++ .../types/intake/trace_filter_param.py | 60 ++ .../types/intake/trace_list_params.py | 49 ++ .../types/intake/trace_retrieve_params.py | 32 ++ .../types/intake/trace_sort_field.py | 22 + .../nemo_platform/types/intake/traces_page.py | 37 ++ .../tests/api_resources/intake/test_traces.py | 305 ++++++++++ sdk/stainless.yaml | 9 + .../nmp/core/auth/assets/static-authz.yaml | 18 + services/intake/src/nmp/intake/service.py | 3 +- .../src/nmp/intake/spans/api/dependencies.py | 10 +- .../intake/src/nmp/intake/spans/api/traces.py | 141 +++++ .../nmp/intake/spans/api/traces_schemas.py | 87 +++ .../intake/src/nmp/intake/spans/domain.py | 77 ++- .../nmp/intake/spans/ingest/atif_mapping.py | 51 +- .../intake/spans/ingest/chat_completions.py | 15 +- .../intake/src/nmp/intake/spans/service.py | 30 + .../src/nmp/intake/spans/trace_repository.py | 540 ++++++++++++++++++ .../integration/spans/test_atif_ingest.py | 145 ++++- .../spans/test_chat_completions_ingest.py | 12 + .../integration/spans/test_traces_read.py | 158 +++++ services/intake/tests/test_atif_v17.py | 52 +- services/intake/tests/test_spans_schemas.py | 70 ++- services/intake/tests/test_traces_api.py | 42 ++ .../test_traces_clickhouse_repository.py | 253 ++++++++ web/packages/sdk/generated/platform/api.ts | 424 ++++++++++++++ .../schema/ChatCompletionsIngestRequest.ts | 3 +- .../generated/platform/schema/GetTraceMode.ts | 15 + .../platform/schema/GetTraceParams.ts | 16 + .../platform/schema/ListTracesMode.ts | 15 + .../platform/schema/ListTracesParams.ts | 34 ++ .../sdk/generated/platform/schema/Trace.ts | 38 ++ .../generated/platform/schema/TraceFilter.ts | 35 ++ .../platform/schema/TraceSortField.ts | 15 + .../generated/platform/schema/TracesPage.ts | 21 + .../platform/schema/TracesPageFilter.ts | 13 + .../sdk/generated/platform/schema/index.ts | 9 + .../sdk/generated/platform/zod/ingest.ts | 12 +- .../sdk/generated/platform/zod/traces.ts | 201 +++++++ 58 files changed, 5049 insertions(+), 52 deletions(-) create mode 100644 packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_retrieve_params.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_sort_field.py create mode 100644 sdk/python/nemo-platform/src/nemo_platform/types/intake/traces_page.py create mode 100644 sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py create mode 100644 services/intake/src/nmp/intake/spans/api/traces.py create mode 100644 services/intake/src/nmp/intake/spans/api/traces_schemas.py create mode 100644 services/intake/src/nmp/intake/spans/trace_repository.py create mode 100644 services/intake/tests/integration/spans/test_traces_read.py create mode 100644 services/intake/tests/test_traces_api.py create mode 100644 services/intake/tests/test_traces_clickhouse_repository.py create mode 100644 web/packages/sdk/generated/platform/schema/GetTraceMode.ts create mode 100644 web/packages/sdk/generated/platform/schema/GetTraceParams.ts create mode 100644 web/packages/sdk/generated/platform/schema/ListTracesMode.ts create mode 100644 web/packages/sdk/generated/platform/schema/ListTracesParams.ts create mode 100644 web/packages/sdk/generated/platform/schema/Trace.ts create mode 100644 web/packages/sdk/generated/platform/schema/TraceFilter.ts create mode 100644 web/packages/sdk/generated/platform/schema/TraceSortField.ts create mode 100644 web/packages/sdk/generated/platform/schema/TracesPage.ts create mode 100644 web/packages/sdk/generated/platform/schema/TracesPageFilter.ts create mode 100644 web/packages/sdk/generated/platform/zod/traces.ts diff --git a/docs/auth/authorization/permissions-reference.md b/docs/auth/authorization/permissions-reference.md index ce305b6e19..48221146ed 100644 --- a/docs/auth/authorization/permissions-reference.md +++ b/docs/auth/authorization/permissions-reference.md @@ -82,6 +82,7 @@ For token-level access restrictions, see [API Scopes](api-scopes.md). For the RB | `intake.spans.(read \| list)` | Read, list intake spans | ✓ | ✓ | ✓ | | `intake.tasks.(read \| list)` | Read, list intake tasks | ✓ | ✓ | ✓ | | `intake.tasks.(create \| update \| delete)` | Create, update, delete intake tasks | | ✓ | ✓ | +| `intake.traces.read` | Read intake traces | ✓ | ✓ | ✓ | ## Jobs API diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index e2eedea801..73b45d08e5 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -6874,6 +6874,129 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces: + get: + tags: + - Traces + summary: List Traces + operationId: list_traces_apis_intake_v2_workspaces__workspace__traces_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/TraceSortField' + default: -started_at + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/TraceFilter' + description: Filter root-span-backed traces by id, session_id, rolled-up status, + root span started_at, and root-span evaluation context fields. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/TracesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces/{id}: + get: + tags: + - Traces + summary: Get Trace + operationId: get_trace_apis_intake_v2_workspaces__workspace__traces__id__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: id + in: path + required: true + schema: + type: string + title: Id + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Trace' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/jobs/v2/execution-profiles: get: tags: @@ -12944,10 +13067,14 @@ components: $ref: '#/components/schemas/FlexibleEntryResponse' session_id: title: Session Id + description: Groups related chat-completions calls without forcing them + into the same trace. type: string trace_id: title: Trace Id - description: Defaults to session_id when omitted. + description: Opt into joining an existing trace built via OTel or ATIF. + This is not a grouping mechanism for chat-completions calls; use session_id + to group related calls. type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' @@ -29864,6 +29991,157 @@ components: - judge_model title: TopicAdherenceMetricResponse description: Response type for TopicAdherence metrics. + Trace: + properties: + id: + type: string + title: Id + root_span_id: + title: Root Span Id + type: string + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + title: Name + type: string + evaluation_context: + $ref: '#/components/schemas/SpanEvaluationContext' + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + duration_ms: + title: Duration Ms + type: number + status: + $ref: '#/components/schemas/SpanStatus' + input_tokens: + title: Input Tokens + type: integer + minimum: 0.0 + output_tokens: + title: Output Tokens + type: integer + minimum: 0.0 + cached_tokens: + title: Cached Tokens + type: integer + minimum: 0.0 + total_tokens: + title: Total Tokens + type: integer + minimum: 0.0 + cost_usd: + title: Cost Usd + type: number + cost_input_usd: + title: Cost Input Usd + type: number + cost_output_usd: + title: Cost Output Usd + type: number + span_count: + title: Span Count + type: integer + minimum: 0.0 + error_count: + title: Error Count + type: integer + minimum: 0.0 + type: object + required: + - id + - session_id + - workspace + - started_at + - status + title: Trace + TraceFilter: + properties: + id: + description: Filter by canonical Intake trace id. + title: Id + type: string + session_id: + description: Filter by session id. + title: Session Id + type: string + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: Filter by rolled-up trace status. + started_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string + evaluation_sha: + description: Filter by root-span evaluation sha. + title: Evaluation Sha + type: string + evaluation_run_id: + description: Filter by root-span evaluation run id. + title: Evaluation Run Id + type: string + dataset_id: + description: Filter by root-span dataset id. + title: Dataset Id + type: string + dataset_name: + description: Filter by root-span dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter by root-span dataset version. + title: Dataset Version + type: string + test_case_id: + description: Filter by root-span dataset test case id. + title: Test Case Id + type: string + title: TraceFilter + type: object + TraceSortField: + type: string + enum: + - started_at + - -started_at + title: TraceSortField + TracesPage: + properties: + data: + items: + $ref: '#/components/schemas/Trace' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: TracesPage TracingConfig: properties: enabled: diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index e2eedea801..73b45d08e5 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -6874,6 +6874,129 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces: + get: + tags: + - Traces + summary: List Traces + operationId: list_traces_apis_intake_v2_workspaces__workspace__traces_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/TraceSortField' + default: -started_at + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/TraceFilter' + description: Filter root-span-backed traces by id, session_id, rolled-up status, + root span started_at, and root-span evaluation context fields. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/TracesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces/{id}: + get: + tags: + - Traces + summary: Get Trace + operationId: get_trace_apis_intake_v2_workspaces__workspace__traces__id__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: id + in: path + required: true + schema: + type: string + title: Id + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Trace' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/jobs/v2/execution-profiles: get: tags: @@ -12944,10 +13067,14 @@ components: $ref: '#/components/schemas/FlexibleEntryResponse' session_id: title: Session Id + description: Groups related chat-completions calls without forcing them + into the same trace. type: string trace_id: title: Trace Id - description: Defaults to session_id when omitted. + description: Opt into joining an existing trace built via OTel or ATIF. + This is not a grouping mechanism for chat-completions calls; use session_id + to group related calls. type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' @@ -29864,6 +29991,157 @@ components: - judge_model title: TopicAdherenceMetricResponse description: Response type for TopicAdherence metrics. + Trace: + properties: + id: + type: string + title: Id + root_span_id: + title: Root Span Id + type: string + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + title: Name + type: string + evaluation_context: + $ref: '#/components/schemas/SpanEvaluationContext' + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + duration_ms: + title: Duration Ms + type: number + status: + $ref: '#/components/schemas/SpanStatus' + input_tokens: + title: Input Tokens + type: integer + minimum: 0.0 + output_tokens: + title: Output Tokens + type: integer + minimum: 0.0 + cached_tokens: + title: Cached Tokens + type: integer + minimum: 0.0 + total_tokens: + title: Total Tokens + type: integer + minimum: 0.0 + cost_usd: + title: Cost Usd + type: number + cost_input_usd: + title: Cost Input Usd + type: number + cost_output_usd: + title: Cost Output Usd + type: number + span_count: + title: Span Count + type: integer + minimum: 0.0 + error_count: + title: Error Count + type: integer + minimum: 0.0 + type: object + required: + - id + - session_id + - workspace + - started_at + - status + title: Trace + TraceFilter: + properties: + id: + description: Filter by canonical Intake trace id. + title: Id + type: string + session_id: + description: Filter by session id. + title: Session Id + type: string + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: Filter by rolled-up trace status. + started_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string + evaluation_sha: + description: Filter by root-span evaluation sha. + title: Evaluation Sha + type: string + evaluation_run_id: + description: Filter by root-span evaluation run id. + title: Evaluation Run Id + type: string + dataset_id: + description: Filter by root-span dataset id. + title: Dataset Id + type: string + dataset_name: + description: Filter by root-span dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter by root-span dataset version. + title: Dataset Version + type: string + test_case_id: + description: Filter by root-span dataset test case id. + title: Test Case Id + type: string + title: TraceFilter + type: object + TraceSortField: + type: string + enum: + - started_at + - -started_at + title: TraceSortField + TracesPage: + properties: + data: + items: + $ref: '#/components/schemas/Trace' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: TracesPage TracingConfig: properties: enabled: diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index e2eedea801..73b45d08e5 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -6874,6 +6874,129 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces: + get: + tags: + - Traces + summary: List Traces + operationId: list_traces_apis_intake_v2_workspaces__workspace__traces_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/TraceSortField' + default: -started_at + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/TraceFilter' + description: Filter root-span-backed traces by id, session_id, rolled-up status, + root span started_at, and root-span evaluation context fields. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/TracesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces/{id}: + get: + tags: + - Traces + summary: Get Trace + operationId: get_trace_apis_intake_v2_workspaces__workspace__traces__id__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: id + in: path + required: true + schema: + type: string + title: Id + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Trace' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/jobs/v2/execution-profiles: get: tags: @@ -12944,10 +13067,14 @@ components: $ref: '#/components/schemas/FlexibleEntryResponse' session_id: title: Session Id + description: Groups related chat-completions calls without forcing them + into the same trace. type: string trace_id: title: Trace Id - description: Defaults to session_id when omitted. + description: Opt into joining an existing trace built via OTel or ATIF. + This is not a grouping mechanism for chat-completions calls; use session_id + to group related calls. type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' @@ -29864,6 +29991,157 @@ components: - judge_model title: TopicAdherenceMetricResponse description: Response type for TopicAdherence metrics. + Trace: + properties: + id: + type: string + title: Id + root_span_id: + title: Root Span Id + type: string + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + title: Name + type: string + evaluation_context: + $ref: '#/components/schemas/SpanEvaluationContext' + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + duration_ms: + title: Duration Ms + type: number + status: + $ref: '#/components/schemas/SpanStatus' + input_tokens: + title: Input Tokens + type: integer + minimum: 0.0 + output_tokens: + title: Output Tokens + type: integer + minimum: 0.0 + cached_tokens: + title: Cached Tokens + type: integer + minimum: 0.0 + total_tokens: + title: Total Tokens + type: integer + minimum: 0.0 + cost_usd: + title: Cost Usd + type: number + cost_input_usd: + title: Cost Input Usd + type: number + cost_output_usd: + title: Cost Output Usd + type: number + span_count: + title: Span Count + type: integer + minimum: 0.0 + error_count: + title: Error Count + type: integer + minimum: 0.0 + type: object + required: + - id + - session_id + - workspace + - started_at + - status + title: Trace + TraceFilter: + properties: + id: + description: Filter by canonical Intake trace id. + title: Id + type: string + session_id: + description: Filter by session id. + title: Session Id + type: string + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: Filter by rolled-up trace status. + started_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string + evaluation_sha: + description: Filter by root-span evaluation sha. + title: Evaluation Sha + type: string + evaluation_run_id: + description: Filter by root-span evaluation run id. + title: Evaluation Run Id + type: string + dataset_id: + description: Filter by root-span dataset id. + title: Dataset Id + type: string + dataset_name: + description: Filter by root-span dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter by root-span dataset version. + title: Dataset Version + type: string + test_case_id: + description: Filter by root-span dataset test case id. + title: Test Case Id + type: string + title: TraceFilter + type: object + TraceSortField: + type: string + enum: + - started_at + - -started_at + title: TraceSortField + TracesPage: + properties: + data: + items: + $ref: '#/components/schemas/Trace' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: TracesPage TracingConfig: properties: enabled: diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/__init__.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/__init__.py index 27d5f7ac62..1568509980 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/__init__.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/__init__.py @@ -4,7 +4,7 @@ # NOTE: This file is auto-generated from __future__ import annotations -from nemo_platform_ext.cli.commands.api.intake import apps, entries, evaluator_results, exports, ingest, spans +from nemo_platform_ext.cli.commands.api.intake import apps, entries, evaluator_results, exports, ingest, spans, traces from nemo_platform_ext.cli.core.help_formatter import create_typer_app app = create_typer_app(name="intake", help="Intake operations") @@ -15,3 +15,4 @@ app.add_typer(exports.app, name="exports") app.add_typer(ingest.app, name="ingest") app.add_typer(spans.app, name="spans") +app.add_typer(traces.app, name="traces") diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py index b27841ac16..d7e74db383 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/ingest/chat_completions.py @@ -41,8 +41,19 @@ def create_chat_completions( ] = None, evaluation_context: Annotated[str | None, typer.Option("--evaluation-context", help="JSON string")] = None, provider: Annotated[str | None, typer.Option("--provider")] = None, - session_id: Annotated[str | None, typer.Option("--session-id")] = None, - trace_id: Annotated[str | None, typer.Option("--trace-id", help="Defaults to session_id when omitted.")] = None, + session_id: Annotated[ + str | None, + typer.Option( + "--session-id", help="Groups related chat-completions calls without forcing them into the same trace." + ), + ] = None, + trace_id: Annotated[ + str | None, + typer.Option( + "--trace-id", + help="Opt into joining an existing trace built via OTel or ATIF. This is not a grouping mechanism for chat-completions calls; use session_id to group related calls.", + ), + ] = None, input_file: Annotated[ str | None, typer.Option("--input-file", help="Path to JSON file (use '-' for stdin)", rich_help_panel="Input Options"), diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py new file mode 100644 index 0000000000..dbb711613b --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/api/intake/traces.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# NOTE: This file is auto-generated +from __future__ import annotations + +from typing import Annotated, Literal + +import typer + +from nemo_platform_ext.cli.core.api import build_kwargs, merge_filter_dict +from nemo_platform_ext.cli.core.code_generator import handle_code_generation +from nemo_platform_ext.cli.core.context import CLIContext +from nemo_platform_ext.cli.core.errors import handle_errors +from nemo_platform_ext.cli.core.formatters import Column, check_output_columns_with_format, format_output +from nemo_platform_ext.cli.core.help_formatter import collect_warnings, create_typer_app +from nemo_platform_ext.cli.core.pagination import PaginationType, fetch_all_pages, warn_if_more_pages +from nemo_platform_ext.cli.core.types import ( + EntityOutputFormatOption, + ListOutputFormatOption, + NoTruncateOption, + OutputColumnsOption, +) + +app = create_typer_app(name="traces", help="Manage traces") + + +@app.command("list") +@collect_warnings +@handle_errors +def list_traces( + ctx: typer.Context, + workspace: Annotated[str | None, typer.Option("--workspace")] = None, + filter: Annotated[ + str | None, + typer.Option( + "--filter", + metavar="FILTER_JSON", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, rolled-up status, root span started_at, and root-span evaluation context fields.", + rich_help_panel="Filter Options", + ), + ] = None, + filter_id: Annotated[str | None, typer.Option("--filter.id", rich_help_panel="Filter Options")] = None, + filter_dataset_id: Annotated[ + str | None, typer.Option("--filter.dataset-id", rich_help_panel="Filter Options") + ] = None, + filter_dataset_name: Annotated[ + str | None, typer.Option("--filter.dataset-name", rich_help_panel="Filter Options") + ] = None, + filter_dataset_version: Annotated[ + str | None, typer.Option("--filter.dataset-version", rich_help_panel="Filter Options") + ] = None, + filter_evaluation_id: Annotated[ + str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") + ] = None, + filter_evaluation_run_id: Annotated[ + str | None, typer.Option("--filter.evaluation-run-id", rich_help_panel="Filter Options") + ] = None, + filter_evaluation_sha: Annotated[ + str | None, typer.Option("--filter.evaluation-sha", rich_help_panel="Filter Options") + ] = None, + filter_session_id: Annotated[ + str | None, typer.Option("--filter.session-id", rich_help_panel="Filter Options") + ] = None, + filter_status: Annotated[str | None, typer.Option("--filter.status", rich_help_panel="Filter Options")] = None, + filter_test_case_id: Annotated[ + str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") + ] = None, + mode: Annotated[ + Literal["summary", "detailed"] | None, + typer.Option( + "--mode", + help="Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.", + ), + ] = None, + page: Annotated[int | None, typer.Option("--page", help="Page number.")] = None, + page_size: Annotated[int | None, typer.Option("--page-size", help="Page size.")] = None, + sort: Annotated[Literal["started_at", "-started_at"] | None, typer.Option("--sort")] = None, + output_format: ListOutputFormatOption = None, + no_truncate: NoTruncateOption = None, + columns: OutputColumnsOption = None, + all_pages: Annotated[bool, typer.Option("--all-pages", help="Fetch all pages")] = False, +) -> None: + """List Traces""" + state: CLIContext = ctx.obj + output_format = state.get_output_format(output_format) + + check_output_columns_with_format(columns, output_format) + + default_columns = [ + Column("name", None), + Column("workspace", None), + Column("created_at", None), + ] + if columns is None or str(columns).strip() == "default": + columns = default_columns + + kwargs = build_kwargs( + workspace=workspace, + filter=merge_filter_dict( + filter, + id=filter_id, + dataset_id=filter_dataset_id, + dataset_name=filter_dataset_name, + dataset_version=filter_dataset_version, + evaluation_id=filter_evaluation_id, + evaluation_run_id=filter_evaluation_run_id, + evaluation_sha=filter_evaluation_sha, + session_id=filter_session_id, + status=filter_status, + test_case_id=filter_test_case_id, + ), + mode=mode, + page=page, + page_size=page_size, + sort=sort, + ) + + if handle_code_generation(["intake", "traces"], "list", kwargs, output_format, state): + return + + client = state.get_client() + path_args = () + pagination_type = PaginationType.PAGE_NUMBER + if all_pages: + items = fetch_all_pages( + client.intake.traces.list, + path_args=path_args, + body_args=kwargs, + pagination_type=pagination_type, + ) + else: + items = client.intake.traces.list(*path_args, **kwargs) + + format_output( + items, + is_list=True, + output_format=output_format, + output_columns=columns, + no_truncate=state.get_no_truncate(no_truncate), + timestamp_format=state.get_timestamp_format(), + ) + if not all_pages: + warn_if_more_pages(items, pagination_type) + + +@app.command("get") +@collect_warnings +@handle_errors +def retrieve_traces( + ctx: typer.Context, + id: Annotated[str, typer.Argument()], + workspace: Annotated[str | None, typer.Option("--workspace")] = None, + mode: Annotated[ + Literal["summary", "detailed"] | None, + typer.Option( + "--mode", + help="Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.", + ), + ] = None, + output_format: EntityOutputFormatOption = None, +) -> None: + """Get Trace""" + state: CLIContext = ctx.obj + output_format = state.get_output_format(output_format) + + kwargs = build_kwargs( + workspace=workspace, + mode=mode, + ) + if handle_code_generation(["intake", "traces"], "retrieve", kwargs, output_format, state): + return + + client = state.get_client() + result = client.intake.traces.retrieve(id, **kwargs) + + format_output( + result, + is_list=False, + output_format=output_format, + no_truncate=state.get_no_truncate(), + timestamp_format=state.get_timestamp_format(), + ) diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index e2eedea801..73b45d08e5 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -6874,6 +6874,129 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces: + get: + tags: + - Traces + summary: List Traces + operationId: list_traces_apis_intake_v2_workspaces__workspace__traces_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + minimum: 1 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + maximum: 1000 + minimum: 1 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/TraceSortField' + default: -started_at + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/TraceFilter' + description: Filter root-span-backed traces by id, session_id, rolled-up status, + root span started_at, and root-span evaluation context fields. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/TracesPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/intake/v2/workspaces/{workspace}/traces/{id}: + get: + tags: + - Traces + summary: Get Trace + operationId: get_trace_apis_intake_v2_workspaces__workspace__traces__id__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: id + in: path + required: true + schema: + type: string + title: Id + - name: mode + in: query + required: false + schema: + enum: + - summary + - detailed + type: string + description: Use summary for root-span trace fields only, or detailed to + include token, cost, and span-count rollups. + default: detailed + title: Mode + description: Use summary for root-span trace fields only, or detailed to include + token, cost, and span-count rollups. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Trace' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/jobs/v2/execution-profiles: get: tags: @@ -12944,10 +13067,14 @@ components: $ref: '#/components/schemas/FlexibleEntryResponse' session_id: title: Session Id + description: Groups related chat-completions calls without forcing them + into the same trace. type: string trace_id: title: Trace Id - description: Defaults to session_id when omitted. + description: Opt into joining an existing trace built via OTel or ATIF. + This is not a grouping mechanism for chat-completions calls; use session_id + to group related calls. type: string evaluation_context: $ref: '#/components/schemas/EvaluationContext' @@ -29864,6 +29991,157 @@ components: - judge_model title: TopicAdherenceMetricResponse description: Response type for TopicAdherence metrics. + Trace: + properties: + id: + type: string + title: Id + root_span_id: + title: Root Span Id + type: string + session_id: + type: string + title: Session Id + workspace: + type: string + title: Workspace + name: + title: Name + type: string + evaluation_context: + $ref: '#/components/schemas/SpanEvaluationContext' + started_at: + type: string + format: date-time + title: Started At + ended_at: + title: Ended At + type: string + format: date-time + duration_ms: + title: Duration Ms + type: number + status: + $ref: '#/components/schemas/SpanStatus' + input_tokens: + title: Input Tokens + type: integer + minimum: 0.0 + output_tokens: + title: Output Tokens + type: integer + minimum: 0.0 + cached_tokens: + title: Cached Tokens + type: integer + minimum: 0.0 + total_tokens: + title: Total Tokens + type: integer + minimum: 0.0 + cost_usd: + title: Cost Usd + type: number + cost_input_usd: + title: Cost Input Usd + type: number + cost_output_usd: + title: Cost Output Usd + type: number + span_count: + title: Span Count + type: integer + minimum: 0.0 + error_count: + title: Error Count + type: integer + minimum: 0.0 + type: object + required: + - id + - session_id + - workspace + - started_at + - status + title: Trace + TraceFilter: + properties: + id: + description: Filter by canonical Intake trace id. + title: Id + type: string + session_id: + description: Filter by session id. + title: Session Id + type: string + status: + allOf: + - $ref: '#/components/schemas/SpanStatus' + description: Filter by rolled-up trace status. + started_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Filter by root span start timestamp. + evaluation_id: + description: Filter by root-span evaluation id. + title: Evaluation Id + type: string + evaluation_sha: + description: Filter by root-span evaluation sha. + title: Evaluation Sha + type: string + evaluation_run_id: + description: Filter by root-span evaluation run id. + title: Evaluation Run Id + type: string + dataset_id: + description: Filter by root-span dataset id. + title: Dataset Id + type: string + dataset_name: + description: Filter by root-span dataset name. + title: Dataset Name + type: string + dataset_version: + description: Filter by root-span dataset version. + title: Dataset Version + type: string + test_case_id: + description: Filter by root-span dataset test case id. + title: Test Case Id + type: string + title: TraceFilter + type: object + TraceSortField: + type: string + enum: + - started_at + - -started_at + title: TraceSortField + TracesPage: + properties: + data: + items: + $ref: '#/components/schemas/Trace' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. + type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object + type: object + required: + - data + title: TracesPage TracingConfig: properties: enabled: diff --git a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml index a13a775b7f..7584b7ba95 100644 --- a/sdk/python/nemo-platform/.nmpcontext/stainless.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/stainless.yaml @@ -1181,3 +1181,12 @@ resources: evaluator_results: methods: list: get /apis/intake/v2/workspaces/{workspace}/spans/{span_id}/evaluator-results + traces: + models: + trace: Trace + trace_filter: TraceFilter + trace_sort_field: TraceSortField + traces_page: TracesPage + methods: + list: get /apis/intake/v2/workspaces/{workspace}/traces + retrieve: get /apis/intake/v2/workspaces/{workspace}/traces/{id} diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/__init__.py index 7405f71774..0a80114be4 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/__init__.py @@ -4,7 +4,7 @@ # NOTE: This file is auto-generated from __future__ import annotations -from nemo_platform.cli.commands.api.intake import apps, entries, evaluator_results, exports, ingest, spans +from nemo_platform.cli.commands.api.intake import apps, entries, evaluator_results, exports, ingest, spans, traces from nemo_platform.cli.core.help_formatter import create_typer_app app = create_typer_app(name="intake", help="Intake operations") @@ -15,3 +15,4 @@ app.add_typer(exports.app, name="exports") app.add_typer(ingest.app, name="ingest") app.add_typer(spans.app, name="spans") +app.add_typer(traces.app, name="traces") diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py index b8e7840966..8a0ee877db 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/ingest/chat_completions.py @@ -41,8 +41,19 @@ def create_chat_completions( ] = None, evaluation_context: Annotated[str | None, typer.Option("--evaluation-context", help="JSON string")] = None, provider: Annotated[str | None, typer.Option("--provider")] = None, - session_id: Annotated[str | None, typer.Option("--session-id")] = None, - trace_id: Annotated[str | None, typer.Option("--trace-id", help="Defaults to session_id when omitted.")] = None, + session_id: Annotated[ + str | None, + typer.Option( + "--session-id", help="Groups related chat-completions calls without forcing them into the same trace." + ), + ] = None, + trace_id: Annotated[ + str | None, + typer.Option( + "--trace-id", + help="Opt into joining an existing trace built via OTel or ATIF. This is not a grouping mechanism for chat-completions calls; use session_id to group related calls.", + ), + ] = None, input_file: Annotated[ str | None, typer.Option("--input-file", help="Path to JSON file (use '-' for stdin)", rich_help_panel="Input Options"), diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py new file mode 100644 index 0000000000..8a8f273ba2 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/api/intake/traces.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# NOTE: This file is auto-generated +from __future__ import annotations + +from typing import Annotated, Literal + +import typer + +from nemo_platform.cli.core.api import build_kwargs, merge_filter_dict +from nemo_platform.cli.core.code_generator import handle_code_generation +from nemo_platform.cli.core.context import CLIContext +from nemo_platform.cli.core.errors import handle_errors +from nemo_platform.cli.core.formatters import Column, check_output_columns_with_format, format_output +from nemo_platform.cli.core.help_formatter import collect_warnings, create_typer_app +from nemo_platform.cli.core.pagination import PaginationType, fetch_all_pages, warn_if_more_pages +from nemo_platform.cli.core.types import ( + EntityOutputFormatOption, + ListOutputFormatOption, + NoTruncateOption, + OutputColumnsOption, +) + +app = create_typer_app(name="traces", help="Manage traces") + + +@app.command("list") +@collect_warnings +@handle_errors +def list_traces( + ctx: typer.Context, + workspace: Annotated[str | None, typer.Option("--workspace")] = None, + filter: Annotated[ + str | None, + typer.Option( + "--filter", + metavar="FILTER_JSON", + help="Use --filter with JSON for complex/nested queries, or --filter.FIELD options for simple fields. Both can be combined, with field options taking precedence.\nJSON-only fields:\n started_at: {gte: str, lte: str}\n\nFilter root-span-backed traces by id, session_id, rolled-up status, root span started_at, and root-span evaluation context fields.", + rich_help_panel="Filter Options", + ), + ] = None, + filter_id: Annotated[str | None, typer.Option("--filter.id", rich_help_panel="Filter Options")] = None, + filter_dataset_id: Annotated[ + str | None, typer.Option("--filter.dataset-id", rich_help_panel="Filter Options") + ] = None, + filter_dataset_name: Annotated[ + str | None, typer.Option("--filter.dataset-name", rich_help_panel="Filter Options") + ] = None, + filter_dataset_version: Annotated[ + str | None, typer.Option("--filter.dataset-version", rich_help_panel="Filter Options") + ] = None, + filter_evaluation_id: Annotated[ + str | None, typer.Option("--filter.evaluation-id", rich_help_panel="Filter Options") + ] = None, + filter_evaluation_run_id: Annotated[ + str | None, typer.Option("--filter.evaluation-run-id", rich_help_panel="Filter Options") + ] = None, + filter_evaluation_sha: Annotated[ + str | None, typer.Option("--filter.evaluation-sha", rich_help_panel="Filter Options") + ] = None, + filter_session_id: Annotated[ + str | None, typer.Option("--filter.session-id", rich_help_panel="Filter Options") + ] = None, + filter_status: Annotated[str | None, typer.Option("--filter.status", rich_help_panel="Filter Options")] = None, + filter_test_case_id: Annotated[ + str | None, typer.Option("--filter.test-case-id", rich_help_panel="Filter Options") + ] = None, + mode: Annotated[ + Literal["summary", "detailed"] | None, + typer.Option( + "--mode", + help="Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.", + ), + ] = None, + page: Annotated[int | None, typer.Option("--page", help="Page number.")] = None, + page_size: Annotated[int | None, typer.Option("--page-size", help="Page size.")] = None, + sort: Annotated[Literal["started_at", "-started_at"] | None, typer.Option("--sort")] = None, + output_format: ListOutputFormatOption = None, + no_truncate: NoTruncateOption = None, + columns: OutputColumnsOption = None, + all_pages: Annotated[bool, typer.Option("--all-pages", help="Fetch all pages")] = False, +) -> None: + """List Traces""" + state: CLIContext = ctx.obj + output_format = state.get_output_format(output_format) + + check_output_columns_with_format(columns, output_format) + + default_columns = [ + Column("name", None), + Column("workspace", None), + Column("created_at", None), + ] + if columns is None or str(columns).strip() == "default": + columns = default_columns + + kwargs = build_kwargs( + workspace=workspace, + filter=merge_filter_dict( + filter, + id=filter_id, + dataset_id=filter_dataset_id, + dataset_name=filter_dataset_name, + dataset_version=filter_dataset_version, + evaluation_id=filter_evaluation_id, + evaluation_run_id=filter_evaluation_run_id, + evaluation_sha=filter_evaluation_sha, + session_id=filter_session_id, + status=filter_status, + test_case_id=filter_test_case_id, + ), + mode=mode, + page=page, + page_size=page_size, + sort=sort, + ) + + if handle_code_generation(["intake", "traces"], "list", kwargs, output_format, state): + return + + client = state.get_client() + path_args = () + pagination_type = PaginationType.PAGE_NUMBER + if all_pages: + items = fetch_all_pages( + client.intake.traces.list, + path_args=path_args, + body_args=kwargs, + pagination_type=pagination_type, + ) + else: + items = client.intake.traces.list(*path_args, **kwargs) + + format_output( + items, + is_list=True, + output_format=output_format, + output_columns=columns, + no_truncate=state.get_no_truncate(no_truncate), + timestamp_format=state.get_timestamp_format(), + ) + if not all_pages: + warn_if_more_pages(items, pagination_type) + + +@app.command("get") +@collect_warnings +@handle_errors +def retrieve_traces( + ctx: typer.Context, + id: Annotated[str, typer.Argument()], + workspace: Annotated[str | None, typer.Option("--workspace")] = None, + mode: Annotated[ + Literal["summary", "detailed"] | None, + typer.Option( + "--mode", + help="Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.", + ), + ] = None, + output_format: EntityOutputFormatOption = None, +) -> None: + """Get Trace""" + state: CLIContext = ctx.obj + output_format = state.get_output_format(output_format) + + kwargs = build_kwargs( + workspace=workspace, + mode=mode, + ) + if handle_code_generation(["intake", "traces"], "retrieve", kwargs, output_format, state): + return + + client = state.get_client() + result = client.intake.traces.retrieve(id, **kwargs) + + format_output( + result, + is_list=False, + output_format=output_format, + no_truncate=state.get_no_truncate(), + timestamp_format=state.get_timestamp_format(), + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/__init__.py index 1725c40bd2..42cbfd158a 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/__init__.py @@ -47,6 +47,14 @@ IntakeResourceWithStreamingResponse, AsyncIntakeResourceWithStreamingResponse, ) +from .traces import ( + TracesResource, + AsyncTracesResource, + TracesResourceWithRawResponse, + AsyncTracesResourceWithRawResponse, + TracesResourceWithStreamingResponse, + AsyncTracesResourceWithStreamingResponse, +) from .entries import ( EntriesResource, AsyncEntriesResource, @@ -109,6 +117,12 @@ "AsyncSpansResourceWithRawResponse", "SpansResourceWithStreamingResponse", "AsyncSpansResourceWithStreamingResponse", + "TracesResource", + "AsyncTracesResource", + "TracesResourceWithRawResponse", + "AsyncTracesResourceWithRawResponse", + "TracesResourceWithStreamingResponse", + "AsyncTracesResourceWithStreamingResponse", "IntakeResource", "AsyncIntakeResource", "IntakeResourceWithRawResponse", diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md index 10694c0774..cb0f734079 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/api.md @@ -253,3 +253,16 @@ from nemo_platform.types.intake.spans import EvaluatorResultListResponse Methods: - client.intake.spans.evaluator_results.list(span_id, \*, workspace) -> EvaluatorResultListResponse + +## Traces + +Types: + +```python +from nemo_platform.types.intake import Trace, TraceFilter, TraceSortField, TracesPage +``` + +Methods: + +- client.intake.traces.retrieve(id, \*, workspace, \*\*params) -> Trace +- client.intake.traces.list(\*, workspace, \*\*params) -> SyncDefaultPagination[Trace] diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py index d15dd46d00..8ca19b6cef 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/ingest/chat_completions.py @@ -99,7 +99,11 @@ def create( Common optional fields: `id`, `created`, `model`, `usage`, `system_fingerprint`, etc. - trace_id: Defaults to session_id when omitted. + session_id: Groups related chat-completions calls without forcing them into the same trace. + + trace_id: Opt into joining an existing trace built via OTel or ATIF. This is not a + grouping mechanism for chat-completions calls; use session_id to group related + calls. extra_headers: Send extra headers @@ -192,7 +196,11 @@ async def create( Common optional fields: `id`, `created`, `model`, `usage`, `system_fingerprint`, etc. - trace_id: Defaults to session_id when omitted. + session_id: Groups related chat-completions calls without forcing them into the same trace. + + trace_id: Opt into joining an existing trace built via OTel or ATIF. This is not a + grouping mechanism for chat-completions calls; use session_id to group related + calls. extra_headers: Send extra headers diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/intake.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/intake.py index 4ac44a770d..e936c22349 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/intake.py +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/intake.py @@ -17,6 +17,14 @@ from __future__ import annotations +from .traces import ( + TracesResource, + AsyncTracesResource, + TracesResourceWithRawResponse, + AsyncTracesResourceWithRawResponse, + TracesResourceWithStreamingResponse, + AsyncTracesResourceWithStreamingResponse, +) from ..._compat import cached_property from .apps.apps import ( AppsResource, @@ -96,6 +104,10 @@ def ingest(self) -> IngestResource: def spans(self) -> SpansResource: return SpansResource(self._client) + @cached_property + def traces(self) -> TracesResource: + return TracesResource(self._client) + @cached_property def with_raw_response(self) -> IntakeResourceWithRawResponse: """ @@ -141,6 +153,10 @@ def ingest(self) -> AsyncIngestResource: def spans(self) -> AsyncSpansResource: return AsyncSpansResource(self._client) + @cached_property + def traces(self) -> AsyncTracesResource: + return AsyncTracesResource(self._client) + @cached_property def with_raw_response(self) -> AsyncIntakeResourceWithRawResponse: """ @@ -189,6 +205,10 @@ def ingest(self) -> IngestResourceWithRawResponse: def spans(self) -> SpansResourceWithRawResponse: return SpansResourceWithRawResponse(self._intake.spans) + @cached_property + def traces(self) -> TracesResourceWithRawResponse: + return TracesResourceWithRawResponse(self._intake.traces) + class AsyncIntakeResourceWithRawResponse: def __init__(self, intake: AsyncIntakeResource) -> None: @@ -218,6 +238,10 @@ def ingest(self) -> AsyncIngestResourceWithRawResponse: def spans(self) -> AsyncSpansResourceWithRawResponse: return AsyncSpansResourceWithRawResponse(self._intake.spans) + @cached_property + def traces(self) -> AsyncTracesResourceWithRawResponse: + return AsyncTracesResourceWithRawResponse(self._intake.traces) + class IntakeResourceWithStreamingResponse: def __init__(self, intake: IntakeResource) -> None: @@ -247,6 +271,10 @@ def ingest(self) -> IngestResourceWithStreamingResponse: def spans(self) -> SpansResourceWithStreamingResponse: return SpansResourceWithStreamingResponse(self._intake.spans) + @cached_property + def traces(self) -> TracesResourceWithStreamingResponse: + return TracesResourceWithStreamingResponse(self._intake.traces) + class AsyncIntakeResourceWithStreamingResponse: def __init__(self, intake: AsyncIntakeResource) -> None: @@ -275,3 +303,7 @@ def ingest(self) -> AsyncIngestResourceWithStreamingResponse: @cached_property def spans(self) -> AsyncSpansResourceWithStreamingResponse: return AsyncSpansResourceWithStreamingResponse(self._intake.spans) + + @cached_property + def traces(self) -> AsyncTracesResourceWithStreamingResponse: + return AsyncTracesResourceWithStreamingResponse(self._intake.traces) diff --git a/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py new file mode 100644 index 0000000000..cb25d75498 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/resources/intake/traces.py @@ -0,0 +1,351 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal + +import httpx + +from ..._types import Body, Omit, Query, Headers, NotGiven, omit, not_given +from ..._utils import path_template, maybe_transform, async_maybe_transform +from ..._compat import cached_property +from ..._resource import SyncAPIResource, AsyncAPIResource +from ..._response import ( + to_raw_response_wrapper, + to_streamed_response_wrapper, + async_to_raw_response_wrapper, + async_to_streamed_response_wrapper, +) +from ...pagination import SyncDefaultPagination, AsyncDefaultPagination +from ..._base_client import AsyncPaginator, make_request_options +from ...types.intake import TraceSortField, trace_list_params, trace_retrieve_params +from ...types.intake.trace import Trace +from ...types.intake.trace_sort_field import TraceSortField +from ...types.intake.trace_filter_param import TraceFilterParam + +__all__ = ["TracesResource", "AsyncTracesResource"] + + +class TracesResource(SyncAPIResource): + @cached_property + def with_raw_response(self) -> TracesResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return TracesResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> TracesResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return TracesResourceWithStreamingResponse(self) + + def retrieve( + self, + id: str, + *, + workspace: str | None = None, + mode: Literal["summary", "detailed"] | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> Trace: + """ + Get Trace + + Args: + mode: Use summary for root-span trace fields only, or detailed to include token, cost, + and span-count rollups. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return self._get( + path_template("/apis/intake/v2/workspaces/{workspace}/traces/{id}", workspace=workspace, id=id), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform({"mode": mode}, trace_retrieve_params.TraceRetrieveParams), + ), + cast_to=Trace, + ) + + def list( + self, + *, + workspace: str | None = None, + filter: TraceFilterParam | Omit = omit, + mode: Literal["summary", "detailed"] | Omit = omit, + page: int | Omit = omit, + page_size: int | Omit = omit, + sort: TraceSortField | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> SyncDefaultPagination[Trace]: + """ + List Traces + + Args: + filter: Filter root-span-backed traces by id, session_id, rolled-up status, root span + started_at, and root-span evaluation context fields. + + mode: Use summary for root-span trace fields only, or detailed to include token, cost, + and span-count rollups. + + page: Page number. + + page_size: Page size. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._get_api_list( + path_template("/apis/intake/v2/workspaces/{workspace}/traces", workspace=workspace), + page=SyncDefaultPagination[Trace], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "filter": filter, + "mode": mode, + "page": page, + "page_size": page_size, + "sort": sort, + }, + trace_list_params.TraceListParams, + ), + ), + model=Trace, + ) + + +class AsyncTracesResource(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncTracesResourceWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#accessing-raw-response-data-e-g-headers + """ + return AsyncTracesResourceWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncTracesResourceWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://docs.nvidia.com/nemo/microservices/latest/pysdk/index.html#with_streaming_response + """ + return AsyncTracesResourceWithStreamingResponse(self) + + async def retrieve( + self, + id: str, + *, + workspace: str | None = None, + mode: Literal["summary", "detailed"] | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> Trace: + """ + Get Trace + + Args: + mode: Use summary for root-span trace fields only, or detailed to include token, cost, + and span-count rollups. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + if not id: + raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") + return await self._get( + path_template("/apis/intake/v2/workspaces/{workspace}/traces/{id}", workspace=workspace, id=id), + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=await async_maybe_transform({"mode": mode}, trace_retrieve_params.TraceRetrieveParams), + ), + cast_to=Trace, + ) + + def list( + self, + *, + workspace: str | None = None, + filter: TraceFilterParam | Omit = omit, + mode: Literal["summary", "detailed"] | Omit = omit, + page: int | Omit = omit, + page_size: int | Omit = omit, + sort: TraceSortField | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> AsyncPaginator[Trace, AsyncDefaultPagination[Trace]]: + """ + List Traces + + Args: + filter: Filter root-span-backed traces by id, session_id, rolled-up status, root span + started_at, and root-span evaluation context fields. + + mode: Use summary for root-span trace fields only, or detailed to include token, cost, + and span-count rollups. + + page: Page number. + + page_size: Page size. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if workspace is None: + workspace = self._client._get_workspace_path_param() + if not workspace: + raise ValueError(f"Expected a non-empty value for `workspace` but received {workspace!r}") + return self._get_api_list( + path_template("/apis/intake/v2/workspaces/{workspace}/traces", workspace=workspace), + page=AsyncDefaultPagination[Trace], + options=make_request_options( + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + timeout=timeout, + query=maybe_transform( + { + "filter": filter, + "mode": mode, + "page": page, + "page_size": page_size, + "sort": sort, + }, + trace_list_params.TraceListParams, + ), + ), + model=Trace, + ) + + +class TracesResourceWithRawResponse: + def __init__(self, traces: TracesResource) -> None: + self._traces = traces + + self.retrieve = to_raw_response_wrapper( + traces.retrieve, + ) + self.list = to_raw_response_wrapper( + traces.list, + ) + + +class AsyncTracesResourceWithRawResponse: + def __init__(self, traces: AsyncTracesResource) -> None: + self._traces = traces + + self.retrieve = async_to_raw_response_wrapper( + traces.retrieve, + ) + self.list = async_to_raw_response_wrapper( + traces.list, + ) + + +class TracesResourceWithStreamingResponse: + def __init__(self, traces: TracesResource) -> None: + self._traces = traces + + self.retrieve = to_streamed_response_wrapper( + traces.retrieve, + ) + self.list = to_streamed_response_wrapper( + traces.list, + ) + + +class AsyncTracesResourceWithStreamingResponse: + def __init__(self, traces: AsyncTracesResource) -> None: + self._traces = traces + + self.retrieve = async_to_streamed_response_wrapper( + traces.retrieve, + ) + self.list = async_to_streamed_response_wrapper( + traces.list, + ) diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py index 76934f37da..df686605ac 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/__init__.py @@ -20,6 +20,7 @@ from .app import App as App from .span import Span as Span from .entry import Entry as Entry +from .trace import Trace as Trace from .usage import Usage as Usage from .apps_page import AppsPage as AppsPage from .span_kind import SpanKind as SpanKind @@ -27,6 +28,7 @@ from .spans_page import SpansPage as SpansPage from .entrys_page import EntrysPage as EntrysPage from .span_status import SpanStatus as SpanStatus +from .traces_page import TracesPage as TracesPage from .usage_param import UsageParam as UsageParam from .user_rating import UserRating as UserRating from .message_role import MessageRole as MessageRole @@ -42,19 +44,23 @@ from .evaluator_result import EvaluatorResult as EvaluatorResult from .flexible_message import FlexibleMessage as FlexibleMessage from .span_list_params import SpanListParams as SpanListParams +from .trace_sort_field import TraceSortField as TraceSortField from .app_create_params import AppCreateParams as AppCreateParams from .entry_list_params import EntryListParams as EntryListParams from .span_filter_param import SpanFilterParam as SpanFilterParam +from .trace_list_params import TraceListParams as TraceListParams from .user_action_event import UserActionEvent as UserActionEvent from .user_rating_param import UserRatingParam as UserRatingParam from .entry_filter_param import EntryFilterParam as EntryFilterParam from .entry_patch_params import EntryPatchParams as EntryPatchParams from .float_filter_param import FloatFilterParam as FloatFilterParam +from .trace_filter_param import TraceFilterParam as TraceFilterParam from .entry_context_param import EntryContextParam as EntryContextParam from .entry_create_params import EntryCreateParams as EntryCreateParams from .export_config_param import ExportConfigParam as ExportConfigParam from .user_feedback_event import UserFeedbackEvent as UserFeedbackEvent from .export_preview_params import ExportPreviewParams as ExportPreviewParams +from .trace_retrieve_params import TraceRetrieveParams as TraceRetrieveParams from .evaluator_result_event import EvaluatorResultEvent as EvaluatorResultEvent from .evaluator_results_page import EvaluatorResultsPage as EvaluatorResultsPage from .flexible_entry_request import FlexibleEntryRequest as FlexibleEntryRequest diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py index 2d660434c7..31604153a9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/ingest/chat_completion_create_params.py @@ -56,6 +56,11 @@ class ChatCompletionCreateParams(TypedDict, total=False): provider: str session_id: str + """Groups related chat-completions calls without forcing them into the same trace.""" trace_id: str - """Defaults to session_id when omitted.""" + """Opt into joining an existing trace built via OTel or ATIF. + + This is not a grouping mechanism for chat-completions calls; use session_id to + group related calls. + """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py new file mode 100644 index 0000000000..d1008ece85 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from datetime import datetime + +from ..._models import BaseModel +from .span_status import SpanStatus +from .span_evaluation_context import SpanEvaluationContext + +__all__ = ["Trace"] + + +class Trace(BaseModel): + id: str + + session_id: str + + started_at: datetime + + status: SpanStatus + + workspace: str + + cached_tokens: Optional[int] = None + + cost_input_usd: Optional[float] = None + + cost_output_usd: Optional[float] = None + + cost_usd: Optional[float] = None + + duration_ms: Optional[float] = None + + ended_at: Optional[datetime] = None + + error_count: Optional[int] = None + + evaluation_context: Optional[SpanEvaluationContext] = None + + input_tokens: Optional[int] = None + + name: Optional[str] = None + + output_tokens: Optional[int] = None + + root_span_id: Optional[str] = None + + span_count: Optional[int] = None + + total_tokens: Optional[int] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py new file mode 100644 index 0000000000..85fdf123a8 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_filter_param.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +from .span_status import SpanStatus +from ..shared_params.datetime_filter import DatetimeFilter + +__all__ = ["TraceFilterParam"] + + +class TraceFilterParam(TypedDict, total=False): + id: str + """Filter by canonical Intake trace id.""" + + dataset_id: str + """Filter by root-span dataset id.""" + + dataset_name: str + """Filter by root-span dataset name.""" + + dataset_version: str + """Filter by root-span dataset version.""" + + evaluation_id: str + """Filter by root-span evaluation id.""" + + evaluation_run_id: str + """Filter by root-span evaluation run id.""" + + evaluation_sha: str + """Filter by root-span evaluation sha.""" + + session_id: str + """Filter by session id.""" + + started_at: DatetimeFilter + """Filter by root span start timestamp.""" + + status: SpanStatus + """Filter by rolled-up trace status.""" + + test_case_id: str + """Filter by root-span dataset test case id.""" diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py new file mode 100644 index 0000000000..67f6818ee2 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_list_params.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +from .trace_sort_field import TraceSortField +from .trace_filter_param import TraceFilterParam + +__all__ = ["TraceListParams"] + + +class TraceListParams(TypedDict, total=False): + workspace: str + + filter: TraceFilterParam + """ + Filter root-span-backed traces by id, session_id, rolled-up status, root span + started_at, and root-span evaluation context fields. + """ + + mode: Literal["summary", "detailed"] + """ + Use summary for root-span trace fields only, or detailed to include token, cost, + and span-count rollups. + """ + + page: int + """Page number.""" + + page_size: int + """Page size.""" + + sort: TraceSortField diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_retrieve_params.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_retrieve_params.py new file mode 100644 index 0000000000..a8115ddeb1 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_retrieve_params.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +__all__ = ["TraceRetrieveParams"] + + +class TraceRetrieveParams(TypedDict, total=False): + workspace: str + + mode: Literal["summary", "detailed"] + """ + Use summary for root-span trace fields only, or detailed to include token, cost, + and span-count rollups. + """ diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_sort_field.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_sort_field.py new file mode 100644 index 0000000000..07c6e2b17f --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/trace_sort_field.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing_extensions import Literal, TypeAlias + +__all__ = ["TraceSortField"] + +TraceSortField: TypeAlias = Literal["started_at", "-started_at"] diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/intake/traces_page.py b/sdk/python/nemo-platform/src/nemo_platform/types/intake/traces_page.py new file mode 100644 index 0000000000..a58ce14e93 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/types/intake/traces_page.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Optional + +from .trace import Trace +from ..._models import BaseModel +from ..shared.pagination_data import PaginationData + +__all__ = ["TracesPage"] + + +class TracesPage(BaseModel): + data: List[Trace] + + filter: Optional[Dict[str, object]] = None + """Filtering information.""" + + pagination: Optional[PaginationData] = None + """Pagination information.""" + + sort: Optional[str] = None + """The field on which the results are sorted.""" diff --git a/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py new file mode 100644 index 0000000000..b633ce34bd --- /dev/null +++ b/sdk/python/nemo-platform/tests/api_resources/intake/test_traces.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from tests.utils import assert_matches_type +from nemo_platform import NeMoPlatform, AsyncNeMoPlatform +from nemo_platform._utils import parse_datetime +from nemo_platform.pagination import SyncDefaultPagination, AsyncDefaultPagination +from nemo_platform.types.intake import Trace + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestTraces: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_retrieve(self, client: NeMoPlatform) -> None: + trace = client.intake.traces.retrieve( + id="id", + workspace="workspace", + ) + assert_matches_type(Trace, trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_retrieve_with_all_params(self, client: NeMoPlatform) -> None: + trace = client.intake.traces.retrieve( + id="id", + workspace="workspace", + mode="summary", + ) + assert_matches_type(Trace, trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_retrieve(self, client: NeMoPlatform) -> None: + response = client.intake.traces.with_raw_response.retrieve( + id="id", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + trace = response.parse() + assert_matches_type(Trace, trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_retrieve(self, client: NeMoPlatform) -> None: + with client.intake.traces.with_streaming_response.retrieve( + id="id", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + trace = response.parse() + assert_matches_type(Trace, trace, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_retrieve(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.intake.traces.with_raw_response.retrieve( + id="id", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + client.intake.traces.with_raw_response.retrieve( + id="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_list(self, client: NeMoPlatform) -> None: + trace = client.intake.traces.list( + workspace="workspace", + ) + assert_matches_type(SyncDefaultPagination[Trace], trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_list_with_all_params(self, client: NeMoPlatform) -> None: + trace = client.intake.traces.list( + workspace="workspace", + filter={ + "id": "id", + "dataset_id": "dataset_id", + "dataset_name": "dataset_name", + "dataset_version": "dataset_version", + "evaluation_id": "evaluation_id", + "evaluation_run_id": "evaluation_run_id", + "evaluation_sha": "evaluation_sha", + "session_id": "session_id", + "started_at": { + "gte": parse_datetime("2019-12-27T18:11:19.117Z"), + "lte": parse_datetime("2019-12-27T18:11:19.117Z"), + }, + "status": "success", + "test_case_id": "test_case_id", + }, + mode="summary", + page=1, + page_size=1, + sort="started_at", + ) + assert_matches_type(SyncDefaultPagination[Trace], trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_list(self, client: NeMoPlatform) -> None: + response = client.intake.traces.with_raw_response.list( + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + trace = response.parse() + assert_matches_type(SyncDefaultPagination[Trace], trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_list(self, client: NeMoPlatform) -> None: + with client.intake.traces.with_streaming_response.list( + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + trace = response.parse() + assert_matches_type(SyncDefaultPagination[Trace], trace, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_path_params_list(self, client: NeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + client.intake.traces.with_raw_response.list( + workspace="", + ) + + +class TestAsyncTraces: + parametrize = pytest.mark.parametrize( + "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + trace = await async_client.intake.traces.retrieve( + id="id", + workspace="workspace", + ) + assert_matches_type(Trace, trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_retrieve_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + trace = await async_client.intake.traces.retrieve( + id="id", + workspace="workspace", + mode="summary", + ) + assert_matches_type(Trace, trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.intake.traces.with_raw_response.retrieve( + id="id", + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + trace = await response.parse() + assert_matches_type(Trace, trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.intake.traces.with_streaming_response.retrieve( + id="id", + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + trace = await response.parse() + assert_matches_type(Trace, trace, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_retrieve(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.intake.traces.with_raw_response.retrieve( + id="id", + workspace="", + ) + + with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): + await async_client.intake.traces.with_raw_response.retrieve( + id="", + workspace="workspace", + ) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_list(self, async_client: AsyncNeMoPlatform) -> None: + trace = await async_client.intake.traces.list( + workspace="workspace", + ) + assert_matches_type(AsyncDefaultPagination[Trace], trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_list_with_all_params(self, async_client: AsyncNeMoPlatform) -> None: + trace = await async_client.intake.traces.list( + workspace="workspace", + filter={ + "id": "id", + "dataset_id": "dataset_id", + "dataset_name": "dataset_name", + "dataset_version": "dataset_version", + "evaluation_id": "evaluation_id", + "evaluation_run_id": "evaluation_run_id", + "evaluation_sha": "evaluation_sha", + "session_id": "session_id", + "started_at": { + "gte": parse_datetime("2019-12-27T18:11:19.117Z"), + "lte": parse_datetime("2019-12-27T18:11:19.117Z"), + }, + "status": "success", + "test_case_id": "test_case_id", + }, + mode="summary", + page=1, + page_size=1, + sort="started_at", + ) + assert_matches_type(AsyncDefaultPagination[Trace], trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_list(self, async_client: AsyncNeMoPlatform) -> None: + response = await async_client.intake.traces.with_raw_response.list( + workspace="workspace", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + trace = await response.parse() + assert_matches_type(AsyncDefaultPagination[Trace], trace, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_list(self, async_client: AsyncNeMoPlatform) -> None: + async with async_client.intake.traces.with_streaming_response.list( + workspace="workspace", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + trace = await response.parse() + assert_matches_type(AsyncDefaultPagination[Trace], trace, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_path_params_list(self, async_client: AsyncNeMoPlatform) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `workspace` but received ''"): + await async_client.intake.traces.with_raw_response.list( + workspace="", + ) diff --git a/sdk/stainless.yaml b/sdk/stainless.yaml index a13a775b7f..7584b7ba95 100644 --- a/sdk/stainless.yaml +++ b/sdk/stainless.yaml @@ -1181,3 +1181,12 @@ resources: evaluator_results: methods: list: get /apis/intake/v2/workspaces/{workspace}/spans/{span_id}/evaluator-results + traces: + models: + trace: Trace + trace_filter: TraceFilter + trace_sort_field: TraceSortField + traces_page: TracesPage + methods: + list: get /apis/intake/v2/workspaces/{workspace}/traces + retrieve: get /apis/intake/v2/workspaces/{workspace}/traces/{id} diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index e0ad5b26cd..3ec61b179b 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -194,6 +194,9 @@ authz: description: "List intake spans" read: description: "Read intake spans" + traces: + read: + description: "Read intake traces" tasks: create: description: "Create intake tasks" @@ -350,6 +353,7 @@ authz: - intake.spans.read - intake.tasks.list - intake.tasks.read + - intake.traces.read - jobs.list - jobs.read - models.adapters.list @@ -1376,6 +1380,20 @@ authz: scopes: - intake:read - platform:read + /apis/intake/v2/workspaces/{workspace}/traces: + get: + permissions: + - intake.traces.read + scopes: + - intake:read + - platform:read + /apis/intake/v2/workspaces/{workspace}/traces/{id}: + get: + permissions: + - intake.traces.read + scopes: + - intake:read + - platform:read /apis/jobs/v2/execution-profiles: get: permissions: diff --git a/services/intake/src/nmp/intake/service.py b/services/intake/src/nmp/intake/service.py index 98f83b9af7..998362fc3e 100644 --- a/services/intake/src/nmp/intake/service.py +++ b/services/intake/src/nmp/intake/service.py @@ -12,7 +12,7 @@ from nmp.intake.api.v2.exports import endpoints as exports from nmp.intake.api.v2.tasks import endpoints as tasks from nmp.intake.config import IntakeConfig -from nmp.intake.spans.api import evaluator_results, spans +from nmp.intake.spans.api import evaluator_results, spans, traces from nmp.intake.spans.clickhouse_client import ClickHouseSettings, ClickHouseSpanClient from nmp.intake.spans.ingest import atif, chat_completions, otlp @@ -47,6 +47,7 @@ def get_routers(self) -> List[RouterConfig]: RouterConfig(entries.router, tag="Entries", description="Entry management endpoints"), RouterConfig(exports.router, tag="Exports", description="Export endpoints"), RouterConfig(spans.router, tag="Spans", description="ClickHouse-backed span read endpoints"), + RouterConfig(traces.router, tag="Traces", description="ClickHouse-backed trace summary read endpoints"), RouterConfig( evaluator_results.router, tag="Evaluator Results", diff --git a/services/intake/src/nmp/intake/spans/api/dependencies.py b/services/intake/src/nmp/intake/spans/api/dependencies.py index d7bca97928..4586bc7ee2 100644 --- a/services/intake/src/nmp/intake/spans/api/dependencies.py +++ b/services/intake/src/nmp/intake/spans/api/dependencies.py @@ -12,6 +12,7 @@ from nmp.intake.spans.evaluator_results_repository import EvaluatorResultsRepository from nmp.intake.spans.service import IntakeSpansService from nmp.intake.spans.span_repository import SpanRepository +from nmp.intake.spans.trace_repository import TraceRepository async def require_workspace_access( @@ -49,6 +50,12 @@ def get_span_repository( return SpanRepository(client) +def get_trace_repository( + client: Annotated[ClickHouseSpanClient, Depends(get_clickhouse_client)], +) -> TraceRepository: + return TraceRepository(client) + + def get_evaluator_results_repository( client: Annotated[ClickHouseSpanClient, Depends(get_clickhouse_client)], ) -> EvaluatorResultsRepository: @@ -57,9 +64,10 @@ def get_evaluator_results_repository( def get_spans_service( span_repository: Annotated[SpanRepository, Depends(get_span_repository)], + trace_repository: Annotated[TraceRepository, Depends(get_trace_repository)], evaluator_results_repository: Annotated[EvaluatorResultsRepository, Depends(get_evaluator_results_repository)], ) -> IntakeSpansService: - return IntakeSpansService(span_repository, evaluator_results_repository) + return IntakeSpansService(span_repository, trace_repository, evaluator_results_repository) SpansServiceDep = Annotated[IntakeSpansService, Depends(get_spans_service)] diff --git a/services/intake/src/nmp/intake/spans/api/traces.py b/services/intake/src/nmp/intake/spans/api/traces.py new file mode 100644 index 0000000000..9c27fa4e3f --- /dev/null +++ b/services/intake/src/nmp/intake/spans/api/traces.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read API for ClickHouse-backed Intake trace summaries.""" + +from __future__ import annotations + +from datetime import timedelta + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from nmp.common.api.common import Page +from nmp.common.api.filter import FilterOperator +from nmp.common.api.parsed_filter import ParsedFilter, make_filter_dep +from nmp.common.api.utils import generate_openapi_extra_params +from nmp.intake.spans.api.dependencies import SpansServiceDep, require_workspace_access, validate_list_query_params +from nmp.intake.spans.api.query_filters import ( + filter_comparisons, + require_datetime_value, + require_enum_value, + require_string_value, +) +from nmp.intake.spans.api.traces_schemas import Trace, TraceFilter, TraceMode, TraceSortField +from nmp.intake.spans.domain import SpanAttributeFilter, SpanStatus, TraceListFilter +from nmp.intake.spans.service import TraceNotFoundError +from nmp.intake.spans.storage import utc_now + +router = APIRouter(dependencies=[Depends(require_workspace_access)]) +API_TAG = "Traces" +DEFAULT_LIST_LOOKBACK_DAYS = 30 +ROOT_ATTRIBUTE_FILTER_FIELDS = frozenset( + { + "evaluation_id", + "evaluation_sha", + "evaluation_run_id", + "dataset_id", + "dataset_name", + "dataset_version", + "test_case_id", + } +) + + +@router.get( + "/v2/workspaces/{workspace}/traces", + response_model=Page[Trace], + response_model_exclude_none=True, + tags=[API_TAG], + openapi_extra=generate_openapi_extra_params( + filter_schema=TraceFilter, + filter_description=( + "Filter root-span-backed traces by id, session_id, rolled-up status, root span started_at, " + "and root-span evaluation context fields." + ), + ), +) +async def list_traces( + workspace: str, + request: Request, + service: SpansServiceDep, + page: int = Query(default=1, ge=1, description="Page number."), + page_size: int = Query(default=10, ge=1, le=1000, description="Page size."), + sort: TraceSortField = Query(default=TraceSortField.STARTED_AT_DESC), + mode: TraceMode = Query( + default="detailed", + description="Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.", + ), + parsed: ParsedFilter = Depends(make_filter_dep(TraceFilter)), +) -> Page[Trace]: + validate_list_query_params(request, additional_params={"mode"}) + filters = _trace_filter(workspace, parsed) + _apply_default_time_bound(filters) + result = await service.list_traces( + filters=filters, + page=page, + page_size=page_size, + sort=sort.value, + mode=mode, + ) + traces = [Trace.from_domain(trace) for trace in result.data] + return Page[Trace]( + data=traces, + pagination=result.pagination, + sort=sort, + filter=parsed.to_response(), + ) + + +@router.get( + "/v2/workspaces/{workspace}/traces/{id}", + response_model=Trace, + response_model_exclude_none=True, + tags=[API_TAG], +) +async def get_trace( + workspace: str, + id: str, + service: SpansServiceDep, + mode: TraceMode = Query( + default="detailed", + description="Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.", + ), +) -> Trace: + try: + trace = await service.get_trace(workspace=workspace, trace_id=id, mode=mode) + except TraceNotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Trace {workspace}/{id} not found") + return Trace.from_domain(trace) + + +def _trace_filter(workspace: str, parsed: ParsedFilter) -> TraceListFilter: + filters = TraceListFilter(workspace=workspace) + for comparison in filter_comparisons(parsed): + if comparison.field == "id": + filters.trace_id = require_string_value(comparison) + elif comparison.field == "session_id": + filters.session_id = require_string_value(comparison) + elif comparison.field == "status": + filters.status = require_enum_value(comparison, SpanStatus) + elif comparison.field == "started_at" and comparison.operator == FilterOperator.GTE: + filters.started_at_gte = require_datetime_value(comparison) + elif comparison.field == "started_at" and comparison.operator == FilterOperator.LTE: + filters.started_at_lte = require_datetime_value(comparison) + elif comparison.field in ROOT_ATTRIBUTE_FILTER_FIELDS: + filters.root_attribute_filters.append( + SpanAttributeFilter( + field=comparison.field, + operator=comparison.operator.value, + value=require_string_value(comparison), + ) + ) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported trace filter: {comparison.field} {comparison.operator.value}", + ) + return filters + + +def _apply_default_time_bound(filters: TraceListFilter) -> None: + if filters.started_at_gte is None and filters.started_at_lte is None: + filters.started_at_gte = utc_now() - timedelta(days=DEFAULT_LIST_LOOKBACK_DAYS) diff --git a/services/intake/src/nmp/intake/spans/api/traces_schemas.py b/services/intake/src/nmp/intake/spans/api/traces_schemas.py new file mode 100644 index 0000000000..a844eaebc4 --- /dev/null +++ b/services/intake/src/nmp/intake/spans/api/traces_schemas.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pydantic schemas for ClickHouse-backed Intake trace summaries.""" + +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum +from typing import Literal, Self + +from nmp.common.entities.values import DatetimeFilter +from nmp.intake.spans.api.spans_schemas import SpanEvaluationContext +from nmp.intake.spans.domain import IntakeTrace, SpanStatus +from pydantic import BaseModel, ConfigDict, Field + + +class TraceSortField(StrEnum): + STARTED_AT_ASC = "started_at" + STARTED_AT_DESC = "-started_at" + + +TraceMode = Literal["summary", "detailed"] + + +class TraceFilter(BaseModel): + id: str | None = Field(default=None, description="Filter by canonical Intake trace id.") + session_id: str | None = Field(default=None, description="Filter by session id.") + status: SpanStatus | None = Field(default=None, description="Filter by rolled-up trace status.") + started_at: DatetimeFilter | None = Field(default=None, description="Filter by root span start timestamp.") + evaluation_id: str | None = Field(default=None, description="Filter by root-span evaluation id.") + evaluation_sha: str | None = Field(default=None, description="Filter by root-span evaluation sha.") + evaluation_run_id: str | None = Field(default=None, description="Filter by root-span evaluation run id.") + dataset_id: str | None = Field(default=None, description="Filter by root-span dataset id.") + dataset_name: str | None = Field(default=None, description="Filter by root-span dataset name.") + dataset_version: str | None = Field(default=None, description="Filter by root-span dataset version.") + test_case_id: str | None = Field(default=None, description="Filter by root-span dataset test case id.") + + +class Trace(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + id: str + root_span_id: str | None = None + session_id: str + workspace: str + name: str | None = None + evaluation_context: SpanEvaluationContext | None = None + started_at: datetime + ended_at: datetime | None = None + duration_ms: float | None = None + status: SpanStatus + input_tokens: int | None = Field(default=None, ge=0) + output_tokens: int | None = Field(default=None, ge=0) + cached_tokens: int | None = Field(default=None, ge=0) + total_tokens: int | None = Field(default=None, ge=0) + cost_usd: float | None = None + cost_input_usd: float | None = None + cost_output_usd: float | None = None + span_count: int | None = Field(default=None, ge=0) + error_count: int | None = Field(default=None, ge=0) + + @classmethod + def from_domain(cls, trace: IntakeTrace) -> Self: + return cls( + id=trace.id, + root_span_id=trace.root_span_id, + session_id=trace.session_id, + workspace=trace.workspace, + name=trace.name, + evaluation_context=SpanEvaluationContext.model_validate(trace.evaluation_context.model_dump()) + if trace.evaluation_context is not None + else None, + started_at=trace.started_at, + ended_at=trace.ended_at, + duration_ms=trace.duration_ms, + status=trace.status, + input_tokens=trace.input_tokens, + output_tokens=trace.output_tokens, + cached_tokens=trace.cached_tokens, + total_tokens=trace.total_tokens, + cost_usd=trace.cost_usd, + cost_input_usd=trace.cost_input_usd, + cost_output_usd=trace.cost_output_usd, + span_count=trace.span_count, + error_count=trace.error_count, + ) diff --git a/services/intake/src/nmp/intake/spans/domain.py b/services/intake/src/nmp/intake/spans/domain.py index 5825399880..7344826c65 100644 --- a/services/intake/src/nmp/intake/spans/domain.py +++ b/services/intake/src/nmp/intake/spans/domain.py @@ -7,9 +7,9 @@ from datetime import datetime from enum import StrEnum -from typing import Any, Self +from typing import Any, Literal, Self -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator class SpanKind(StrEnum): @@ -82,6 +82,79 @@ class SpanListFilter(BaseModel): attribute_filters: list[SpanAttributeFilter] = Field(default_factory=list) +class TraceListFilter(BaseModel): + workspace: str + trace_id: str | None = None + session_id: str | None = None + source_format: str | None = None + status: SpanStatus | None = None + started_at_gte: datetime | None = None + started_at_lte: datetime | None = None + root_attribute_filters: list[SpanAttributeFilter] = Field(default_factory=list) + span_attribute_filters: list[SpanAttributeFilter] = Field(default_factory=list) + + +TraceMode = Literal["summary", "detailed"] + + +class TraceEvaluationContext(BaseModel): + # Read model for root-span evaluation context. Historical rows may have + # partial context, so this intentionally does not reuse ingest validation. + model_config = ConfigDict(extra="forbid") + + evaluation_id: str | None = None + evaluation_sha: str | None = None + evaluation_run_id: str | None = None + dataset_id: str | None = None + dataset_name: str | None = None + dataset_version: str | None = None + test_case_id: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + def has_scalar_values(self) -> bool: + return any( + value is not None + for value in ( + self.evaluation_id, + self.evaluation_sha, + self.evaluation_run_id, + self.dataset_id, + self.dataset_name, + self.dataset_version, + self.test_case_id, + ) + ) + + +class IntakeTrace(BaseModel): + id: str + root_span_id: str | None = None + workspace: str + session_id: str + source_format: str + name: str | None = None + input: str | None = None + output: str | None = None + project: str | None = None + evaluation_context: TraceEvaluationContext | None = None + started_at: datetime + ended_at: datetime | None = None + duration_ms: float | None = None + ingested_at: datetime + status: SpanStatus + input_tokens: int | None = Field(default=None, ge=0) + output_tokens: int | None = Field(default=None, ge=0) + cached_tokens: int | None = Field(default=None, ge=0) + total_tokens: int | None = Field(default=None, ge=0) + cost_usd: float | None = None + cost_input_usd: float | None = None + cost_output_usd: float | None = None + models: list[str] | None = None + providers: list[str] | None = None + span_count: int | None = Field(default=None, ge=0) + error_count: int | None = Field(default=None, ge=0) + + class EvaluatorResultDataType(StrEnum): NUMERIC = "NUMERIC" CATEGORICAL = "CATEGORICAL" diff --git a/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py b/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py index 50f66bc751..a638b17401 100644 --- a/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py +++ b/services/intake/src/nmp/intake/spans/ingest/atif_mapping.py @@ -119,22 +119,18 @@ def _trajectory_to_span( # atif.raw for now. Only subagent_trajectory_ref entries are materialized as # lightweight delegation spans until embedded trajectory expansion has # explicit trace identity and parentage semantics. - final_metrics = trajectory.final_metrics # ATIF span IDs are trace-native by design: session_id is the trace identity, # while evaluation_context is queryable metadata on the root span. + # + # Token/cost accounting belongs on the spans that incurred the LLM calls + # (the agent steps), not duplicated onto the trajectory coordinator span. + # The trace-level rollup sums per-step metrics; writing trajectory.final_metrics + # here too would double-count any source that emits both (e.g. opencode). external_span_id = stable_id(workspace, trajectory.session_id, "trajectory", prefix="span") attribute_bags = _span_attributes( model=trajectory.agent.model_name, agent_name=trajectory.agent.name, evaluation_context=trajectory.evaluation_context, - input_tokens=final_metrics.total_prompt_tokens if final_metrics is not None else None, - output_tokens=final_metrics.total_completion_tokens if final_metrics is not None else None, - cached_tokens=final_metrics.total_cached_tokens if final_metrics is not None else None, - total_tokens=_sum_ints( - final_metrics.total_prompt_tokens if final_metrics is not None else None, - final_metrics.total_completion_tokens if final_metrics is not None else None, - ), - cost_total_usd=_decimal(final_metrics.total_cost_usd) if final_metrics is not None else None, raw_attributes=raw_attributes, ) return IntakeSpan( @@ -144,15 +140,15 @@ def _trajectory_to_span( source_format="atif", external_span_id=external_span_id, kind=SpanKind.AGENT, - name="atif-trajectory", - # A tool call can fail while the overall trajectory remains a valid run. - # Child error spans stay queryable without rolling the root to ERROR. - status=SpanStatus.SUCCESS, + name=trajectory.agent.name, + status=SpanStatus.ERROR if _trajectory_has_error(trajectory) else SpanStatus.SUCCESS, start_time=_trajectory_started_at(trajectory, ingested_at), end_time=_trajectory_ended_at(trajectory), attributes_string=attribute_bags.string, attributes_number=attribute_bags.number, attributes_bool=attribute_bags.boolean, + input=_trajectory_input(trajectory) or "", + output=_trajectory_output(trajectory) or "", event_ts=ingested_at, ) @@ -554,6 +550,35 @@ def _step_metrics(step: AtifStep) -> AtifMetrics | None: return step.metrics if isinstance(step, AtifStepAgent) else None +def _trajectory_input(trajectory: AtifTrajectory) -> str | None: + for step in trajectory.steps: + if step.source == "user": + return _string_or_json(step.message) + return None + + +def _trajectory_output(trajectory: AtifTrajectory) -> str | None: + for step in reversed(trajectory.steps): + if isinstance(step, AtifStepAgent): + if step.message != "": + return _string_or_json(step.message) + return _step_output(step) + return None + + +def _trajectory_has_error(trajectory: AtifTrajectory) -> bool: + for step in trajectory.steps: + if not isinstance(step, AtifStepAgent): + continue + observation = _step_observation(step) + if observation is None: + continue + for result in observation.results: + if _tool_result_is_error(step, result): + return True + return False + + def _step_tool_calls(step: AtifStep) -> list[AtifToolCall]: if not isinstance(step, AtifStepAgent): return [] diff --git a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py index ab0c8366cc..aa78ee3fa8 100644 --- a/services/intake/src/nmp/intake/spans/ingest/chat_completions.py +++ b/services/intake/src/nmp/intake/spans/ingest/chat_completions.py @@ -37,8 +37,17 @@ class ChatCompletionsIngestRequest(BaseModel): request: FlexibleEntryRequest response: FlexibleEntryResponse - session_id: str | None = None - trace_id: str | None = Field(default=None, description="Defaults to session_id when omitted.") + session_id: str | None = Field( + default=None, + description="Groups related chat-completions calls without forcing them into the same trace.", + ) + trace_id: str | None = Field( + default=None, + description=( + "Opt into joining an existing trace built via OTel or ATIF. This is not a grouping mechanism for " + "chat-completions calls; use session_id to group related calls." + ), + ) evaluation_context: EvaluationContext | None = None provider: str | None = None @@ -76,7 +85,7 @@ def _chat_completion_to_span( usage = _dict_or_empty(response.get("usage")) external_span_id = _external_span_id(response, request) - trace_id = body.trace_id or body.session_id or external_span_id + trace_id = body.trace_id or external_span_id session_id = body.session_id or trace_id error = response.get("error") if isinstance(response.get("error"), dict) else None diff --git a/services/intake/src/nmp/intake/spans/service.py b/services/intake/src/nmp/intake/spans/service.py index 00281ed411..e71c266405 100644 --- a/services/intake/src/nmp/intake/spans/service.py +++ b/services/intake/src/nmp/intake/spans/service.py @@ -10,11 +10,15 @@ EvaluatorResult, EvaluatorResultListFilter, IntakeSpan, + IntakeTrace, SpanListFilter, TraceBatch, + TraceListFilter, + TraceMode, ) from nmp.intake.spans.evaluator_results_repository import EvaluatorResultsRepository from nmp.intake.spans.span_repository import SpanRepository +from nmp.intake.spans.trace_repository import TraceRepository class SpanNotFoundError(Exception): @@ -31,13 +35,22 @@ def __init__(self, workspace: str, evaluator_result_id: str) -> None: self.evaluator_result_id = evaluator_result_id +class TraceNotFoundError(Exception): + def __init__(self, workspace: str, trace_id: str) -> None: + super().__init__(f"Trace {workspace}/{trace_id} not found") + self.workspace = workspace + self.trace_id = trace_id + + class IntakeSpansService: def __init__( self, span_repository: SpanRepository, + trace_repository: TraceRepository, evaluator_results_repository: EvaluatorResultsRepository, ) -> None: self._spans = span_repository + self._traces = trace_repository self._evaluator_results = evaluator_results_repository async def ingest_batch(self, batch: TraceBatch) -> None: @@ -61,6 +74,23 @@ async def get_span(self, *, workspace: str, span_id: str) -> IntakeSpan: raise SpanNotFoundError(workspace, span_id) return span + async def list_traces( + self, + *, + filters: TraceListFilter, + page: int, + page_size: int, + sort: str, + mode: TraceMode, + ) -> PaginatedResult[IntakeTrace]: + return await self._traces.list_traces(filters=filters, page=page, page_size=page_size, sort=sort, mode=mode) + + async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> IntakeTrace: + trace = await self._traces.get_trace(workspace=workspace, trace_id=trace_id, mode=mode) + if trace is None: + raise TraceNotFoundError(workspace, trace_id) + return trace + async def create_evaluator_result(self, result: EvaluatorResult) -> EvaluatorResult: """Persist one evaluator_result. Loose target — no span existence check.""" diff --git a/services/intake/src/nmp/intake/spans/trace_repository.py b/services/intake/src/nmp/intake/spans/trace_repository.py new file mode 100644 index 0000000000..8356e03784 --- /dev/null +++ b/services/intake/src/nmp/intake/spans/trace_repository.py @@ -0,0 +1,540 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ClickHouse implementation of Intake trace reads.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from nmp.common.api.common import PaginatedResult +from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient +from nmp.intake.spans.domain import IntakeTrace, TraceEvaluationContext, TraceListFilter, TraceMode +from nmp.intake.spans.span_attribute_bags import SpanAttributeBags +from nmp.intake.spans.span_attribute_catalog import COST_SCALE, SpanAttributeField, spec_for_field, where_clause +from nmp.intake.spans.span_semantic_attributes import SpanSemanticAttributes +from nmp.intake.spans.storage import make_pagination, normalize_span_status, result_rows + +TRACE_SORT_COLUMNS = { + "started_at": "started_at", +} + +METRIC_ATTRIBUTE_FIELDS = { + "input_tokens": SpanAttributeField.INPUT_TOKENS, + "output_tokens": SpanAttributeField.OUTPUT_TOKENS, + "cached_tokens": SpanAttributeField.CACHED_TOKENS, + "total_tokens": SpanAttributeField.TOTAL_TOKENS, + "cost_usd": SpanAttributeField.COST_TOTAL_USD, + "cost_input_usd": SpanAttributeField.COST_INPUT_USD, + "cost_output_usd": SpanAttributeField.COST_OUTPUT_USD, +} + +TRACE_COLUMNS = [ + "id", + "workspace", + "session_id", + "source_format", + "root_span_id", + "name", + "input", + "output", + "started_at", + "ended_at", + "status", + *METRIC_ATTRIBUTE_FIELDS.keys(), + "models", + "providers", + "span_count", + "error_count", + "root_attributes_string", + "ingested_at", +] + +_CURRENT_SPAN_IDENTITY_COLUMNS = ( + "workspace", + "source_format", + "trace_id", + "external_span_id", + "id", +) +_CURRENT_SPAN_VALUE_COLUMNS = ( + "session_id", + "external_parent_span_id", + "kind", + "name", + "status", + "start_time", + "end_time", + "attributes_string", + "attributes_number", + "attributes_bool", + "input", + "output", + "event_ts", + "is_deleted", +) + +_ZERO_DATETIME = datetime.fromtimestamp(0, tz=timezone.utc) +_ZERO_DATETIME_SQL = "toDateTime64(0, 6)" + + +class TraceRepository: + def __init__(self, client: ClickHouseSpanClient) -> None: + self._client = client + + async def list_traces( + self, + *, + filters: TraceListFilter, + page: int, + page_size: int, + sort: str, + mode: TraceMode, + ) -> PaginatedResult[IntakeTrace]: + trace_sql, parameters = _trace_rows_sql(self._client.table("spans"), filters, mode=mode) + outer_where_sql, outer_parameters = _trace_outer_where(filters) + all_parameters = {**parameters, **outer_parameters} + + total_result = await self._client.query( + f""" + SELECT count() + FROM ({trace_sql}) AS traces + WHERE {outer_where_sql} + """, + parameters=all_parameters, + ) + total_results = int(total_result.result_rows[0][0]) + + offset = (page - 1) * page_size + rows_result = await self._client.query( + f""" + SELECT * + FROM ({trace_sql}) AS traces + WHERE {outer_where_sql} + ORDER BY {_order_by(sort)} + LIMIT %(limit)s OFFSET %(offset)s + """, + parameters={**all_parameters, "limit": page_size, "offset": offset}, + ) + rows = result_rows(rows_result) + traces = [_row_to_trace(row) for row in rows] + return PaginatedResult( + data=traces, + pagination=make_pagination( + page=page, + page_size=page_size, + current_page_size=len(traces), + total_results=total_results, + ), + ) + + async def get_trace(self, *, workspace: str, trace_id: str, mode: TraceMode) -> IntakeTrace | None: + result = await self.list_traces( + filters=TraceListFilter(workspace=workspace, trace_id=trace_id), + page=1, + page_size=1, + sort="-started_at", + mode=mode, + ) + return result.data[0] if result.data else None + + +def _trace_rows_sql(table: str, filters: TraceListFilter, *, mode: TraceMode) -> tuple[str, dict[str, Any]]: + summary_sql, summary_parameters = _trace_summary_sql(table, filters) + if mode == "detailed": + rollup_sql, rollup_parameters = _trace_aggregates_sql(table, filters) + include_aggregates = True + elif mode == "summary": + rollup_sql, rollup_parameters = _trace_status_sql(table, filters) + include_aggregates = False + else: + raise ValueError(f"Unsupported trace mode: {mode}") + + query = f""" + SELECT + {_trace_select_columns(include_aggregates=include_aggregates)} + FROM ({summary_sql}) AS traces + ANY INNER JOIN ({rollup_sql}) AS rollups + ON traces.workspace = rollups.workspace + AND traces.source_format = rollups.source_format + AND traces.trace_id = rollups.trace_id + """ + return query, {**summary_parameters, **rollup_parameters} + + +def _trace_select_columns(*, include_aggregates: bool) -> str: + aggregate_columns = [ + f"rollups.{column} AS {column}" if include_aggregates else f"NULL AS {column}" + for column in ( + *METRIC_ATTRIBUTE_FIELDS.keys(), + "models", + "providers", + "span_count", + "error_count", + ) + ] + columns = [ + "traces.id AS id", + "traces.workspace AS workspace", + "traces.session_id AS session_id", + "traces.source_format AS source_format", + "traces.root_span_id AS root_span_id", + "traces.name AS name", + "traces.input AS input", + "traces.output AS output", + "traces.started_at AS started_at", + "traces.ended_at AS ended_at", + "rollups.status AS status", + *aggregate_columns, + "traces.root_attributes_string AS root_attributes_string", + "traces.ingested_at AS ingested_at", + ] + return ",\n ".join(columns) + + +def _trace_summary_sql(table: str, filters: TraceListFilter) -> tuple[str, dict[str, Any]]: + root_alias = "root_spans" + base_where_sql, parameters = _trace_summary_where(table, filters, qualifier=root_alias) + query = f""" + SELECT + {root_alias}.trace_id AS trace_id, + {root_alias}.trace_id AS id, + {root_alias}.workspace AS workspace, + {root_alias}.session_id AS session_id, + {root_alias}.source_format AS source_format, + nullIf({root_alias}.external_span_id, '') AS root_span_id, + nullIf({root_alias}.name, '') AS name, + nullIf({root_alias}.input, '') AS input, + nullIf({root_alias}.output, '') AS output, + {root_alias}.start_time AS started_at, + nullIf({root_alias}.end_time, {_ZERO_DATETIME_SQL}) AS ended_at, + {root_alias}.attributes_string AS root_attributes_string, + {root_alias}.event_ts AS ingested_at + FROM {_current_spans_sql(table)} AS {root_alias} + WHERE {base_where_sql} + ORDER BY {root_alias}.start_time ASC, {root_alias}.id ASC + LIMIT 1 BY {root_alias}.workspace, {root_alias}.source_format, {root_alias}.trace_id + """ + return query, parameters + + +def _trace_status_sql(table: str, filters: TraceListFilter) -> tuple[str, dict[str, Any]]: + source_alias = "trace_spans" + base_where_sql, parameters = _trace_rollup_where(table, filters, qualifier=source_alias) + query = f""" + SELECT + {source_alias}.workspace AS workspace, + {source_alias}.source_format AS source_format, + {source_alias}.trace_id AS trace_id, + {_rolled_up_status_sql(source_alias)} AS status + FROM {_current_spans_sql(table)} AS {source_alias} + WHERE {base_where_sql} + GROUP BY {source_alias}.workspace, {source_alias}.source_format, {source_alias}.trace_id + """ + return query, parameters + + +def _trace_aggregates_sql(table: str, filters: TraceListFilter) -> tuple[str, dict[str, Any]]: + source_alias = "trace_spans" + base_where_sql, parameters = _trace_rollup_where(table, filters, qualifier=source_alias) + metric_columns, metric_parameters = _metric_columns(source_alias) + parameters.update(metric_parameters) + + model_spec = spec_for_field(SpanAttributeField.MODEL) + provider_spec = spec_for_field(SpanAttributeField.PROVIDER) + parameters["model_key"] = model_spec.bag_key + parameters["provider_key"] = provider_spec.bag_key + + query = f""" + SELECT + {source_alias}.workspace AS workspace, + {source_alias}.source_format AS source_format, + {source_alias}.trace_id AS trace_id, + {_rolled_up_status_sql(source_alias)} AS status, + {metric_columns}, + arraySort(groupUniqArrayIf( + {source_alias}.attributes_string[%(model_key)s], + has(mapKeys({source_alias}.attributes_string), %(model_key)s) + AND {source_alias}.attributes_string[%(model_key)s] != '' + )) AS models, + arraySort(groupUniqArrayIf( + {source_alias}.attributes_string[%(provider_key)s], + has(mapKeys({source_alias}.attributes_string), %(provider_key)s) + AND {source_alias}.attributes_string[%(provider_key)s] != '' + )) AS providers, + count() AS span_count, + countIf({source_alias}.status = 'error') AS error_count + FROM {_current_spans_sql(table)} AS {source_alias} + WHERE {base_where_sql} + GROUP BY {source_alias}.workspace, {source_alias}.source_format, {source_alias}.trace_id + """ + return query, parameters + + +def _rolled_up_status_sql(source_alias: str) -> str: + return f""" + multiIf( + countIf({source_alias}.status = 'error') > 0, 'error', + countIf({source_alias}.status = 'cancelled') > 0, 'cancelled', + countIf({source_alias}.status = 'unknown') = count(), 'unknown', + 'success' + ) + """ + + +def _trace_summary_where(table: str, filters: TraceListFilter, *, qualifier: str) -> tuple[str, dict[str, Any]]: + clauses, parameters = _trace_identity_where(table, filters, qualifier=qualifier) + + def column(name: str) -> str: + return f"{qualifier}.{name}" + + clauses.append(f"{column('external_parent_span_id')} = ''") + if filters.started_at_gte is not None: + clauses.append(f"{column('start_time')} >= %(started_at_gte)s") + parameters["started_at_gte"] = filters.started_at_gte + if filters.started_at_lte is not None: + clauses.append(f"{column('start_time')} <= %(started_at_lte)s") + parameters["started_at_lte"] = filters.started_at_lte + + return " AND ".join(clauses), parameters + + +def _trace_rollup_where(table: str, filters: TraceListFilter, *, qualifier: str) -> tuple[str, dict[str, Any]]: + clauses, parameters = _trace_identity_where(table, filters, qualifier=qualifier) + return " AND ".join(clauses), parameters + + +def _trace_identity_where( + table: str, + filters: TraceListFilter, + *, + qualifier: str, +) -> tuple[list[str], dict[str, Any]]: + def column(name: str) -> str: + return f"{qualifier}.{name}" + + clauses = [f"{column('workspace')} = %(workspace)s", f"{column('is_deleted')} = 0"] + parameters: dict[str, Any] = {"workspace": filters.workspace} + + if filters.trace_id is not None: + clauses.append(f"{column('trace_id')} = %(trace_id)s") + parameters["trace_id"] = filters.trace_id + if filters.session_id is not None: + clauses.append(f"{column('session_id')} = %(session_id)s") + parameters["session_id"] = filters.session_id + if filters.source_format is not None: + clauses.append(f"{column('source_format')} = %(source_format)s") + parameters["source_format"] = filters.source_format + + root_sql, root_parameters = _candidate_subquery( + table=table, + workspace=filters.workspace, + attribute_filters=filters.root_attribute_filters, + root_only=True, + prefix="root_candidate", + ) + if root_sql: + clauses.append(f"({column('workspace')}, {column('source_format')}, {column('trace_id')}) IN ({root_sql})") + parameters.update(root_parameters) + + span_sql, span_parameters = _candidate_subquery( + table=table, + workspace=filters.workspace, + attribute_filters=filters.span_attribute_filters, + root_only=False, + prefix="span_candidate", + ) + if span_sql: + clauses.append(f"({column('workspace')}, {column('source_format')}, {column('trace_id')}) IN ({span_sql})") + parameters.update(span_parameters) + + return clauses, parameters + + +def _candidate_subquery( + *, + table: str, + workspace: str, + attribute_filters: list[Any], + root_only: bool, + prefix: str, +) -> tuple[str | None, dict[str, Any]]: + if not attribute_filters: + return None, {} + + clauses = ["workspace = %(workspace)s", "is_deleted = 0"] + if root_only: + clauses.append("external_parent_span_id = ''") + + parameters: dict[str, Any] = {"workspace": workspace} + for index, attribute_filter in enumerate(attribute_filters): + clause, clause_parameters = where_clause( + attribute_filter.field, + attribute_filter.operator, + attribute_filter.value, + param_prefix=f"{prefix}_{index}", + ) + clauses.append(f"({clause})") + parameters.update(clause_parameters) + + return ( + f""" + SELECT workspace, source_format, trace_id + FROM {_current_spans_sql(table)} AS candidate_spans + WHERE {" AND ".join(clauses)} + """, + parameters, + ) + + +def _current_spans_sql(table: str) -> str: + source_alias = "span_versions" + columns = [ + *[f"{source_alias}.{column} AS {column}" for column in _CURRENT_SPAN_IDENTITY_COLUMNS], + *[ + f"argMax({source_alias}.{column}, ({source_alias}.event_ts, {source_alias}.is_deleted)) AS {column}" + for column in _CURRENT_SPAN_VALUE_COLUMNS + ], + ] + columns_sql = ",\n ".join(columns) + group_by_sql = ", ".join(f"{source_alias}.{column}" for column in _CURRENT_SPAN_IDENTITY_COLUMNS) + return f""" + ( + SELECT + {columns_sql} + FROM {table} AS {source_alias} + WHERE {source_alias}.workspace = %(workspace)s + GROUP BY {group_by_sql} + ) + """ + + +def _trace_outer_where(filters: TraceListFilter) -> tuple[str, dict[str, Any]]: + clauses = ["1 = 1"] + parameters: dict[str, Any] = {} + + if filters.status is not None: + clauses.append("status = %(status)s") + parameters["status"] = filters.status.value + + return " AND ".join(clauses), parameters + + +def _metric_columns(source_alias: str) -> tuple[str, dict[str, Any]]: + parameters: dict[str, Any] = {} + columns: list[str] = [] + for alias, field in METRIC_ATTRIBUTE_FIELDS.items(): + spec = spec_for_field(field) + key_param = f"{alias}_key" + parameters[key_param] = spec.bag_key + number_bag = f"{source_alias}.attributes_number" + has_expr = f"has(mapKeys({number_bag}), %({key_param})s)" + sum_expr = f"sumIf({number_bag}[%({key_param})s], {has_expr})" + if spec.scale is not None: + value_expr = f"{sum_expr} / {COST_SCALE}" + else: + value_expr = sum_expr + columns.append(f"if(countIf({has_expr}) = 0, NULL, {value_expr}) AS {alias}") + return ",\n ".join(columns), parameters + + +def _order_by(sort: str) -> str: + direction = "DESC" if sort.startswith("-") else "ASC" + field = sort.removeprefix("-") + column = TRACE_SORT_COLUMNS.get(field) + if column is None: + raise ValueError(f"Unsupported trace sort field: {field}") + return f"{column} {direction}, id ASC" + + +def _row_to_trace(row: dict[str, Any]) -> IntakeTrace: + root_attributes = dict(row.get("root_attributes_string") or {}) + attribute_bags = SpanAttributeBags.from_domain_maps( + attributes_string=root_attributes, + attributes_number={}, + attributes_bool={}, + ) + semantic_attributes = SpanSemanticAttributes.from_bags(attribute_bags) + ended_at = _none_if_zero_datetime(row.get("ended_at")) + return IntakeTrace( + id=row["id"], + root_span_id=row.get("root_span_id") or None, + workspace=row["workspace"], + session_id=row["session_id"], + source_format=row["source_format"], + name=row.get("name") or None, + input=row.get("input") or None, + output=row.get("output") or None, + project=semantic_attributes.project, + evaluation_context=_evaluation_context(semantic_attributes, attribute_bags), + started_at=row["started_at"], + ended_at=ended_at, + duration_ms=_duration_ms(row["started_at"], ended_at), + ingested_at=row["ingested_at"], + status=normalize_span_status(row.get("status")), + input_tokens=_int_or_none(row.get("input_tokens")), + output_tokens=_int_or_none(row.get("output_tokens")), + cached_tokens=_int_or_none(row.get("cached_tokens")), + total_tokens=_int_or_none(row.get("total_tokens")), + cost_usd=_float_or_none(row.get("cost_usd")), + cost_input_usd=_float_or_none(row.get("cost_input_usd")), + cost_output_usd=_float_or_none(row.get("cost_output_usd")), + models=_string_list_or_none(row.get("models")), + providers=_string_list_or_none(row.get("providers")), + span_count=_int_or_none(row.get("span_count")), + error_count=_int_or_none(row.get("error_count")), + ) + + +def _evaluation_context( + attributes: SpanSemanticAttributes, + attribute_bags: SpanAttributeBags, +) -> TraceEvaluationContext | None: + metadata = attribute_bags.evaluation_metadata() + context = TraceEvaluationContext( + evaluation_id=attributes.evaluation_id, + evaluation_sha=attributes.evaluation_sha, + evaluation_run_id=attributes.evaluation_run_id, + dataset_id=attributes.dataset_id, + dataset_name=attributes.dataset_name, + dataset_version=attributes.dataset_version, + test_case_id=attributes.test_case_id, + metadata=metadata or {}, + ) + if metadata is None and not context.has_scalar_values(): + return None + return context + + +def _duration_ms(started_at: datetime, ended_at: datetime | None) -> float | None: + if ended_at is None: + return None + return (ended_at - started_at).total_seconds() * 1000 + + +def _none_if_zero_datetime(value: Any) -> datetime | None: + if value is None: + return None + if value == _ZERO_DATETIME or value.timestamp() == 0: + return None + return value + + +def _int_or_none(value: Any) -> int | None: + if value is None: + return None + return int(value) + + +def _float_or_none(value: Any) -> float | None: + if value is None: + return None + return float(value) + + +def _string_list_or_none(value: Any) -> list[str] | None: + if value is None: + return None + values = [str(item) for item in value if str(item)] + return values diff --git a/services/intake/tests/integration/spans/test_atif_ingest.py b/services/intake/tests/integration/spans/test_atif_ingest.py index c02c2f7a9c..3cde2bdd3a 100644 --- a/services/intake/tests/integration/spans/test_atif_ingest.py +++ b/services/intake/tests/integration/spans/test_atif_ingest.py @@ -321,7 +321,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert len(spans) == 7 spans_by_name = {span["name"]: span for span in spans} assert set(spans_by_name) == { - "atif-trajectory", + "sample-agent", "user-1", "agent-2", "Bash", @@ -331,16 +331,22 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( } assert {span["session_id"] for span in spans} == {"d074dfb7-3691-443c-b137-720d75e40afa"} - trajectory = spans_by_name["atif-trajectory"] + trajectory = spans_by_name["sample-agent"] assert trajectory["kind"] == "AGENT" assert trajectory["source"] == "atif" + assert trajectory["status"] == "error" + assert trajectory["input"] == body["steps"][0]["message"] + assert trajectory["output"] == body["steps"][2]["message"] assert trajectory["model"] == "provider/sample-model" assert trajectory["agent_name"] == "sample-agent" - assert trajectory["input_tokens"] == 51701 - assert trajectory["output_tokens"] == 255 - assert trajectory["cached_tokens"] == 0 - assert trajectory["total_tokens"] == 51956 - assert Decimal(str(trajectory["cost_total_usd"])) == Decimal("0.264321") + # Token and cost accounting lives on the agent step spans that incurred the + # LLM calls, not on the trajectory coordinator. The trace-level rollup sums + # per-step metrics; see _trajectory_to_span. + assert "input_tokens" not in trajectory + assert "output_tokens" not in trajectory + assert "cached_tokens" not in trajectory + assert "total_tokens" not in trajectory + assert "cost_total_usd" not in trajectory assert trajectory["evaluation_context"] == evaluation_context assert "attributes_string" not in trajectory trajectory_raw = json.loads(trajectory["raw_attributes"]) @@ -351,7 +357,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert trajectory["ended_at"] == "2026-05-04T19:06:45.570079" for span in spans: - if span["name"] == "atif-trajectory": + if span["name"] == "sample-agent": continue assert "evaluation_context" not in span @@ -362,7 +368,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert evaluation_response.status_code == 200, evaluation_response.text evaluation_spans = evaluation_response.json()["data"] assert len(evaluation_spans) == 1 - assert evaluation_spans[0]["name"] == "atif-trajectory" + assert evaluation_spans[0]["name"] == "sample-agent" for field, value in { "evaluation_id": evaluation_context["evaluation_id"], @@ -380,7 +386,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert filtered.status_code == 200, filtered.text filtered_spans = filtered.json()["data"] assert len(filtered_spans) == 1 - assert filtered_spans[0]["name"] == "atif-trajectory" + assert filtered_spans[0]["name"] == "sample-agent" assert filtered_spans[0]["evaluation_context"][field] == value evaluator_span = spans_by_name["harbor.verifier"] @@ -483,7 +489,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert evaluation_roots_response.status_code == 200, evaluation_roots_response.text evaluation_roots = evaluation_roots_response.json()["data"] assert len(evaluation_roots) == 2 - assert {span["name"] for span in evaluation_roots} == {"atif-trajectory"} + assert {span["name"] for span in evaluation_roots} == {"sample-agent"} assert {span["evaluation_context"]["evaluation_run_id"] for span in evaluation_roots} == {evaluation_run_id} assert {span["session_id"] for span in evaluation_roots} == { "d074dfb7-3691-443c-b137-720d75e40afa", @@ -509,7 +515,7 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert other_evaluation_response.status_code == 200, other_evaluation_response.text other_spans = other_evaluation_response.json()["data"] assert len(other_spans) == 1 - assert other_spans[0]["name"] == "atif-trajectory" + assert other_spans[0]["name"] == "sample-agent" assert other_spans[0]["span_id"] == trajectory["span_id"] same_session_response = client.get( @@ -519,3 +525,118 @@ def test_atif_ingest_accepts_example_trajectory_and_reconstructs_read_side_data( assert same_session_response.status_code == 200, same_session_response.text same_session_spans_by_name = {span["name"]: span for span in same_session_response.json()["data"]} assert same_session_spans_by_name["user-1"]["span_id"] == user_step["span_id"] + + +def test_atif_trace_tokens_do_not_double_count_when_trajectory_and_steps_both_carry_metrics( + client: TestClient, +): + """Regression: emitters like opencode populate both trajectory.final_metrics AND + per-step metrics that sum to it. The trajectory span must NOT carry token attributes, + or the trace-level rollup would sum them and report 2x the real total. + """ + body = { + "schema_version": "ATIF-v1.6", + "session_id": "atif-rollup-no-double-count", + "agent": { + "name": "opencode", + "version": "1.14.33", + "model_name": "test-provider/test-model", + }, + "final_metrics": { + # If the trajectory span were to keep these as attributes, the rollup + # would double-count against the per-step metrics below. + "total_prompt_tokens": 30000, + "total_completion_tokens": 600, + "total_cached_tokens": 0, + "total_cost_usd": 0.45, + "total_steps": 2, + }, + "steps": [ + { + "step_id": 1, + "timestamp": "2026-05-04T19:00:00Z", + "source": "user", + "message": "Help me with a task.", + }, + { + "step_id": 2, + "timestamp": "2026-05-04T19:00:05Z", + "source": "agent", + "model_name": "test-provider/test-model", + "message": "Here is the first response.", + "metrics": { + "prompt_tokens": 12000, + "completion_tokens": 250, + "cached_tokens": 0, + "cost_usd": 0.18, + }, + }, + { + "step_id": 3, + "timestamp": "2026-05-04T19:00:10Z", + "source": "agent", + "model_name": "test-provider/test-model", + "message": "Here is the follow-up.", + "metrics": { + "prompt_tokens": 18000, + "completion_tokens": 350, + "cached_tokens": 0, + "cost_usd": 0.27, + }, + }, + ], + } + + ingest_response = client.post("/apis/intake/v2/workspaces/default/ingest/atif", json=body) + assert ingest_response.status_code == 201, ingest_response.text + + # Trajectory span (kind=AGENT) carries no token or cost attributes. + spans_response = client.get( + "/apis/intake/v2/workspaces/default/spans", + params={ + "filter[session_id]": body["session_id"], + "page_size": 20, + "sort": "started_at", + }, + ) + assert spans_response.status_code == 200, spans_response.text + spans = spans_response.json()["data"] + trajectory = next(span for span in spans if span["kind"] == "AGENT") + for field in ( + "input_tokens", + "output_tokens", + "cached_tokens", + "total_tokens", + "cost_usd", + "cost_input_usd", + "cost_output_usd", + "cost_total_usd", + ): + assert field not in trajectory, f"trajectory span unexpectedly carries {field}: {trajectory.get(field)}" + + # Per-step LLM spans carry their own metrics. + llm_steps = sorted( + (span for span in spans if span["kind"] == "LLM"), + key=lambda span: span["started_at"], + ) + assert len(llm_steps) == 2 + assert llm_steps[0]["input_tokens"] == 12000 + assert llm_steps[0]["output_tokens"] == 250 + assert llm_steps[1]["input_tokens"] == 18000 + assert llm_steps[1]["output_tokens"] == 350 + + # Trace-level rollup equals the sum of per-step metrics, NOT 2x. + traces_response = client.get( + "/apis/intake/v2/workspaces/default/traces", + params={"filter[session_id]": body["session_id"], "page_size": 10}, + ) + assert traces_response.status_code == 200, traces_response.text + traces = traces_response.json()["data"] + assert len(traces) == 1 + trace = traces[0] + assert trace["input_tokens"] == 30000, ( + f"expected per-step sum (30000); got {trace['input_tokens']} — double-counted?" + ) + assert trace["output_tokens"] == 600 + assert trace["total_tokens"] == 30600 + assert trace["cost_usd"] == pytest.approx(0.45) diff --git a/services/intake/tests/integration/spans/test_chat_completions_ingest.py b/services/intake/tests/integration/spans/test_chat_completions_ingest.py index 0b26d66d60..be328df19f 100644 --- a/services/intake/tests/integration/spans/test_chat_completions_ingest.py +++ b/services/intake/tests/integration/spans/test_chat_completions_ingest.py @@ -12,6 +12,7 @@ INGEST_URL = "/apis/intake/v2/workspaces/default/ingest/chat-completions" SPANS_URL = "/apis/intake/v2/workspaces/default/spans" +TRACES_URL = "/apis/intake/v2/workspaces/default/traces" EVALUATION_CONTEXT = { "evaluation_id": "chat-eval", "evaluation_sha": "chat-eval-sha", @@ -224,6 +225,17 @@ def test_chat_completions_ingest_groups_by_session_id(client: TestClient): assert {s["span_id"] for s in spans} == {"chatcmpl-turn-0", "chatcmpl-turn-1"} assert all(s["session_id"] == "shared-session" for s in spans) + traces_response = client.get( + TRACES_URL, + params={"filter[session_id]": "shared-session", "mode": "summary", "page_size": 10}, + ) + assert traces_response.status_code == 200, traces_response.text + traces = traces_response.json()["data"] + assert len(traces) == 2 + assert {trace["id"] for trace in traces} == {"chatcmpl-turn-0", "chatcmpl-turn-1"} + assert {trace["session_id"] for trace in traces} == {"shared-session"} + assert all("total_tokens" not in trace for trace in traces) + # --------------------------------------------------------------------------- # error response diff --git a/services/intake/tests/integration/spans/test_traces_read.py b/services/intake/tests/integration/spans/test_traces_read.py new file mode 100644 index 0000000000..305e26c9a4 --- /dev/null +++ b/services/intake/tests/integration/spans/test_traces_read.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Trace summary read API tests.""" + +from datetime import datetime, timezone +from decimal import Decimal + +from fastapi.testclient import TestClient + + +def test_traces_read_returns_core_trace_summary(client: TestClient, make_otlp_request): + base_ns = int(datetime.now(timezone.utc).replace(microsecond=0).timestamp() * 1_000_000_000) + body = make_otlp_request( + [ + { + "name": "root-agent", + "span_id": "0000000000000001", + "start_time_unix_nano": base_ns, + "end_time_unix_nano": base_ns + 100_000_000, + "attributes": { + "openinference.span.kind": "AGENT", + "gen_ai.conversation.id": "trace-session", + "project": "project-a", + "evaluation.run_id": "run-a", + "dataset.name": "dataset-a", + "evaluation.metadata": {"split": "dev"}, + "deployment.environment.name": "prod", + "tag.tags": ["trace-read"], + "metadata": {"owner": "trace-test"}, + "input.value": '{"task":"solve"}', + "output.value": '{"answer":"done"}', + }, + }, + { + "name": "llm-call", + "span_id": "0000000000000002", + "parent_span_id": "0000000000000001", + "start_time_unix_nano": base_ns + 1_000_000_000, + "end_time_unix_nano": base_ns + 1_200_000_000, + "attributes": { + "openinference.span.kind": "LLM", + "gen_ai.conversation.id": "trace-session", + "gen_ai.system": "openai", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.usage.input_tokens": 420, + "gen_ai.usage.output_tokens": 310, + "gen_ai.usage.cached_tokens": 128, + "gen_ai.usage.total_tokens": 858, + "llm.cost.prompt": 0.0024, + "llm.cost.completion": 0.0037, + "llm.cost.total": 0.0061, + }, + }, + ] + ) + + ingest_response = client.post( + "/apis/intake/v2/workspaces/default/ingest/otlp/v1/traces", + content=body, + headers={"Content-Type": "application/x-protobuf"}, + ) + assert ingest_response.status_code == 200, ingest_response.text + + traces_response = client.get( + "/apis/intake/v2/workspaces/default/traces", + params={ + "filter[session_id]": "trace-session", + "filter[evaluation_run_id]": "run-a", + "page_size": 20, + }, + ) + assert traces_response.status_code == 200, traces_response.text + payload = traces_response.json() + assert payload["pagination"]["total_results"] == 1 + trace = payload["data"][0] + assert trace["id"] == "00000000000000000000000000000001" + assert trace["session_id"] == "trace-session" + assert trace["workspace"] == "default" + assert trace["root_span_id"] == "0000000000000001" + assert trace["name"] == "root-agent" + assert trace["status"] == "success" + assert trace["input_tokens"] == 420 + assert trace["output_tokens"] == 310 + assert trace["cached_tokens"] == 128 + assert trace["total_tokens"] == 858 + assert Decimal(str(trace["cost_usd"])) == Decimal("0.0061") + assert Decimal(str(trace["cost_input_usd"])) == Decimal("0.0024") + assert Decimal(str(trace["cost_output_usd"])) == Decimal("0.0037") + assert trace["span_count"] == 2 + assert trace["error_count"] == 0 + assert trace["evaluation_context"]["evaluation_run_id"] == "run-a" + assert trace["evaluation_context"]["dataset_name"] == "dataset-a" + assert trace["evaluation_context"]["metadata"] == {"split": "dev"} + assert "source_format" not in trace + assert "input" not in trace + assert "output" not in trace + assert "project" not in trace + assert "models" not in trace + + get_response = client.get(f"/apis/intake/v2/workspaces/default/traces/{trace['id']}") + assert get_response.status_code == 200, get_response.text + assert get_response.json()["id"] == trace["id"] + + summary_response = client.get( + "/apis/intake/v2/workspaces/default/traces", + params={"filter[session_id]": "trace-session", "mode": "summary", "page_size": 20}, + ) + assert summary_response.status_code == 200, summary_response.text + summary_trace = summary_response.json()["data"][0] + assert summary_trace["id"] == trace["id"] + assert summary_trace["status"] == "success" + assert summary_trace["evaluation_context"]["evaluation_run_id"] == "run-a" + assert "input_tokens" not in summary_trace + assert "cost_usd" not in summary_trace + assert "span_count" not in summary_trace + + +def test_traces_read_picks_earliest_root_when_trace_has_multiple_roots(client: TestClient, make_otlp_request): + base_ns = int(datetime.now(timezone.utc).replace(microsecond=0).timestamp() * 1_000_000_000) + body = make_otlp_request( + [ + { + "name": "earliest-root", + "span_id": "0000000000000101", + "start_time_unix_nano": base_ns, + "end_time_unix_nano": base_ns + 100_000_000, + "attributes": {"gen_ai.conversation.id": "multi-root-session"}, + }, + { + "name": "later-root", + "span_id": "0000000000000102", + "start_time_unix_nano": base_ns + 1_000_000_000, + "end_time_unix_nano": base_ns + 1_100_000_000, + "attributes": {"gen_ai.conversation.id": "multi-root-session"}, + }, + ], + trace_id="00000000000000000000000000000101", + ) + + ingest_response = client.post( + "/apis/intake/v2/workspaces/default/ingest/otlp/v1/traces", + content=body, + headers={"Content-Type": "application/x-protobuf"}, + ) + assert ingest_response.status_code == 200, ingest_response.text + + traces_response = client.get( + "/apis/intake/v2/workspaces/default/traces", + params={"filter[session_id]": "multi-root-session", "page_size": 20}, + ) + assert traces_response.status_code == 200, traces_response.text + payload = traces_response.json() + assert payload["pagination"]["total_results"] == 1 + trace = payload["data"][0] + assert trace["id"] == "00000000000000000000000000000101" + assert trace["name"] == "earliest-root" + assert trace["root_span_id"] == "0000000000000101" diff --git a/services/intake/tests/test_atif_v17.py b/services/intake/tests/test_atif_v17.py index 6b56066d63..f76bf0fe92 100644 --- a/services/intake/tests/test_atif_v17.py +++ b/services/intake/tests/test_atif_v17.py @@ -9,9 +9,11 @@ import pytest from nmp.intake.spans.api.spans_schemas import Span +from nmp.intake.spans.domain import SpanStatus from nmp.intake.spans.ingest.atif import AtifIngestRequest from nmp.intake.spans.ingest.atif_domain import ( AtifAgent, + AtifStepAgent, AtifStepUser, AtifSubagentTrajectoryRef, AtifTrajectory, @@ -102,7 +104,7 @@ def test_atif_v17_embedded_subagent_trajectories_are_preserved_but_not_expanded( ingested_at=datetime(2026, 5, 18, tzinfo=timezone.utc), ) - assert [span.name for span in spans] == ["atif-trajectory"] + assert [span.name for span in spans] == ["root"] root_raw = json.loads(spans[0].attributes_string["atif.raw"]) assert root_raw["subagent_trajectories"][0]["trajectory_id"] == "sub-trajectory" assert root_raw["subagent_trajectories"][0]["steps"][0]["message"] == "subagent work" @@ -154,7 +156,11 @@ def test_atif_v17_subagent_ref_requires_resolution_key() -> None: ], } ) - legacy_ref = legacy.steps[0].observation.results[0].subagent_trajectory_ref[0] + assert isinstance(legacy.steps[0], AtifStepAgent) + assert legacy.steps[0].observation is not None + legacy_refs = legacy.steps[0].observation.results[0].subagent_trajectory_ref + assert legacy_refs is not None + legacy_ref = legacy_refs[0] assert legacy_ref.session_id == "trace-session-id" assert AtifSubagentTrajectoryRef(trajectory_id="sub-trajectory").trajectory_id == "sub-trajectory" @@ -207,7 +213,7 @@ def test_atif_mapping_writes_evaluation_context_only_on_root_span() -> None: ingested_at=datetime(2026, 5, 18, tzinfo=timezone.utc), ) - root = next(span for span in spans if span.name == "atif-trajectory") + root = next(span for span in spans if span.name == "sample-agent") child = next(span for span in spans if span.name == "user-1") assert root.attributes_string["evaluation.id"] == EVALUATION_CONTEXT["evaluation_id"] assert root.attributes_string["evaluation.sha"] == EVALUATION_CONTEXT["evaluation_sha"] @@ -239,6 +245,46 @@ def test_atif_mapping_writes_evaluation_context_only_on_root_span() -> None: assert "evaluation.metadata" not in child.attributes_string +def test_atif_mapping_populates_root_content_and_rolls_child_errors() -> None: + trajectory = AtifTrajectory.model_validate( + { + "schema_version": "ATIF-v1.7", + "session_id": "trace-session-id", + "agent": {"name": "sample-agent", "version": "1.0.0"}, + "steps": [ + {"step_id": 1, "source": "user", "message": "solve the task"}, + { + "step_id": 2, + "source": "agent", + "message": "using a tool", + "tool_calls": [{"tool_call_id": "call-1", "function_name": "Bash"}], + "observation": { + "results": [ + { + "source_call_id": "call-1", + "content": "Exit code 1\n[error] failed", + } + ] + }, + }, + {"step_id": 3, "source": "agent", "message": "final answer"}, + ], + } + ) + + spans = trajectory_to_spans( + workspace="default", + trajectory=trajectory, + ingested_at=datetime(2026, 5, 18, tzinfo=timezone.utc), + ) + + root = spans[0] + assert root.name == "sample-agent" + assert root.input == "solve the task" + assert root.output == "final answer" + assert root.status == SpanStatus.ERROR + + def test_atif_mapping_span_ids_are_trace_native_and_ignore_evaluation_run_id() -> None: base = { "schema_version": "ATIF-v1.5", diff --git a/services/intake/tests/test_spans_schemas.py b/services/intake/tests/test_spans_schemas.py index fce2c5db05..69c5231b28 100644 --- a/services/intake/tests/test_spans_schemas.py +++ b/services/intake/tests/test_spans_schemas.py @@ -8,7 +8,8 @@ import pytest from nmp.intake.spans.api.spans_schemas import Span -from nmp.intake.spans.domain import IntakeSpan, SpanKind, SpanStatus +from nmp.intake.spans.api.traces_schemas import Trace +from nmp.intake.spans.domain import IntakeSpan, IntakeTrace, SpanKind, SpanStatus, TraceEvaluationContext from nmp.intake.spans.storage import json_dumps_preserve from pydantic import ValidationError @@ -59,3 +60,70 @@ def test_span_response_raw_attributes_merges_atif_raw_with_unknown_attributes(): "custom.number": 1.25, "custom.bool": True, } + + +def test_trace_response_maps_core_trace_fields(): + started_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + ended_at = datetime(2026, 1, 1, 0, 0, 2, 500000, tzinfo=timezone.utc) + trace = IntakeTrace( + id="trace-a", + workspace="workspace-a", + session_id="session-a", + source_format="otel", + root_span_id="span-root", + name="root", + input="root input", + output="root output", + environment="prod", + tags=["red", "blue"], + metadata={"owner": "intake"}, + project="project-a", + evaluation_context=TraceEvaluationContext( + evaluation_id="eval-a", + evaluation_run_id="run-a", + dataset_name="dataset-a", + metadata={"split": "dev"}, + ), + started_at=started_at, + ended_at=ended_at, + duration_ms=2500, + ingested_at=ended_at, + status=SpanStatus.ERROR, + input_tokens=420, + output_tokens=310, + cached_tokens=128, + total_tokens=858, + cost_usd=0.0061, + cost_input_usd=0.0024, + cost_output_usd=0.0037, + models=["model-a"], + providers=["openai"], + span_count=2, + error_count=1, + ) + + response = Trace.from_domain(trace) + + assert response.id == "trace-a" + assert response.root_span_id == "span-root" + assert response.session_id == "session-a" + assert response.workspace == "workspace-a" + assert response.name == "root" + assert response.started_at == started_at + assert response.ended_at == ended_at + assert response.status == SpanStatus.ERROR + assert response.duration_ms == 2500 + assert response.input_tokens == 420 + assert response.output_tokens == 310 + assert response.cached_tokens == 128 + assert response.total_tokens == 858 + assert response.cost_usd == 0.0061 + assert response.cost_input_usd == 0.0024 + assert response.cost_output_usd == 0.0037 + assert response.span_count == 2 + assert response.error_count == 1 + assert response.evaluation_context is not None + assert response.evaluation_context.evaluation_id == "eval-a" + assert response.evaluation_context.evaluation_run_id == "run-a" + assert response.evaluation_context.dataset_name == "dataset-a" + assert response.evaluation_context.metadata == {"split": "dev"} diff --git a/services/intake/tests/test_traces_api.py b/services/intake/tests/test_traces_api.py new file mode 100644 index 0000000000..d6f01bfea3 --- /dev/null +++ b/services/intake/tests/test_traces_api.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Trace API filter tests.""" + +import json +from datetime import datetime, timezone + +from nmp.common.api.filter import parse_json_filter +from nmp.common.api.parsed_filter import ParsedFilter +from nmp.intake.spans.api.traces import _trace_filter +from nmp.intake.spans.domain import SpanStatus + + +def test_trace_filter_maps_public_fields_to_repository_filter(): + started_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + filters = _trace_filter( + "workspace-a", + _parsed_filter( + { + "id": "trace-a", + "session_id": "session-a", + "status": "error", + "started_at": {"$gte": started_at.isoformat()}, + "evaluation_run_id": "run-a", + } + ), + ) + + assert filters.workspace == "workspace-a" + assert filters.trace_id == "trace-a" + assert filters.session_id == "session-a" + assert filters.status == SpanStatus.ERROR + assert filters.started_at_gte == started_at + assert len(filters.root_attribute_filters) == 1 + assert filters.root_attribute_filters[0].field == "evaluation_run_id" + assert filters.root_attribute_filters[0].value == "run-a" + assert not filters.span_attribute_filters + + +def _parsed_filter(value: dict[str, object]) -> ParsedFilter: + return ParsedFilter(operation=parse_json_filter(json.dumps(value))) diff --git a/services/intake/tests/test_traces_clickhouse_repository.py b/services/intake/tests/test_traces_clickhouse_repository.py new file mode 100644 index 0000000000..9bc9deaff5 --- /dev/null +++ b/services/intake/tests/test_traces_clickhouse_repository.py @@ -0,0 +1,253 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Trace repository tests.""" + +from datetime import datetime, timedelta, timezone +from typing import cast + +import pytest +from nmp.intake.spans.clickhouse_client import ClickHouseSpanClient +from nmp.intake.spans.domain import SpanAttributeFilter, TraceListFilter +from nmp.intake.spans.trace_repository import TRACE_COLUMNS, TraceRepository, _order_by + + +class _QueryResult: + def __init__(self, rows: list[tuple[object, ...]], columns: list[str] | None = None) -> None: + self.result_rows = rows + self.column_names = columns or [] + + +class _Client: + def __init__(self, query_results: list[_QueryResult] | None = None) -> None: + self.queries: list[str] = [] + self.parameters: list[dict[str, object]] = [] + self.query_results = query_results or [] + + def table(self, name: str) -> str: + return name + + async def query(self, query: str, *, parameters: dict[str, object]) -> _QueryResult: + self.queries.append(query) + self.parameters.append(parameters) + if self.query_results: + return self.query_results.pop(0) + if query.lstrip().startswith("SELECT count()"): + return _QueryResult([(0,)]) + return _QueryResult([]) + + +def _repository(client: _Client) -> TraceRepository: + return TraceRepository(cast(ClickHouseSpanClient, client)) + + +def test_order_by_whitelists_supported_trace_sort_keys(): + assert _order_by("started_at") == "started_at ASC, id ASC" + assert _order_by("-started_at") == "started_at DESC, id ASC" + + +def test_order_by_rejects_unsupported_trace_sort_keys(): + with pytest.raises(ValueError, match="Unsupported trace sort field"): + _order_by("started_at DESC; DROP TABLE spans") + + +@pytest.mark.asyncio +async def test_summary_mode_reads_root_spans_without_metric_aggregates(): + client = _Client() + repository = _repository(client) + + await repository.list_traces( + filters=TraceListFilter(workspace="workspace-a"), + page=1, + page_size=10, + sort="started_at", + mode="summary", + ) + + assert client.queries[0].lstrip().startswith("SELECT count()") + assert "FINAL" not in client.queries[0] + assert ( + "argMax(span_versions.status, (span_versions.event_ts, span_versions.is_deleted)) AS status" + in client.queries[0] + ) + assert "AS root_spans" in client.queries[0] + assert "root_spans.external_parent_span_id = ''" in client.queries[0] + assert "LIMIT 1 BY root_spans.workspace, root_spans.source_format, root_spans.trace_id" in client.queries[0] + assert "sumIf" not in client.queries[0] + assert "groupUniqArrayIf" not in client.queries[0] + + +@pytest.mark.asyncio +async def test_detailed_mode_adds_trace_aggregate_block(): + client = _Client() + repository = _repository(client) + + await repository.list_traces( + filters=TraceListFilter(workspace="workspace-a"), + page=1, + page_size=10, + sort="started_at", + mode="detailed", + ) + + assert "FINAL" not in client.queries[0] + assert "AS root_spans" in client.queries[0] + assert "AS trace_spans" in client.queries[0] + assert "sumIf" in client.queries[0] + assert "groupUniqArrayIf" in client.queries[0] + assert "count() AS span_count" in client.queries[0] + + +@pytest.mark.asyncio +async def test_list_traces_maps_detailed_row(): + started_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + ended_at = started_at + timedelta(milliseconds=2500) + ingested_at = started_at + timedelta(seconds=3) + row = _trace_row(started_at=started_at, ended_at=ended_at, ingested_at=ingested_at) + client = _Client(query_results=[_QueryResult([(1,)]), _QueryResult([row], TRACE_COLUMNS)]) + repository = _repository(client) + + result = await repository.list_traces( + filters=TraceListFilter(workspace="workspace-a"), + page=1, + page_size=10, + sort="-started_at", + mode="detailed", + ) + + trace = result.data[0] + assert trace.id == "trace-a" + assert trace.session_id == "session-a" + assert trace.root_span_id == "span-root" + assert trace.name == "root" + assert trace.input == "root input" + assert trace.output == "root output" + assert trace.duration_ms == 2500 + assert trace.project == "project-a" + assert trace.evaluation_context is not None + assert trace.evaluation_context.evaluation_run_id == "run-a" + assert trace.input_tokens == 420 + assert trace.output_tokens == 310 + assert trace.cached_tokens == 128 + assert trace.total_tokens == 858 + assert trace.cost_usd == 0.0061 + assert trace.cost_input_usd == 0.0024 + assert trace.cost_output_usd == 0.0037 + assert trace.models == ["model-a", "model-b"] + assert trace.providers == ["openai"] + assert trace.span_count == 3 + assert trace.error_count == 1 + + +@pytest.mark.asyncio +async def test_summary_mode_maps_no_aggregate_fields(): + started_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + row = _trace_row(started_at=started_at, ended_at=None, ingested_at=started_at, detailed=False) + client = _Client(query_results=[_QueryResult([(1,)]), _QueryResult([row], TRACE_COLUMNS)]) + repository = _repository(client) + + result = await repository.list_traces( + filters=TraceListFilter(workspace="workspace-a"), + page=1, + page_size=10, + sort="-started_at", + mode="summary", + ) + + trace = result.data[0] + assert trace.status.value == "error" + assert trace.input_tokens is None + assert trace.total_tokens is None + assert trace.cost_usd is None + assert trace.models is None + assert trace.providers is None + assert trace.span_count is None + assert trace.error_count is None + + +@pytest.mark.asyncio +async def test_trace_started_at_filter_is_applied_to_root_spans(): + started_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + client = _Client() + repository = _repository(client) + + await repository.list_traces( + filters=TraceListFilter(workspace="workspace-a", started_at_gte=started_at), + page=1, + page_size=10, + sort="started_at", + mode="summary", + ) + + assert "root_spans.start_time >= %(started_at_gte)s" in client.queries[0] + assert client.parameters[0]["started_at_gte"] == started_at + + +@pytest.mark.asyncio +async def test_root_and_any_span_filters_select_candidate_trace_ids(): + client = _Client() + repository = _repository(client) + + await repository.list_traces( + filters=TraceListFilter( + workspace="workspace-a", + root_attribute_filters=[ + SpanAttributeFilter(field="evaluation_run_id", operator="$eq", value="run-a"), + ], + span_attribute_filters=[ + SpanAttributeFilter(field="model", operator="$eq", value="model-a"), + ], + ), + page=1, + page_size=10, + sort="started_at", + mode="detailed", + ) + + assert "(root_spans.workspace, root_spans.source_format, root_spans.trace_id) IN" in client.queries[0] + assert "(trace_spans.workspace, trace_spans.source_format, trace_spans.trace_id) IN" in client.queries[0] + assert "FINAL" not in client.queries[0] + assert "external_parent_span_id = ''" in client.queries[0] + assert client.parameters[0]["root_candidate_0_key"] == "evaluation.run_id" + assert client.parameters[0]["root_candidate_0_value"] == "run-a" + assert client.parameters[0]["span_candidate_0_key"] == "gen_ai.request.model" + assert client.parameters[0]["span_candidate_0_value"] == "model-a" + + +def _trace_row( + *, + started_at: datetime, + ended_at: datetime | None, + ingested_at: datetime, + detailed: bool = True, +) -> tuple[object, ...]: + values: dict[str, object | None] = { + "id": "trace-a", + "workspace": "workspace-a", + "session_id": "session-a", + "source_format": "otel", + "root_span_id": "span-root", + "name": "root", + "input": "root input", + "output": "root output", + "started_at": started_at, + "ended_at": ended_at, + "status": "error", + "input_tokens": 420 if detailed else None, + "output_tokens": 310 if detailed else None, + "cached_tokens": 128 if detailed else None, + "total_tokens": 858 if detailed else None, + "cost_usd": 0.0061 if detailed else None, + "cost_input_usd": 0.0024 if detailed else None, + "cost_output_usd": 0.0037 if detailed else None, + "models": ["model-a", "model-b"] if detailed else None, + "providers": ["openai"] if detailed else None, + "span_count": 3 if detailed else None, + "error_count": 1 if detailed else None, + "root_attributes_string": { + "project.name": "project-a", + "evaluation.run_id": "run-a", + }, + "ingested_at": ingested_at, + } + return tuple(values[column] for column in TRACE_COLUMNS) diff --git a/web/packages/sdk/generated/platform/api.ts b/web/packages/sdk/generated/platform/api.ts index 7a52c7aaab..15f074dc70 100644 --- a/web/packages/sdk/generated/platform/api.ts +++ b/web/packages/sdk/generated/platform/api.ts @@ -135,6 +135,7 @@ import type { GatewayProxyPostBody, GatewayProxyPut200, GatewayProxyPutBody, + GetTraceParams, GuardrailCheckRequest, GuardrailCheckResponse, GuardrailConfig, @@ -159,6 +160,7 @@ import type { ListFilesetFilesResponse, ListSpansParams, ListTasksParams, + ListTracesParams, ListVirtualModelsParams, LogQueryRequest, MetricEvaluationJob, @@ -266,6 +268,8 @@ import type { ToolCallingMetricResponse, TopicAdherenceMetricInput, TopicAdherenceMetricResponse, + Trace, + TracesPage, UpdateAdapterRequest, UpdateFilesetRequest, UpdateModelDeploymentConfigRequest, @@ -22834,6 +22838,426 @@ export function useListEvaluatorResultsForSpanSuspense< return { ...query, queryKey: queryOptions.queryKey }; } +/** + * @summary List Traces + */ +export const listTraces = (workspace: string, params?: ListTracesParams, signal?: AbortSignal) => { + return customFetch({ + url: `/apis/intake/v2/workspaces/${encodeURIComponent(String(workspace))}/traces`, + method: 'GET', + params, + signal, + }); +}; + +export const getListTracesQueryKey = (workspace: string, params?: ListTracesParams) => { + return [`/apis/intake/v2/workspaces/${workspace}/traces`, ...(params ? [params] : [])] as const; +}; + +export const getListTracesQueryOptions = < + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + } +) => { + const { query: queryOptions } = options ?? {}; + + const queryKey = queryOptions?.queryKey ?? getListTracesQueryKey(workspace, params); + + const queryFn: QueryFunction>> = ({ signal }) => + listTraces(workspace, params, signal); + + return { queryKey, queryFn, enabled: !!workspace, ...queryOptions } as UseQueryOptions< + Awaited>, + TError, + TData + > & { queryKey: DataTag }; +}; + +export type ListTracesQueryResult = NonNullable>>; +export type ListTracesQueryError = ErrorType; + +export function useListTraces< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params: undefined | ListTracesParams, + options: { + query: Partial>, TError, TData>> & + Pick< + DefinedInitialDataOptions< + Awaited>, + TError, + Awaited> + >, + 'initialData' + >; + }, + queryClient?: QueryClient +): DefinedUseQueryResult & { queryKey: DataTag }; +export function useListTraces< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>> & + Pick< + UndefinedInitialDataOptions< + Awaited>, + TError, + Awaited> + >, + 'initialData' + >; + }, + queryClient?: QueryClient +): UseQueryResult & { queryKey: DataTag }; +export function useListTraces< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseQueryResult & { queryKey: DataTag }; +/** + * @summary List Traces + */ + +export function useListTraces< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseQueryResult & { queryKey: DataTag } { + const queryOptions = getListTracesQueryOptions(workspace, params, options); + + const query = useQuery(queryOptions, queryClient) as UseQueryResult & { + queryKey: DataTag; + }; + + return { ...query, queryKey: queryOptions.queryKey }; +} + +export const getListTracesSuspenseQueryOptions = < + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + } +) => { + const { query: queryOptions } = options ?? {}; + + const queryKey = queryOptions?.queryKey ?? getListTracesQueryKey(workspace, params); + + const queryFn: QueryFunction>> = ({ signal }) => + listTraces(workspace, params, signal); + + return { queryKey, queryFn, ...queryOptions } as UseSuspenseQueryOptions< + Awaited>, + TError, + TData + > & { queryKey: DataTag }; +}; + +export type ListTracesSuspenseQueryResult = NonNullable>>; +export type ListTracesSuspenseQueryError = ErrorType; + +export function useListTracesSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params: undefined | ListTracesParams, + options: { + query: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag }; +export function useListTracesSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag }; +export function useListTracesSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag }; +/** + * @summary List Traces + */ + +export function useListTracesSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + params?: ListTracesParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag } { + const queryOptions = getListTracesSuspenseQueryOptions(workspace, params, options); + + const query = useSuspenseQuery(queryOptions, queryClient) as UseSuspenseQueryResult< + TData, + TError + > & { queryKey: DataTag }; + + return { ...query, queryKey: queryOptions.queryKey }; +} + +/** + * @summary Get Trace + */ +export const getTrace = ( + workspace: string, + id: string, + params?: GetTraceParams, + signal?: AbortSignal +) => { + return customFetch({ + url: `/apis/intake/v2/workspaces/${encodeURIComponent(String(workspace))}/traces/${encodeURIComponent(String(id))}`, + method: 'GET', + params, + signal, + }); +}; + +export const getGetTraceQueryKey = (workspace: string, id: string, params?: GetTraceParams) => { + return [ + `/apis/intake/v2/workspaces/${workspace}/traces/${id}`, + ...(params ? [params] : []), + ] as const; +}; + +export const getGetTraceQueryOptions = < + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + } +) => { + const { query: queryOptions } = options ?? {}; + + const queryKey = queryOptions?.queryKey ?? getGetTraceQueryKey(workspace, id, params); + + const queryFn: QueryFunction>> = ({ signal }) => + getTrace(workspace, id, params, signal); + + return { queryKey, queryFn, enabled: !!(workspace && id), ...queryOptions } as UseQueryOptions< + Awaited>, + TError, + TData + > & { queryKey: DataTag }; +}; + +export type GetTraceQueryResult = NonNullable>>; +export type GetTraceQueryError = ErrorType; + +export function useGetTrace< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params: undefined | GetTraceParams, + options: { + query: Partial>, TError, TData>> & + Pick< + DefinedInitialDataOptions< + Awaited>, + TError, + Awaited> + >, + 'initialData' + >; + }, + queryClient?: QueryClient +): DefinedUseQueryResult & { queryKey: DataTag }; +export function useGetTrace< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>> & + Pick< + UndefinedInitialDataOptions< + Awaited>, + TError, + Awaited> + >, + 'initialData' + >; + }, + queryClient?: QueryClient +): UseQueryResult & { queryKey: DataTag }; +export function useGetTrace< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseQueryResult & { queryKey: DataTag }; +/** + * @summary Get Trace + */ + +export function useGetTrace< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseQueryResult & { queryKey: DataTag } { + const queryOptions = getGetTraceQueryOptions(workspace, id, params, options); + + const query = useQuery(queryOptions, queryClient) as UseQueryResult & { + queryKey: DataTag; + }; + + return { ...query, queryKey: queryOptions.queryKey }; +} + +export const getGetTraceSuspenseQueryOptions = < + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + } +) => { + const { query: queryOptions } = options ?? {}; + + const queryKey = queryOptions?.queryKey ?? getGetTraceQueryKey(workspace, id, params); + + const queryFn: QueryFunction>> = ({ signal }) => + getTrace(workspace, id, params, signal); + + return { queryKey, queryFn, ...queryOptions } as UseSuspenseQueryOptions< + Awaited>, + TError, + TData + > & { queryKey: DataTag }; +}; + +export type GetTraceSuspenseQueryResult = NonNullable>>; +export type GetTraceSuspenseQueryError = ErrorType; + +export function useGetTraceSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params: undefined | GetTraceParams, + options: { + query: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag }; +export function useGetTraceSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag }; +export function useGetTraceSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag }; +/** + * @summary Get Trace + */ + +export function useGetTraceSuspense< + TData = Awaited>, + TError = ErrorType, +>( + workspace: string, + id: string, + params?: GetTraceParams, + options?: { + query?: Partial>, TError, TData>>; + }, + queryClient?: QueryClient +): UseSuspenseQueryResult & { queryKey: DataTag } { + const queryOptions = getGetTraceSuspenseQueryOptions(workspace, id, params, options); + + const query = useSuspenseQuery(queryOptions, queryClient) as UseSuspenseQueryResult< + TData, + TError + > & { queryKey: DataTag }; + + return { ...query, queryKey: queryOptions.queryKey }; +} + /** * Get all currently configured execution profiles. * @summary Get Execution Profiles diff --git a/web/packages/sdk/generated/platform/schema/ChatCompletionsIngestRequest.ts b/web/packages/sdk/generated/platform/schema/ChatCompletionsIngestRequest.ts index b3c6fd72e0..8fcd9ed18e 100644 --- a/web/packages/sdk/generated/platform/schema/ChatCompletionsIngestRequest.ts +++ b/web/packages/sdk/generated/platform/schema/ChatCompletionsIngestRequest.ts @@ -13,8 +13,9 @@ import type { FlexibleEntryResponse } from './FlexibleEntryResponse'; export interface ChatCompletionsIngestRequest { request: FlexibleEntryRequestInput; response: FlexibleEntryResponse; + /** Groups related chat-completions calls without forcing them into the same trace. */ session_id?: string; - /** Defaults to session_id when omitted. */ + /** Opt into joining an existing trace built via OTel or ATIF. This is not a grouping mechanism for chat-completions calls; use session_id to group related calls. */ trace_id?: string; evaluation_context?: EvaluationContext; provider?: string; diff --git a/web/packages/sdk/generated/platform/schema/GetTraceMode.ts b/web/packages/sdk/generated/platform/schema/GetTraceMode.ts new file mode 100644 index 0000000000..2e490527a3 --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/GetTraceMode.ts @@ -0,0 +1,15 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ + +export type GetTraceMode = (typeof GetTraceMode)[keyof typeof GetTraceMode]; + +export const GetTraceMode = { + summary: 'summary', + detailed: 'detailed', +} as const; diff --git a/web/packages/sdk/generated/platform/schema/GetTraceParams.ts b/web/packages/sdk/generated/platform/schema/GetTraceParams.ts new file mode 100644 index 0000000000..44e76269de --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/GetTraceParams.ts @@ -0,0 +1,16 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ +import type { GetTraceMode } from './GetTraceMode'; + +export type GetTraceParams = { + /** + * Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups. + */ + mode?: GetTraceMode; +}; diff --git a/web/packages/sdk/generated/platform/schema/ListTracesMode.ts b/web/packages/sdk/generated/platform/schema/ListTracesMode.ts new file mode 100644 index 0000000000..1f3c3826b2 --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/ListTracesMode.ts @@ -0,0 +1,15 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ + +export type ListTracesMode = (typeof ListTracesMode)[keyof typeof ListTracesMode]; + +export const ListTracesMode = { + summary: 'summary', + detailed: 'detailed', +} as const; diff --git a/web/packages/sdk/generated/platform/schema/ListTracesParams.ts b/web/packages/sdk/generated/platform/schema/ListTracesParams.ts new file mode 100644 index 0000000000..63a364a55d --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/ListTracesParams.ts @@ -0,0 +1,34 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ +import type { ListTracesMode } from './ListTracesMode'; +import type { TraceFilter } from './TraceFilter'; +import type { TraceSortField } from './TraceSortField'; + +export type ListTracesParams = { + /** + * Page number. + * @minimum 1 + */ + page?: number; + /** + * Page size. + * @minimum 1 + * @maximum 1000 + */ + page_size?: number; + sort?: TraceSortField; + /** + * Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups. + */ + mode?: ListTracesMode; + /** + * Filter root-span-backed traces by id, session_id, rolled-up status, root span started_at, and root-span evaluation context fields. + */ + filter?: TraceFilter; +}; diff --git a/web/packages/sdk/generated/platform/schema/Trace.ts b/web/packages/sdk/generated/platform/schema/Trace.ts new file mode 100644 index 0000000000..d4ae0b7a20 --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/Trace.ts @@ -0,0 +1,38 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ +import type { SpanEvaluationContext } from './SpanEvaluationContext'; +import type { SpanStatus } from './SpanStatus'; + +export interface Trace { + id: string; + root_span_id?: string; + session_id: string; + workspace: string; + name?: string; + evaluation_context?: SpanEvaluationContext; + started_at: string; + ended_at?: string; + duration_ms?: number; + status: SpanStatus; + /** @minimum 0 */ + input_tokens?: number; + /** @minimum 0 */ + output_tokens?: number; + /** @minimum 0 */ + cached_tokens?: number; + /** @minimum 0 */ + total_tokens?: number; + cost_usd?: number; + cost_input_usd?: number; + cost_output_usd?: number; + /** @minimum 0 */ + span_count?: number; + /** @minimum 0 */ + error_count?: number; +} diff --git a/web/packages/sdk/generated/platform/schema/TraceFilter.ts b/web/packages/sdk/generated/platform/schema/TraceFilter.ts new file mode 100644 index 0000000000..56d2074135 --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/TraceFilter.ts @@ -0,0 +1,35 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ +import type { DatetimeFilter } from './DatetimeFilter'; +import type { SpanStatus } from './SpanStatus'; + +export interface TraceFilter { + /** Filter by canonical Intake trace id. */ + id?: string; + /** Filter by session id. */ + session_id?: string; + /** Filter by rolled-up trace status. */ + status?: SpanStatus; + /** Filter by root span start timestamp. */ + started_at?: DatetimeFilter; + /** Filter by root-span evaluation id. */ + evaluation_id?: string; + /** Filter by root-span evaluation sha. */ + evaluation_sha?: string; + /** Filter by root-span evaluation run id. */ + evaluation_run_id?: string; + /** Filter by root-span dataset id. */ + dataset_id?: string; + /** Filter by root-span dataset name. */ + dataset_name?: string; + /** Filter by root-span dataset version. */ + dataset_version?: string; + /** Filter by root-span dataset test case id. */ + test_case_id?: string; +} diff --git a/web/packages/sdk/generated/platform/schema/TraceSortField.ts b/web/packages/sdk/generated/platform/schema/TraceSortField.ts new file mode 100644 index 0000000000..ecd2dccec5 --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/TraceSortField.ts @@ -0,0 +1,15 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ + +export type TraceSortField = (typeof TraceSortField)[keyof typeof TraceSortField]; + +export const TraceSortField = { + started_at: 'started_at', + '-started_at': '-started_at', +} as const; diff --git a/web/packages/sdk/generated/platform/schema/TracesPage.ts b/web/packages/sdk/generated/platform/schema/TracesPage.ts new file mode 100644 index 0000000000..0b5c6d6a7c --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/TracesPage.ts @@ -0,0 +1,21 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ +import type { PaginationData } from './PaginationData'; +import type { Trace } from './Trace'; +import type { TracesPageFilter } from './TracesPageFilter'; + +export interface TracesPage { + data: Trace[]; + /** Pagination information. */ + pagination?: PaginationData; + /** The field on which the results are sorted. */ + sort?: string; + /** Filtering information. */ + filter?: TracesPageFilter; +} diff --git a/web/packages/sdk/generated/platform/schema/TracesPageFilter.ts b/web/packages/sdk/generated/platform/schema/TracesPageFilter.ts new file mode 100644 index 0000000000..6135cb09ff --- /dev/null +++ b/web/packages/sdk/generated/platform/schema/TracesPageFilter.ts @@ -0,0 +1,13 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ + +/** + * Filtering information. + */ +export type TracesPageFilter = { [key: string]: unknown }; diff --git a/web/packages/sdk/generated/platform/schema/index.ts b/web/packages/sdk/generated/platform/schema/index.ts index 946a081372..d0057137d0 100644 --- a/web/packages/sdk/generated/platform/schema/index.ts +++ b/web/packages/sdk/generated/platform/schema/index.ts @@ -434,6 +434,8 @@ export * from './GenerationOptionsLlmParams'; export * from './GenerationRailsOptions'; export * from './GenerationStats'; export * from './GenericSortField'; +export * from './GetTraceMode'; +export * from './GetTraceParams'; export * from './GlinerConfig'; export * from './GLiNERDetection'; export * from './GLiNERDetectionOptions'; @@ -520,6 +522,8 @@ export * from './ListFilesetFilesResponse'; export * from './ListSpansMode'; export * from './ListSpansParams'; export * from './ListTasksParams'; +export * from './ListTracesMode'; +export * from './ListTracesParams'; export * from './ListVirtualModelsParams'; export * from './LLMCallInfo'; export * from './LLMCallInfoRawResponse'; @@ -993,6 +997,11 @@ export * from './TopicAdherenceMetricResponseLabels'; export * from './TopicAdherenceMetricResponseMetricMode'; export * from './TopicAdherenceMetricResponseSupportedJobTypesItem'; export * from './TopicAdherenceMetricSupportedJobTypesItem'; +export * from './Trace'; +export * from './TraceFilter'; +export * from './TraceSortField'; +export * from './TracesPage'; +export * from './TracesPageFilter'; export * from './TracingConfig'; export * from './TrainingHyperparams'; export * from './TrainingHyperparamsQuantizationBits'; diff --git a/web/packages/sdk/generated/platform/zod/ingest.ts b/web/packages/sdk/generated/platform/zod/ingest.ts index 0b23ffa4ca..29c6a8568d 100644 --- a/web/packages/sdk/generated/platform/zod/ingest.ts +++ b/web/packages/sdk/generated/platform/zod/ingest.ts @@ -315,8 +315,16 @@ export const IngestChatCompletionBody = zod.object({ .describe( 'Flexible entry response that accepts any object shape.\n\nThis flexibility enables the Intake service to store responses from various LLM providers\nand future model types without requiring schema updates.\n\nRequired: either `choices` (successful response) or `error` (failed call).\nCommon optional fields: `id`, `created`, `model`, `usage`, `system_fingerprint`, etc.' ), - session_id: zod.string().optional(), - trace_id: zod.string().optional().describe('Defaults to session_id when omitted.'), + session_id: zod + .string() + .optional() + .describe('Groups related chat-completions calls without forcing them into the same trace.'), + trace_id: zod + .string() + .optional() + .describe( + 'Opt into joining an existing trace built via OTel or ATIF. This is not a grouping mechanism for chat-completions calls; use session_id to group related calls.' + ), evaluation_context: zod .object({ evaluation_id: zod.string().optional(), diff --git a/web/packages/sdk/generated/platform/zod/traces.ts b/web/packages/sdk/generated/platform/zod/traces.ts new file mode 100644 index 0000000000..4126a701a9 --- /dev/null +++ b/web/packages/sdk/generated/platform/zod/traces.ts @@ -0,0 +1,201 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Generated by Orval 🍺 + * Do not edit manually. + * Nemo Platform API + */ +import * as zod from 'zod'; + +/** + * @summary List Traces + */ +export const ListTracesParams = zod.object({ + workspace: zod.string(), +}); + +export const listTracesQueryPageDefault = 1; + +export const listTracesQueryPageSizeDefault = 10; +export const listTracesQueryPageSizeMax = 1000; + +export const listTracesQuerySortDefault = `-started_at`; +export const listTracesQueryModeDefault = `detailed`; + +export const ListTracesQueryParams = zod.object({ + page: zod.number().min(1).default(listTracesQueryPageDefault).describe('Page number.'), + page_size: zod + .number() + .min(1) + .max(listTracesQueryPageSizeMax) + .default(listTracesQueryPageSizeDefault) + .describe('Page size.'), + sort: zod.enum(['started_at', '-started_at']).default(listTracesQuerySortDefault), + mode: zod + .enum(['summary', 'detailed']) + .default(listTracesQueryModeDefault) + .describe( + 'Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.' + ), + filter: zod + .object({ + id: zod.string().optional().describe('Filter by canonical Intake trace id.'), + session_id: zod.string().optional().describe('Filter by session id.'), + status: zod + .enum(['success', 'error', 'cancelled', 'unknown']) + .optional() + .describe('Filter by rolled-up trace status.'), + started_at: zod + .object({ + $gte: zod + .string() + .datetime({}) + .optional() + .describe('Filter for results greater than or equal to this datetime.'), + $lte: zod + .string() + .datetime({}) + .optional() + .describe('Filter for results less than or equal to this datetime.'), + }) + .optional() + .describe('Filter by root span start timestamp.'), + evaluation_id: zod.string().optional().describe('Filter by root-span evaluation id.'), + evaluation_sha: zod.string().optional().describe('Filter by root-span evaluation sha.'), + evaluation_run_id: zod.string().optional().describe('Filter by root-span evaluation run id.'), + dataset_id: zod.string().optional().describe('Filter by root-span dataset id.'), + dataset_name: zod.string().optional().describe('Filter by root-span dataset name.'), + dataset_version: zod.string().optional().describe('Filter by root-span dataset version.'), + test_case_id: zod.string().optional().describe('Filter by root-span dataset test case id.'), + }) + .optional() + .describe( + 'Filter root-span-backed traces by id, session_id, rolled-up status, root span started_at, and root-span evaluation context fields.' + ), +}); + +export const listTracesResponseDataItemInputTokensMin = 0; + +export const listTracesResponseDataItemOutputTokensMin = 0; + +export const listTracesResponseDataItemCachedTokensMin = 0; + +export const listTracesResponseDataItemTotalTokensMin = 0; + +export const listTracesResponseDataItemSpanCountMin = 0; + +export const listTracesResponseDataItemErrorCountMin = 0; + +export const ListTracesResponse = zod.object({ + data: zod.array( + zod.object({ + id: zod.string(), + root_span_id: zod.string().optional(), + session_id: zod.string(), + workspace: zod.string(), + name: zod.string().optional(), + evaluation_context: zod + .object({ + evaluation_id: zod.string().optional(), + evaluation_sha: zod.string().optional(), + evaluation_run_id: zod.string().optional(), + dataset_id: zod.string().optional(), + dataset_name: zod.string().optional(), + dataset_version: zod.string().optional(), + test_case_id: zod.string().optional(), + metadata: zod.record(zod.string(), zod.unknown()).optional(), + }) + .optional(), + started_at: zod.string().datetime({}), + ended_at: zod.string().datetime({}).optional(), + duration_ms: zod.number().optional(), + status: zod.enum(['success', 'error', 'cancelled', 'unknown']), + input_tokens: zod.number().min(listTracesResponseDataItemInputTokensMin).optional(), + output_tokens: zod.number().min(listTracesResponseDataItemOutputTokensMin).optional(), + cached_tokens: zod.number().min(listTracesResponseDataItemCachedTokensMin).optional(), + total_tokens: zod.number().min(listTracesResponseDataItemTotalTokensMin).optional(), + cost_usd: zod.number().optional(), + cost_input_usd: zod.number().optional(), + cost_output_usd: zod.number().optional(), + span_count: zod.number().min(listTracesResponseDataItemSpanCountMin).optional(), + error_count: zod.number().min(listTracesResponseDataItemErrorCountMin).optional(), + }) + ), + pagination: zod + .object({ + page: zod.number().describe('The current page number.'), + page_size: zod.number().describe('The page size used for the query.'), + current_page_size: zod.number().describe('The size for the current page.'), + total_pages: zod.number().describe('The total number of pages.'), + total_results: zod.number().describe('The total number of results.'), + }) + .optional() + .describe('Pagination information.'), + sort: zod.string().optional().describe('The field on which the results are sorted.'), + filter: zod.record(zod.string(), zod.unknown()).optional().describe('Filtering information.'), +}); + +/** + * @summary Get Trace + */ +export const GetTraceParams = zod.object({ + workspace: zod.string(), + id: zod.string(), +}); + +export const getTraceQueryModeDefault = `detailed`; + +export const GetTraceQueryParams = zod.object({ + mode: zod + .enum(['summary', 'detailed']) + .default(getTraceQueryModeDefault) + .describe( + 'Use summary for root-span trace fields only, or detailed to include token, cost, and span-count rollups.' + ), +}); + +export const getTraceResponseInputTokensMin = 0; + +export const getTraceResponseOutputTokensMin = 0; + +export const getTraceResponseCachedTokensMin = 0; + +export const getTraceResponseTotalTokensMin = 0; + +export const getTraceResponseSpanCountMin = 0; + +export const getTraceResponseErrorCountMin = 0; + +export const GetTraceResponse = zod.object({ + id: zod.string(), + root_span_id: zod.string().optional(), + session_id: zod.string(), + workspace: zod.string(), + name: zod.string().optional(), + evaluation_context: zod + .object({ + evaluation_id: zod.string().optional(), + evaluation_sha: zod.string().optional(), + evaluation_run_id: zod.string().optional(), + dataset_id: zod.string().optional(), + dataset_name: zod.string().optional(), + dataset_version: zod.string().optional(), + test_case_id: zod.string().optional(), + metadata: zod.record(zod.string(), zod.unknown()).optional(), + }) + .optional(), + started_at: zod.string().datetime({}), + ended_at: zod.string().datetime({}).optional(), + duration_ms: zod.number().optional(), + status: zod.enum(['success', 'error', 'cancelled', 'unknown']), + input_tokens: zod.number().min(getTraceResponseInputTokensMin).optional(), + output_tokens: zod.number().min(getTraceResponseOutputTokensMin).optional(), + cached_tokens: zod.number().min(getTraceResponseCachedTokensMin).optional(), + total_tokens: zod.number().min(getTraceResponseTotalTokensMin).optional(), + cost_usd: zod.number().optional(), + cost_input_usd: zod.number().optional(), + cost_output_usd: zod.number().optional(), + span_count: zod.number().min(getTraceResponseSpanCountMin).optional(), + error_count: zod.number().min(getTraceResponseErrorCountMin).optional(), +});