diff --git a/packages/nmp_platform/config/local.yaml b/packages/nmp_platform/config/local.yaml index 337d577803..9dfd8f13f0 100644 --- a/packages/nmp_platform/config/local.yaml +++ b/packages/nmp_platform/config/local.yaml @@ -71,6 +71,14 @@ jobs: ttl_seconds_before_active: 60 ttl_seconds_active: 3600 ttl_seconds_after_finished: 300 + # Auditor jobs always run as Docker containers (auditor-tasks image). Using a + # dedicated profile "auditor" avoids the cpu→subprocess translation that the + # subprocess/default entry above enables for profile "default". + - provider: cpu + profile: auditor + backend: docker + config: + launcher_tool_path: ./services/core/jobs/jobs-launcher/jobs-launcher # Customizer (automodel / unsloth) GPU jobs. The unsloth backend stamps the # profile name "gpu" onto all 4 steps, so both cpu/gpu and gpu/gpu must # exist. `profile` is a name, not a resource claim — `provider` governs diff --git a/plugins/nemo-auditor/openapi/openapi.yaml b/plugins/nemo-auditor/openapi/openapi.yaml index 3122e1cf87..8977bea627 100644 --- a/plugins/nemo-auditor/openapi/openapi.yaml +++ b/plugins/nemo-auditor/openapi/openapi.yaml @@ -44,7 +44,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuditConfig' + $ref: '#/components/schemas/AuditConfigOutput' '422': description: Validation Error content: @@ -143,7 +143,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuditConfig' + $ref: '#/components/schemas/AuditConfigOutput' '422': description: Validation Error content: @@ -181,7 +181,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuditConfig' + $ref: '#/components/schemas/AuditConfigOutput' '422': description: Validation Error content: @@ -216,6 +216,380 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit: + post: + tags: + - Auditor Jobs + summary: Create Job + operationId: create_job_apis_auditor_v2_workspaces__workspace__jobs_audit_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AuditJobRequest' + responses: + '201': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AuditJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + get: + tags: + - Auditor Jobs + summary: List Jobs + operationId: list_jobs_apis_auditor_v2_workspaces__workspace__jobs_audit_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: page + in: query + required: false + schema: + type: integer + exclusiveMinimum: 0 + description: Page number. + default: 1 + title: Page + description: Page number. + - name: page_size + in: query + required: false + schema: + type: integer + exclusiveMinimum: 0 + description: Page size. + default: 10 + title: Page Size + description: Page size. + - name: sort + in: query + required: false + schema: + allOf: + - $ref: '#/components/schemas/AuditJobsSortField' + description: The field to sort by. To sort in decreasing order, use `-` + in front of the field name. + default: -created_at + description: The field to sort by. To sort in decreasing order, use `-` in + front of the field name. + - in: query + name: filter + style: deepObject + required: false + explode: true + schema: + $ref: '#/components/schemas/AuditJobsListFilter' + description: Filter jobs on various criteria. + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AuditJobsPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{job}/results/{name}: + get: + tags: + - Auditor Jobs + summary: Get Job Result + operationId: get_job_result_apis_auditor_v2_workspaces__workspace__jobs_audit__job__results__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: job + in: path + required: true + schema: + type: string + title: Job + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{job}/results/{name}/download: + get: + tags: + - Auditor Jobs + summary: Download Job Result + operationId: download_job_result_apis_auditor_v2_workspaces__workspace__jobs_audit__job__results__name__download_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: job + in: path + required: true + schema: + type: string + title: Job + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/octet-stream: + schema: + type: string + format: binary + '404': + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{name}: + get: + tags: + - Auditor Jobs + summary: Get Job + operationId: get_job_apis_auditor_v2_workspaces__workspace__jobs_audit__name__get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AuditJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + delete: + tags: + - Auditor Jobs + summary: Delete Job + operationId: delete_job_apis_auditor_v2_workspaces__workspace__jobs_audit__name__delete + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '204': + description: Successful Response + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{name}/cancel: + post: + tags: + - Auditor Jobs + summary: Cancel Job + operationId: cancel_job_apis_auditor_v2_workspaces__workspace__jobs_audit__name__cancel_post + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AuditJob' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{name}/logs: + get: + tags: + - Auditor Jobs + summary: Get Job Logs + operationId: get_job_logs_apis_auditor_v2_workspaces__workspace__jobs_audit__name__logs_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + - name: limit + in: query + required: false + schema: + title: Limit + type: integer + - name: page_cursor + in: query + required: false + schema: + title: Page Cursor + type: string + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobLogPage' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{name}/results: + get: + tags: + - Auditor Jobs + summary: List Job Results + operationId: list_job_results_apis_auditor_v2_workspaces__workspace__jobs_audit__name__results_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobListResultResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' + /apis/auditor/v2/workspaces/{workspace}/jobs/audit/{name}/status: + get: + tags: + - Auditor Jobs + summary: Get Job Status + operationId: get_job_status_apis_auditor_v2_workspaces__workspace__jobs_audit__name__status_get + parameters: + - name: workspace + in: path + required: true + schema: + type: string + title: Workspace + - name: name + in: path + required: true + schema: + type: string + title: Name + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/PlatformJobStatusResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /apis/auditor/v2/workspaces/{workspace}/targets: post: tags: @@ -242,7 +616,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuditTarget' + $ref: '#/components/schemas/AuditTargetOutput' '422': description: Validation Error content: @@ -341,7 +715,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuditTarget' + $ref: '#/components/schemas/AuditTargetOutput' '422': description: Validation Error content: @@ -379,7 +753,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuditTarget' + $ref: '#/components/schemas/AuditTargetOutput' '422': description: Validation Error content: @@ -421,7 +795,40 @@ components: type: object title: AuditClassConfig description: Per-class plugin configuration mapping. - AuditConfig: + AuditConfigInput: + properties: + name: + type: string + title: Name + description: Entity name within the workspace + default: '' + workspace: + type: string + pattern: ^[\w\-\+.@:]+$ + title: Workspace + description: Workspace identifier + project: + title: Project + description: The name of the project associated with this entity. + type: string + description: + title: Description + description: Config description + type: string + system: + $ref: '#/components/schemas/AuditSystemData' + run: + $ref: '#/components/schemas/AuditRunData' + plugins: + $ref: '#/components/schemas/AuditPluginsDataInput' + reporting: + $ref: '#/components/schemas/AuditReportData' + type: object + required: + - workspace + title: AuditConfigInput + description: Audit configuration stored in the entity store. + AuditConfigOutput: properties: name: type: string @@ -457,44 +864,215 @@ components: title: Created At readOnly: true type: string - format: date-time - created_by: - title: Created By - readOnly: true - nullable: true + format: date-time + created_by: + title: Created By + readOnly: true + nullable: true + type: string + updated_at: + title: Updated At + readOnly: true + type: string + format: date-time + updated_by: + title: Updated By + readOnly: true + nullable: true + type: string + entity_id: + type: string + title: Entity Id + description: Alias for id for backwards compatibility. + readOnly: true + parent: + title: Parent + description: Parent entity ID for nested entities. + readOnly: true + type: string + type: object + required: + - workspace + - id + - created_at + - created_by + - updated_at + - updated_by + - entity_id + - parent + title: AuditConfigOutput + description: Audit configuration stored in the entity store. + AuditInputSpec: + properties: + config: + anyOf: + - $ref: '#/components/schemas/AuditConfigInput' + - type: string + minLength: 1 + title: Config + target: + anyOf: + - $ref: '#/components/schemas/AuditTargetInput' + - type: string + minLength: 1 + title: Target + max_probe_retries: + type: integer + minimum: 0.0 + title: Max Probe Retries + default: 0 + fail_job_on_retries_exhausted: + type: boolean + title: Fail Job On Retries Exhausted + default: true + additionalProperties: false + type: object + required: + - config + - target + title: AuditInputSpec + description: "User-facing spec \u2014 each field accepts an inline entity payload\ + \ OR a\nworkspace-qualified name string referencing one in the entity store.\n\ + \nResolved by :meth:`AuditJob.to_spec` into a canonical :class:`AuditSpec`\n\ + before :meth:`AuditJob.run` is invoked." + AuditJob: + properties: + id: + title: Id + type: string + name: + type: string + title: Name + description: + title: Description + type: string + project: + title: Project + type: string + workspace: + title: Workspace + type: string + created_at: + title: Created At + type: string + format: date-time + updated_at: + title: Updated At + type: string + format: date-time + spec: + $ref: '#/components/schemas/AuditSpec' + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + title: Status Details + additionalProperties: true + type: object + error_details: + title: Error Details + additionalProperties: true + type: object + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object + type: object + required: + - name + - spec + title: AuditJob + AuditJobRequest: + properties: + name: + title: Name + type: string + description: + title: Description type: string - updated_at: - title: Updated At - readOnly: true + project: + title: Project type: string - format: date-time - updated_by: - title: Updated By - readOnly: true - nullable: true + spec: + $ref: '#/components/schemas/AuditInputSpec' + ownership: + title: Ownership + additionalProperties: true + type: object + custom_fields: + title: Custom Fields + additionalProperties: true + type: object + type: object + required: + - spec + title: AuditJobRequest + AuditJobsListFilter: + additionalProperties: false + properties: + created_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs created at 'gte' datetime or 'lte' datetime. + name: + anyOf: + - $ref: '#/components/schemas/StringFilter' + - type: string + description: Name of the job. + title: Name + workspace: + description: Workspace of the job. + title: Workspace type: string - entity_id: + project: + description: Project containing the job. + title: Project type: string - title: Entity Id - description: Alias for id for backwards compatibility. - readOnly: true - parent: - title: Parent - description: Parent entity ID for nested entities. - readOnly: true + status: + allOf: + - $ref: '#/components/schemas/PlatformJobStatus' + description: The current status. + updated_at: + allOf: + - $ref: '#/components/schemas/DatetimeFilter' + description: Jobs updated at 'gte' datetime or 'lte' datetime. + title: AuditJobsListFilter + type: object + AuditJobsPage: + properties: + data: + items: + $ref: '#/components/schemas/AuditJob' + type: array + title: Data + pagination: + allOf: + - $ref: '#/components/schemas/PaginationData' + description: Pagination information. + sort: + title: Sort + description: The field on which the results are sorted. type: string + filter: + title: Filter + description: Filtering information. + additionalProperties: true + type: object type: object required: - - workspace - - id + - data + title: AuditJobsPage + AuditJobsSortField: + type: string + enum: - created_at - - created_by + - -created_at - updated_at - - updated_by - - entity_id - - parent - title: AuditConfig - description: Audit configuration stored in the entity store. + - -updated_at + title: AuditJobsSortField AuditModuleConfig: additionalProperties: $ref: '#/components/schemas/AuditClassConfig' @@ -683,6 +1261,28 @@ components: default: garak/{version} (LLM vulnerability scanner https://garak.ai) type: object title: AuditRunData + AuditSpec: + properties: + config: + $ref: '#/components/schemas/AuditConfigOutput' + target: + $ref: '#/components/schemas/AuditTargetOutput' + max_probe_retries: + type: integer + minimum: 0.0 + title: Max Probe Retries + default: 0 + fail_job_on_retries_exhausted: + type: boolean + title: Fail Job On Retries Exhausted + default: true + additionalProperties: false + type: object + required: + - config + - target + title: AuditSpec + description: Canonical, fully-resolved spec passed to :meth:`AuditJob.run`. AuditSystemData: properties: verbose: @@ -720,7 +1320,47 @@ components: additionalProperties: false type: object title: AuditSystemData - AuditTarget: + AuditTargetInput: + properties: + name: + type: string + title: Name + description: Entity name within the workspace + default: '' + workspace: + type: string + pattern: ^[\w\-\+.@:]+$ + title: Workspace + description: Workspace identifier + project: + title: Project + description: The name of the project associated with this entity. + type: string + description: + title: Description + description: Target description + type: string + type: + type: string + title: Type + description: Target type (e.g., 'nim', 'openai'). + model: + type: string + title: Model + description: Model identifier. + options: + additionalProperties: true + type: object + title: Options + description: Additional target options. + type: object + required: + - workspace + - type + - model + title: AuditTargetInput + description: Audit target (model under test) stored in the entity store. + AuditTargetOutput: properties: name: type: string @@ -799,7 +1439,7 @@ components: - updated_by - entity_id - parent - title: AuditTarget + title: AuditTargetOutput description: Audit target (model under test) stored in the entity store. ConfigFilter: additionalProperties: false @@ -889,6 +1529,11 @@ components: type: string title: DatetimeFilter type: object + FileStorageType: + type: string + enum: + - fileset + title: FileStorageType HTTPValidationError: properties: detail: @@ -898,6 +1543,303 @@ components: title: Detail type: object title: HTTPValidationError + PaginationData: + properties: + page: + type: integer + title: Page + description: The current page number. + page_size: + type: integer + title: Page Size + description: The page size used for the query. + current_page_size: + type: integer + title: Current Page Size + description: The size for the current page. + total_pages: + type: integer + title: Total Pages + description: The total number of pages. + total_results: + type: integer + title: Total Results + description: The total number of results. + type: object + required: + - page + - page_size + - current_page_size + - total_pages + - total_results + title: PaginationData + PlatformJobListResultResponse: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobResultResponse' + type: array + title: Data + type: object + required: + - data + title: PlatformJobListResultResponse + PlatformJobLog: + properties: + timestamp: + type: string + format: date-time + title: Timestamp + job: + type: string + title: Job + job_step: + type: string + title: Job Step + job_task: + type: string + title: Job Task + message: + type: string + title: Message + type: object + required: + - timestamp + - job + - job_step + - job_task + - message + title: PlatformJobLog + PlatformJobLogPage: + properties: + data: + items: + $ref: '#/components/schemas/PlatformJobLog' + type: array + title: Data + total: + type: integer + title: Total + next_page: + title: Next Page + type: string + prev_page: + title: Prev Page + type: string + type: object + required: + - data + - total + - next_page + - prev_page + title: PlatformJobLogPage + PlatformJobResultResponse: + properties: + name: + type: string + title: Name + job: + type: string + title: Job + workspace: + type: string + title: Workspace + project: + title: Project + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + artifact_url: + type: string + title: Artifact Url + artifact_storage_type: + $ref: '#/components/schemas/FileStorageType' + download_url: + title: Download Url + type: string + type: object + required: + - name + - job + - workspace + - artifact_url + - artifact_storage_type + title: PlatformJobResultResponse + PlatformJobStatus: + type: string + enum: + - created + - pending + - active + - cancelled + - cancelling + - error + - completed + - paused + - pausing + - resuming + title: PlatformJobStatus + description: 'Enumeration of possible job statuses. + + + This enum represents the various states a job can be in during its lifecycle, + + from creation to a terminal state.' + PlatformJobStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + steps: + items: + $ref: '#/components/schemas/PlatformJobStepStatusResponse' + type: array + title: Steps + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - steps + - created_at + - updated_at + title: PlatformJobStatusResponse + PlatformJobStepStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + tasks: + items: + $ref: '#/components/schemas/PlatformJobTaskStatusResponse' + type: array + title: Tasks + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - tasks + - created_at + - updated_at + title: PlatformJobStepStatusResponse + PlatformJobTaskStatusResponse: + properties: + id: + type: string + title: Id + name: + type: string + title: Name + status: + $ref: '#/components/schemas/PlatformJobStatus' + status_details: + additionalProperties: true + type: object + title: Status Details + error_details: + title: Error Details + additionalProperties: true + type: object + error_stack: + title: Error Stack + type: string + created_at: + type: string + format: date-time + title: Created At + updated_at: + type: string + format: date-time + title: Updated At + type: object + required: + - id + - name + - status + - status_details + - error_details + - error_stack + - created_at + - updated_at + title: PlatformJobTaskStatusResponse + StringFilter: + additionalProperties: false + properties: + $eq: + description: Filter for results equal to this value. + title: $Eq + type: string + $like: + description: Filter for results matching this pattern. + title: $Like + type: string + $in: + description: Filter for results in this list of values. + title: $In + items: + type: string + type: array + $nin: + description: Filter for results not in this list of values. + title: $Nin + items: + type: string + type: array + title: StringFilter + type: object TargetFilter: additionalProperties: false description: Query filter for ``GET /v2/workspaces/{workspace}/targets``. diff --git a/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py b/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py index 00d27667b4..a69f60c588 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py +++ b/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py @@ -3,29 +3,34 @@ """Audit job — runs garak against a target using inline config + target. -Local-run only for now: ``nemo auditor audit run --spec-file spec.yaml`` shells -out to a pre-installed garak interpreter (default ``~/.auditor/.venv/bin/python``, -overridable via ``NEMO_AUDITOR_GARAK_PYTHON``), then registers the resulting -JSONL / HTML / hitlog reports as job results via -:meth:`~nemo_platform_plugin.job_results.JobResults.save`. - -The plugin uses a single garak invocation across the whole probe spec — there -is no per-probe splitting and no pause/resume scaffolding (those exist in -``services/auditor`` to support remote runs that the platform may interrupt -and resume; local runs run to completion). +``nemo auditor audit run --spec-file spec.yaml`` shells out to a pre-installed +garak interpreter (default ``/app/.garak_venv/bin/python``, overridable via +``NEMO_AUDITOR_GARAK_PYTHON``). + +The probe spec is expanded into individual per-probe YAML configs tracked +through ``todo/``, ``running/``, ``complete/``, and ``failed/`` directories +under persistent storage. SIGTERM is handled by saving partial results so a +resumed invocation picks up from the last completed probe. Completed per-probe +reports are aggregated via ``garak.analyze.aggregate_reports`` and registered +as job results via :meth:`~nemo_platform_plugin.job_results.JobResults.save`. """ from __future__ import annotations -import asyncio import copy +import glob import json import logging import os +import shutil +import signal import subprocess +import sys from pathlib import Path from typing import Annotated, ClassVar, TypeVar, cast +from uuid import uuid4 +import garakapi import yaml from nemo_auditor.entities import AuditConfig, AuditTarget from nemo_platform import AsyncNeMoPlatform, NeMoPlatform @@ -35,16 +40,16 @@ from nemo_platform_plugin.job import NemoJob from nemo_platform_plugin.job_context import JobContext from nemo_platform_plugin.job_results import JobResults -from pydantic import BaseModel, ConfigDict, StringConstraints +from pydantic import BaseModel, ConfigDict, Field, StringConstraints logger = logging.getLogger(__name__) -DEFAULT_GARAK_PYTHON = "~/.auditor/.venv/bin/python" +DEFAULT_GARAK_PYTHON = "/app/.garak_venv/bin/python" GARAK_PYTHON_ENVVAR = "NEMO_AUDITOR_GARAK_PYTHON" # garak writes reports to /garak// -# with filenames driven by reporting.report_prefix. Same layout -# services/auditor relies on. +# with filenames driven by reporting.report_prefix. Same layout for both +# per-probe runs and the aggregated output. _GARAK_OUTPUT_TYPES = ( ("report-jsonl", ".report.jsonl"), ("report-html", ".report.html"), @@ -66,6 +71,10 @@ _LOG_TAIL_BYTES = 4000 +class GarakFailure(Exception): + """Raised when a garak invocation fails unrecoverably.""" + + # Workspace-qualified-or-bare name reference, e.g. "my-cfg" or "prod/my-cfg". NonEmptyStr = Annotated[str, StringConstraints(min_length=1, strip_whitespace=True)] @@ -82,6 +91,8 @@ class AuditInputSpec(BaseModel): config: AuditConfig | NonEmptyStr target: AuditTarget | NonEmptyStr + max_probe_retries: int = Field(default=0, ge=0) + fail_job_on_retries_exhausted: bool = True class AuditSpec(BaseModel): @@ -91,6 +102,8 @@ class AuditSpec(BaseModel): config: AuditConfig target: AuditTarget + max_probe_retries: int = Field(default=0, ge=0) + fail_job_on_retries_exhausted: bool = True def _garak_config_dict(config: AuditConfig) -> dict: @@ -175,6 +188,8 @@ def _rewrite_options_uris( RuntimeError: sentinel present but no SDK was injected, or the SDK lookup itself failed. """ + import asyncio + queue: list = list(options.values()) while queue: node = queue.pop() @@ -225,12 +240,88 @@ def _build_env(persistent_dir: Path) -> dict[str, str]: return env +def _divide_and_write_confs(config_dict: dict, todo_dir: Path) -> None: + """Expand probe_spec into individual per-probe YAML configs in todo_dir.""" + probe_spec_str = config_dict.get("plugins", {}).get("probe_spec", "") + probe_tags_str = config_dict.get("run", {}).get("probe_tags") or "" + + activated, unknown = garakapi.parse_plugin_spec(probe_spec_str, "probes", probe_tags_str) + if unknown: + raise GarakFailure(f"Invalid probe(s): '{', '.join(unknown)}'") + if not activated: + probe_tags_err = f" and probe tags: {probe_tags_str}" if probe_tags_str else "" + raise GarakFailure(f"No probes found for probe spec: {probe_spec_str}{probe_tags_err}") + + for plugin in activated: + probe = plugin.removeprefix("probes.") + per_probe = {**config_dict, "plugins": {**config_dict.get("plugins", {}), "probe_spec": probe}} + (todo_dir / f"{probe}.yaml").write_text(yaml.safe_dump(per_probe)) + + +def _aggregate_reports( + persistent: Path, + report_dir_name: str, + report_prefix: str, + garak_python: str, +) -> bool: + """Aggregate per-probe reports into a single combined report. + + Returns True if at least one completed probe had a report to aggregate, + False if no per-probe JSONL files were found. + """ + jsonl_pattern = str(persistent / "complete" / "*" / "garak" / report_dir_name / f"{report_prefix}.report.jsonl") + jsonls = glob.glob(jsonl_pattern) + if not jsonls: + return False + + agg_dir = persistent / "garak" / report_dir_name + agg_dir.mkdir(parents=True, exist_ok=True) + agg_jsonl = agg_dir / f"{report_prefix}.report.jsonl" + + result = subprocess.run( + [garak_python, "-m", "garak.analyze.aggregate_reports", "-o", str(agg_jsonl)] + jsonls, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise GarakFailure( + f"garak aggregate_reports failed (rc={result.returncode}): {result.stderr[-_LOG_TAIL_BYTES:]}" + ) + + agg_html = agg_dir / f"{report_prefix}.report.html" + with agg_html.open("w") as html_fd: + result = subprocess.run( + [garak_python, "-m", "garak.analyze.report_digest", "-r", str(agg_jsonl)], + stdout=html_fd, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise GarakFailure(f"garak report_digest failed (rc={result.returncode}): {result.stderr[-_LOG_TAIL_BYTES:]}") + + hitlog_pattern = str(persistent / "complete" / "*" / "garak" / report_dir_name / f"{report_prefix}.hitlog.jsonl") + agg_hitlog = agg_dir / f"{report_prefix}.hitlog.jsonl" + agg_hitlog_tmp = agg_dir / f"{report_prefix}.hitlog.jsonl.tmp" + with agg_hitlog_tmp.open("wb") as out_fd: + for hitlog_path in glob.glob(hitlog_pattern): + with open(hitlog_path, "rb") as in_fd: + shutil.copyfileobj(in_fd, out_fd) + if agg_hitlog_tmp.stat().st_size > 0: + shutil.move(str(agg_hitlog_tmp), str(agg_hitlog)) + else: + agg_hitlog_tmp.unlink() + + return True + + class AuditJob(NemoJob): - """Run an audit (single garak invocation) against a configured target.""" + """Run an audit (per-probe garak invocations) against a configured target.""" name: ClassVar[str] = "audit" description: ClassVar[str] = "Run an auditor scan against a configured target." - container: ClassVar[str] = "cpu-tasks" + container: ClassVar[str] = "auditor-tasks" input_spec_schema: ClassVar[type[BaseModel] | None] = AuditInputSpec spec_schema: ClassVar[type[BaseModel] | None] = AuditSpec @@ -277,7 +368,12 @@ async def to_spec( entity_client=client, kind="audit target", ) - return AuditSpec(config=config, target=target) + return AuditSpec( + config=config, + target=target, + max_probe_retries=input_spec.max_probe_retries, + fail_job_on_retries_exhausted=input_spec.fail_job_on_retries_exhausted, + ) @staticmethod def _resolve_entity_client( @@ -301,6 +397,52 @@ def _resolve_entity_client( "or run with a connected platform SDK." ) + @classmethod + async def compile( + cls, + *, + workspace: str, + spec: BaseModel, + entity_client: object, + job_name: str | None, + async_sdk: AsyncNeMoPlatform, + profile: str | None = None, + options: dict | None = None, + ) -> object: + from nemo_platform_plugin.jobs.api_factory import ( + ContainerSpec, + CPUExecutionProviderSpec, + EnvironmentVariable, + PlatformJobSpec, + PlatformJobStep, + ) + from nemo_platform_plugin.jobs.constants import DEFAULT_JOB_STORAGE_PATH, PERSISTENT_JOB_STORAGE_PATH_ENVVAR + from nemo_platform_plugin.jobs.image import get_qualified_image + + return PlatformJobSpec( + steps=[ + PlatformJobStep( + name="audit-job", + executor=CPUExecutionProviderSpec( + profile=profile or "auditor", + provider="cpu", + container=ContainerSpec( + image=get_qualified_image("auditor-tasks"), + entrypoint=["python", "-m"], + command=["nemo_auditor.tasks.audit"], + ), + ), + config=spec.model_dump(mode="json"), + environment=[ + EnvironmentVariable( + name=PERSISTENT_JOB_STORAGE_PATH_ENVVAR, + value=DEFAULT_JOB_STORAGE_PATH, + ), + ], + ) + ], + ) + def run( self, config: dict, @@ -311,22 +453,17 @@ def run( ) -> dict: spec = AuditSpec.model_validate(config) - work_dir = ctx.storage.ephemeral - work_dir.mkdir(parents=True, exist_ok=True) - ctx.storage.persistent.mkdir(parents=True, exist_ok=True) + persistent = ctx.storage.persistent + persistent.mkdir(parents=True, exist_ok=True) + ctx.storage.ephemeral.mkdir(parents=True, exist_ok=True) - # Render config + (optional) target options to disk for garak to pick up. - garak_config_path = work_dir / "garak_config.yaml" - garak_config_path.write_text(yaml.safe_dump(_garak_config_dict(spec.config))) - - target_opts_path: Path | None = None - if spec.target.options: - # Deep-copy before rewriting so the validated AuditTarget on the spec - # stays pristine; only the on-disk JSON we hand to garak is mutated. - rewritten_options = copy.deepcopy(spec.target.options) - _rewrite_options_uris(rewritten_options, sdk, async_sdk) - target_opts_path = work_dir / "target_options.json" - target_opts_path.write_text(json.dumps(rewritten_options)) + todo_dir = persistent / "todo" + running_dir = persistent / "running" + complete_dir = persistent / "complete" + failed_dir = persistent / "failed" + failed_logs_dir = persistent / "failed_probe_logs" + run_log_path = persistent / "run.log" + target_opts_path = persistent / "target_options.json" garak_python = _resolve_garak_python() if not Path(garak_python).exists(): @@ -336,53 +473,177 @@ def run( "to point at an existing one." ) - cmd = [ - garak_python, - "-m", - "garak", - "--config", - str(garak_config_path), - "--target_type", - spec.target.type, - "--target_name", - spec.target.model, - ] - if target_opts_path is not None: - cmd += ["--generator_option_file", str(target_opts_path)] - - env = _build_env(ctx.storage.persistent) - - logger.info("Running garak: %s (cwd=%s)", " ".join(cmd), work_dir) - completed = subprocess.run( - cmd, - env=env, - cwd=work_dir, - capture_output=True, - text=True, - check=False, - ) - - # garak emits to /garak//.* - report_dir = ctx.storage.persistent / "garak" / spec.config.reporting.report_dir - artifacts = _collect_report_artifacts( - report_dir, - spec.config.reporting.report_prefix, - ctx.results, - ) + try: + # Register SIGTERM handler before the probe loop so partial results + # are saved if the job is paused by the scheduler. + def _on_sigterm(signum, frame): + logger.warning("SIGTERM received — saving partial results and exiting.") + try: + _aggregate_reports( + persistent, + spec.config.reporting.report_dir, + spec.config.reporting.report_prefix, + garak_python, + ) + agg_dir = persistent / "garak" / spec.config.reporting.report_dir + _collect_report_artifacts(agg_dir, spec.config.reporting.report_prefix, ctx.results) + except Exception as exc: + logger.error("Partial aggregation failed during SIGTERM: %s", exc) + sys.exit(0) + + try: + signal.signal(signal.SIGTERM, _on_sigterm) + except ValueError: + # Only supported by jobs scheduler. + pass + + if not running_dir.exists(): + # First run: write per-probe configs and resolve target options. + # + # Everything in this if-block must be idempotent until the final + # mkdir of running_dir because it will be re-run if pause happens + # in the middle of initialization. + if spec.target.options: + rewritten_options = copy.deepcopy(spec.target.options) + _rewrite_options_uris(rewritten_options, sdk, async_sdk) + target_opts_path.write_text(json.dumps(rewritten_options)) + + for d in (todo_dir, complete_dir, failed_dir, failed_logs_dir): + d.mkdir(parents=True, exist_ok=True) + + _divide_and_write_confs(_garak_config_dict(spec.config), todo_dir) + + running_dir.mkdir(parents=True, exist_ok=True) + else: + # Resume: re-queue any probes interrupted mid-flight. + for probe_dir in list(running_dir.iterdir()): + if probe_dir.is_dir(): + probe_yaml = todo_dir / f"{probe_dir.name}.yaml" + if not probe_yaml.exists(): + src = probe_dir / "config.yaml" + if src.exists(): + shutil.copy(src, probe_yaml) + shutil.rmtree(probe_dir) + + env = _build_env(persistent) + base_cmd = [ + garak_python, + "-m", + "garak", + "--target_type", + spec.target.type, + "--target_name", + spec.target.model, + ] + if target_opts_path.exists(): + base_cmd += ["--generator_option_file", str(target_opts_path)] + + n_total = ( + sum(1 for _ in todo_dir.glob("*.yaml")) + + len(list(complete_dir.iterdir())) + + len(list(failed_dir.iterdir())) + ) + n_done = len(list(complete_dir.iterdir())) + len(list(failed_dir.iterdir())) + garak_log = persistent / "garak.log" + + for probe_yaml in sorted(todo_dir.glob("*.yaml")): + probe_name = probe_yaml.stem + probe_dir = running_dir / probe_name + report_marker = ( + probe_dir + / "garak" + / spec.config.reporting.report_dir + / f"{spec.config.reporting.report_prefix}.report.html" + ) - status = "completed" if completed.returncode == 0 else "failed" - self.report_progress( - ctx, - work_done=1, - work_total=1, - status=status, - details={"returncode": str(completed.returncode)}, - ) + shutil.rmtree(probe_dir, ignore_errors=True) + probe_dir.mkdir(parents=True, exist_ok=True) + probe_config = probe_dir / "config.yaml" + shutil.copy(probe_yaml, probe_config) + cmd = base_cmd + ["--config", str(probe_config)] + probe_env = {**env, "XDG_DATA_HOME": str(probe_dir)} + + for retry_n in range(spec.max_probe_retries + 1): + self.report_progress( + ctx, + work_done=n_done, + work_total=n_total, + status="running", + details={"probe": probe_name, "retry": str(retry_n)}, + ) + with run_log_path.open("a") as run_log_fd: + completed = subprocess.run( + cmd, + env=probe_env, + stdout=run_log_fd, + stderr=run_log_fd, + check=False, + ) + + if completed.returncode == 0 and report_marker.exists(): + shutil.move(str(probe_dir), str(complete_dir / probe_name)) + logger.info("Probe %s completed (retry %d).", probe_name, retry_n) + break + + logger.error( + "Probe %s retry %d/%d failed (rc=%d).", + probe_name, + retry_n, + spec.max_probe_retries, + completed.returncode, + ) + attempt_log_dir = failed_logs_dir / probe_name + attempt_log_dir.mkdir(parents=True, exist_ok=True) + if garak_log.exists(): + shutil.copy(garak_log, attempt_log_dir / f"{uuid4()}.log") + garak_log.write_bytes(b"") + else: + shutil.move(str(probe_dir), str(failed_dir / probe_name)) + if garak_log.exists(): + garak_log.write_bytes(b"") + if spec.fail_job_on_retries_exhausted: + raise GarakFailure(f"Retries exhausted for probe {probe_name!r}") + logger.error("Retries exhausted for %s — continuing.", probe_name) + + probe_yaml.unlink() + n_done += 1 + self.report_progress(ctx, work_done=n_done, work_total=n_total, status="running") + + n_complete = len(list(complete_dir.iterdir())) + if n_complete == 0: + raise GarakFailure("All probes failed.") + + has_reports = _aggregate_reports( + persistent, + spec.config.reporting.report_dir, + spec.config.reporting.report_prefix, + garak_python, + ) + agg_report_dir = persistent / "garak" / spec.config.reporting.report_dir + artifacts = ( + _collect_report_artifacts(agg_report_dir, spec.config.reporting.report_prefix, ctx.results) + if has_reports + else {} + ) - return { - "status": status, - "returncode": completed.returncode, - "stdout_tail": completed.stdout[-_LOG_TAIL_BYTES:] if completed.stdout else "", - "stderr_tail": completed.stderr[-_LOG_TAIL_BYTES:] if completed.stderr else "", - "results": artifacts, - } + n_failed = len(list(failed_dir.iterdir())) + status = "completed" if n_failed == 0 else "partial" + self.report_progress( + ctx, + work_done=n_done, + work_total=n_total, + status=status, + details={"probes_complete": str(n_complete), "probes_failed": str(n_failed)}, + ) + return { + "status": status, + "probes_total": n_total, + "probes_complete": n_complete, + "probes_failed": n_failed, + "results": artifacts, + } + + except GarakFailure as exc: + logger.error("Audit job failed: %s", exc) + self.report_progress(ctx, work_done=0, work_total=0, status="failed", details={"error": str(exc)}) + return {"status": "failed", "error": str(exc), "results": {}} diff --git a/plugins/nemo-auditor/src/nemo_auditor/service.py b/plugins/nemo-auditor/src/nemo_auditor/service.py index 9e49f49a8d..96b6576c63 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/service.py +++ b/plugins/nemo-auditor/src/nemo_auditor/service.py @@ -8,7 +8,10 @@ from typing import ClassVar from fastapi import APIRouter +from nemo_auditor.authz import scope +from nemo_auditor.jobs.audit import AuditJob from nemo_platform_plugin.authz import CallerKind, path_rule +from nemo_platform_plugin.jobs.routes import add_job_routes from nemo_platform_plugin.service import NemoService, RouterSpec @@ -54,4 +57,10 @@ async def healthz() -> dict[str, object]: description="Audit target CRUD.", prefix=crud_prefix, ), + RouterSpec( + add_job_routes(AuditJob, authz=scope.child("audit")), + tag="Auditor Jobs", + description="Audit job submission and retrieval.", + prefix=crud_prefix, + ), ] diff --git a/plugins/nemo-auditor/src/nemo_auditor/tasks/audit.py b/plugins/nemo-auditor/src/nemo_auditor/tasks/audit.py new file mode 100644 index 0000000000..1737aa915d --- /dev/null +++ b/plugins/nemo-auditor/src/nemo_auditor/tasks/audit.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Container entrypoint for the audit job. + +Invoked as ``python -m nemo_auditor.tasks.audit`` inside the nmp-cpu-tasks container. +Builds the task SDK, then dispatches to :class:`~nemo_auditor.jobs.audit.AuditJob`. +The SIGTERM handler installed here is overridden by the one in ``AuditJob.run()`` +before the probe loop begins, so partial-result aggregation is handled by the job. +""" + +from __future__ import annotations + +import logging +import signal +import sys +from types import FrameType + +from nemo_auditor.jobs.audit import AuditJob +from nemo_platform_plugin.sdk_provider import get_async_task_sdk, get_task_sdk +from nemo_platform_plugin.tasks.dispatcher import run_task + +logger = logging.getLogger(__name__) + + +def _shutdown_handler(signum: int, frame: FrameType | None) -> None: + logger.warning("Received shutdown signal (%d). Exiting.", signum) + raise SystemExit(0) + + +def main() -> int: + signal.signal(signal.SIGTERM, _shutdown_handler) + try: + sdk = get_task_sdk("auditor") + async_sdk = get_async_task_sdk("auditor") + except Exception: + logger.exception("Failed to build task SDK for auditor") + return 2 + return run_task(AuditJob, sdk=sdk, async_sdk=async_sdk) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/nemo-auditor/tests/test_audit_job.py b/plugins/nemo-auditor/tests/test_audit_job.py index 86ff33d739..f3b2b88fcd 100644 --- a/plugins/nemo-auditor/tests/test_audit_job.py +++ b/plugins/nemo-auditor/tests/test_audit_job.py @@ -8,6 +8,7 @@ import asyncio import json +import signal import subprocess from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch @@ -28,7 +29,10 @@ AuditInputSpec, AuditJob, AuditSpec, + GarakFailure, + _aggregate_reports, _collect_report_artifacts, + _divide_and_write_confs, _garak_config_dict, _rewrite_options_uris, ) @@ -36,6 +40,10 @@ from nemo_platform_plugin.job_context import JobContext, StoragePaths from nemo_platform_plugin.job_results import LocalJobResults +# The probe name returned by parse_plugin_spec for "encoding.InjectAscii85". +_PROBE_NAME = "encoding.InjectAscii85" +_PROBE_FULL = f"probes.{_PROBE_NAME}" + def _make_ctx(tmp_path: Path) -> JobContext: ephemeral = tmp_path / "ephemeral" @@ -79,18 +87,57 @@ def _make_target(**overrides) -> AuditTargetEntity: def _make_spec_dict(**overrides) -> dict: cfg = overrides.pop("config", _make_config()) tgt = overrides.pop("target", _make_target()) - return { + d = { "config": cfg.model_dump(mode="json"), "target": tgt.model_dump(mode="json"), } + d.update(overrides) + return d -def _plant_reports(persistent: Path, prefix: str, kinds: tuple[str, ...]) -> None: - """Plant fake garak report files where ``run`` will look for them.""" - report_dir = persistent / "garak" / "garak_runs" - report_dir.mkdir(parents=True, exist_ok=True) +def _plant_reports(persistent: Path, prefix: str, kinds: tuple[str, ...], report_dir: str = "garak_runs") -> None: + """Plant fake aggregated garak report files where ``_collect_report_artifacts`` will look.""" + d = persistent / "garak" / report_dir + d.mkdir(parents=True, exist_ok=True) for kind in kinds: - (report_dir / f"{prefix}{kind}").write_text(f"fake-{kind}") + (d / f"{prefix}{kind}").write_text(f"fake-{kind}") + + +def _plant_probe_success( + persistent: Path, + probe_name: str, + report_prefix: str = "run1", + report_dir: str = "garak_runs", +) -> None: + """Plant the per-probe HTML success marker that AuditJob uses to detect probe completion.""" + marker = persistent / "running" / probe_name / "garak" / report_dir / f"{report_prefix}.report.html" + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok") + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_garak_python(tmp_path: Path, monkeypatch) -> Path: + """Create an empty file standing in for the garak interpreter and point the + job at it via the env var. AuditJob only checks existence, never executes + it (subprocess.run is patched separately).""" + interp = tmp_path / "garak-python" + interp.touch() + monkeypatch.setenv("NEMO_AUDITOR_GARAK_PYTHON", str(interp)) + return interp + + +@pytest.fixture +def fake_parse_plugin_spec(): + """Patch garakapi.parse_plugin_spec to return a single known probe without + requiring a real garak plugin cache.""" + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = ([_PROBE_FULL], []) + yield mock # --------------------------------------------------------------------------- @@ -118,30 +165,156 @@ def test_preserves_nested_values(self) -> None: # --------------------------------------------------------------------------- -# Subprocess invocation +# _divide_and_write_confs # --------------------------------------------------------------------------- -@pytest.fixture -def fake_garak_python(tmp_path: Path, monkeypatch) -> Path: - """Create an empty file standing in for the garak interpreter and point the - job at it via the env var. AuditJob only checks existence, never executes - it (subprocess.run is patched separately).""" - interp = tmp_path / "garak-python" - interp.touch() - monkeypatch.setenv("NEMO_AUDITOR_GARAK_PYTHON", str(interp)) - return interp +class TestDivideAndWriteConfs: + def test_writes_one_yaml_per_probe(self, tmp_path: Path) -> None: + todo = tmp_path / "todo" + todo.mkdir() + config_dict = { + "plugins": {"probe_spec": "encoding", "detector_spec": "auto"}, + "run": {"probe_tags": None}, + "system": {}, + "reporting": {}, + } + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = (["probes.encoding.InjectAscii85", "probes.encoding.InjectBase16"], []) + _divide_and_write_confs(config_dict, todo) + + yamls = sorted(todo.glob("*.yaml")) + assert len(yamls) == 2 + assert {y.stem for y in yamls} == {"encoding.InjectAscii85", "encoding.InjectBase16"} + + def test_per_probe_yaml_has_single_probe_spec(self, tmp_path: Path) -> None: + todo = tmp_path / "todo" + todo.mkdir() + config_dict = { + "plugins": {"probe_spec": "encoding.InjectAscii85", "detector_spec": "auto"}, + "run": {"probe_tags": None}, + } + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = ([_PROBE_FULL], []) + _divide_and_write_confs(config_dict, todo) + + loaded = yaml.safe_load((todo / f"{_PROBE_NAME}.yaml").read_text()) + assert loaded["plugins"]["probe_spec"] == _PROBE_NAME + # Other plugin keys are preserved. + assert loaded["plugins"]["detector_spec"] == "auto" + + def test_raises_on_empty_activated_list(self, tmp_path: Path) -> None: + todo = tmp_path / "todo" + todo.mkdir() + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = ([], []) + with pytest.raises(GarakFailure, match="No probes found"): + _divide_and_write_confs({"plugins": {"probe_spec": "nonexistent"}, "run": {}}, todo) + + def test_raises_on_unknown_probes(self, tmp_path: Path) -> None: + todo = tmp_path / "todo" + todo.mkdir() + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = ([], ["bad.probe"]) + with pytest.raises(GarakFailure, match="Invalid probe"): + _divide_and_write_confs({"plugins": {"probe_spec": "bad.probe"}, "run": {}}, todo) + + def test_passes_probe_tags_to_parse_plugin_spec(self, tmp_path: Path) -> None: + todo = tmp_path / "todo" + todo.mkdir() + config_dict = {"plugins": {"probe_spec": "all"}, "run": {"probe_tags": "owasp:llm06"}} + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = ([_PROBE_FULL], []) + _divide_and_write_confs(config_dict, todo) + + mock.assert_called_once_with("all", "probes", "owasp:llm06") + + def test_normalises_none_probe_tags_to_empty_string(self, tmp_path: Path) -> None: + todo = tmp_path / "todo" + todo.mkdir() + config_dict = {"plugins": {"probe_spec": "encoding"}, "run": {"probe_tags": None}} + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock: + mock.return_value = ([_PROBE_FULL], []) + _divide_and_write_confs(config_dict, todo) + + mock.assert_called_once_with("encoding", "probes", "") + + +# --------------------------------------------------------------------------- +# _aggregate_reports +# --------------------------------------------------------------------------- + + +class TestAggregateReports: + def test_returns_false_when_no_completed_jsonls(self, tmp_path: Path, fake_garak_python: Path) -> None: + (tmp_path / "complete").mkdir() + result = _aggregate_reports(tmp_path, "garak_runs", "run1", str(fake_garak_python)) + assert result is False + + def test_calls_aggregate_reports_and_report_digest(self, tmp_path: Path, fake_garak_python: Path) -> None: + # Plant a per-probe JSONL so the function has something to aggregate. + jsonl = tmp_path / "complete" / _PROBE_NAME / "garak" / "garak_runs" / "run1.report.jsonl" + jsonl.parent.mkdir(parents=True) + jsonl.write_text("{}") + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) + result = _aggregate_reports(tmp_path, "garak_runs", "run1", str(fake_garak_python)) + + assert result is True + assert mock_run.call_count == 2 + cmds = [call.args[0] for call in mock_run.call_args_list] + assert any("aggregate_reports" in " ".join(c) for c in cmds) + assert any("report_digest" in " ".join(c) for c in cmds) + + def test_raises_garak_failure_when_aggregate_subprocess_fails( + self, tmp_path: Path, fake_garak_python: Path + ) -> None: + jsonl = tmp_path / "complete" / _PROBE_NAME / "garak" / "garak_runs" / "run1.report.jsonl" + jsonl.parent.mkdir(parents=True) + jsonl.write_text("{}") + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1, stderr="boom") + with pytest.raises(GarakFailure, match="aggregate_reports failed"): + _aggregate_reports(tmp_path, "garak_runs", "run1", str(fake_garak_python)) + + def test_concatenates_hitlogs(self, tmp_path: Path, fake_garak_python: Path) -> None: + for probe in ("probeA", "probeB"): + d = tmp_path / "complete" / probe / "garak" / "garak_runs" + d.mkdir(parents=True) + (d / "run1.report.jsonl").write_text("{}") + (d / "run1.hitlog.jsonl").write_bytes(b"hit-" + probe.encode()) + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) + _aggregate_reports(tmp_path, "garak_runs", "run1", str(fake_garak_python)) + + hitlog = tmp_path / "garak" / "garak_runs" / "run1.hitlog.jsonl" + assert hitlog.exists() + contents = hitlog.read_bytes() + assert b"hit-probeA" in contents + assert b"hit-probeB" in contents + + +# --------------------------------------------------------------------------- +# Subprocess invocation +# --------------------------------------------------------------------------- class TestAuditJobRun: - def test_invokes_garak_with_expected_argv_and_env(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_invokes_garak_with_expected_argv_and_env( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: ctx = _make_ctx(tmp_path) spec = _make_spec_dict() with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + # returncode=0 but no HTML planted → probe "fails", GarakFailure caught internally. + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) AuditJob().run(spec, ctx=ctx) + # One call: the probe invocation (aggregation not reached after failure). assert mock_run.call_count == 1 call_args = mock_run.call_args argv = call_args.args[0] @@ -150,48 +323,50 @@ def test_invokes_garak_with_expected_argv_and_env(self, tmp_path: Path, fake_gar assert argv[1:3] == ["-m", "garak"] assert "--config" in argv cfg_idx = argv.index("--config") - assert argv[cfg_idx + 1].endswith("garak_config.yaml") + # Per-probe config is copied into running//config.yaml. + assert argv[cfg_idx + 1].endswith("config.yaml") assert ["--target_type", "test"] == argv[argv.index("--target_type") : argv.index("--target_type") + 2] assert ["--target_name", "test.Blank"] == argv[argv.index("--target_name") : argv.index("--target_name") + 2] # No options on the default target → no --generator_option_file. assert "--generator_option_file" not in argv env = call_args.kwargs["env"] - # Either the test process inherited the var, or the job stubbed it - # to "NOT_SET". Either way the key must be present and non-empty so - # garak doesn't reject startup. for key in ("NIM_API_KEY", "OPENAI_API_KEY", "REST_API_KEY", "OPENAICOMPATIBLE_API_KEY"): assert env[key] - assert env["XDG_DATA_HOME"] == str(ctx.storage.persistent) + # XDG_DATA_HOME is the per-probe running dir, not the persistent root. + assert env["XDG_DATA_HOME"] == str(ctx.storage.persistent / "running" / _PROBE_NAME) assert env["GARAK_LOG_FILE"].endswith("garak.log") - # cwd is the ephemeral working dir. - assert call_args.kwargs["cwd"] == ctx.storage.ephemeral - - def test_yaml_config_only_has_garak_sections(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_yaml_config_only_has_garak_sections( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: ctx = _make_ctx(tmp_path) spec = _make_spec_dict(config=_make_config(description="will-be-stripped")) with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) AuditJob().run(spec, ctx=ctx) - garak_config_path = ctx.storage.ephemeral / "garak_config.yaml" - assert garak_config_path.exists() - loaded = yaml.safe_load(garak_config_path.read_text()) + # The per-probe config is moved to failed/ after the probe fails (no HTML). + probe_config = ctx.storage.persistent / "failed" / _PROBE_NAME / "config.yaml" + assert probe_config.exists() + loaded = yaml.safe_load(probe_config.read_text()) assert set(loaded.keys()) == {"system", "run", "plugins", "reporting"} assert "description" not in loaded assert "name" not in loaded - def test_target_options_written_when_present_and_flag_added(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_target_options_written_when_present_and_flag_added( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: ctx = _make_ctx(tmp_path) spec = _make_spec_dict(target=_make_target(options={"endpoint": "https://example.invalid", "key_env": "X"})) with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) AuditJob().run(spec, ctx=ctx) - opts_path = ctx.storage.ephemeral / "target_options.json" + # Target options are written to persistent storage (survives pause/resume). + opts_path = ctx.storage.persistent / "target_options.json" assert opts_path.exists() assert json.loads(opts_path.read_text()) == {"endpoint": "https://example.invalid", "key_env": "X"} @@ -205,77 +380,314 @@ def test_missing_garak_interpreter_raises_clear_error(self, tmp_path: Path, monk with pytest.raises(FileNotFoundError, match="garak interpreter not found"): AuditJob().run(_make_spec_dict(), ctx=ctx) - def test_completed_run_collects_all_three_artifacts(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_completed_run_collects_all_three_artifacts( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: ctx = _make_ctx(tmp_path) spec = _make_spec_dict() - def fake_run(*args, **kwargs): - _plant_reports( - ctx.storage.persistent, - "run1", - (".report.jsonl", ".report.html", ".hitlog.jsonl"), - ) - return subprocess.CompletedProcess(args=[], returncode=0, stdout="ok", stderr="") - - with patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run): + def fake_run(cmd, **kwargs): + # Plant the per-probe HTML success marker so the probe is treated as complete. + xdg = kwargs["env"]["XDG_DATA_HOME"] + marker = Path(xdg) / "garak" / "garak_runs" / "run1.report.html" + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok") + return subprocess.CompletedProcess(args=[], returncode=0) + + with ( + patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run), + patch("nemo_auditor.jobs.audit._aggregate_reports", return_value=True), + ): + _plant_reports(ctx.storage.persistent, "run1", (".report.jsonl", ".report.html", ".hitlog.jsonl")) result = AuditJob().run(spec, ctx=ctx) assert result["status"] == "completed" - assert result["returncode"] == 0 + assert result["probes_complete"] == 1 assert set(result["results"].keys()) == {"report-jsonl", "report-html", "report-hitlog-jsonl"} for ref in result["results"].values(): assert ref["artifact_url"].startswith("file://") - # Local sink copies to /results/. assert Path(ref["artifact_url"][len("file://") :]).exists() - def test_failed_run_returns_failed_status_and_collects_partial_artifacts( - self, tmp_path: Path, fake_garak_python: Path + def test_failed_run_returns_failed_status( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec ) -> None: ctx = _make_ctx(tmp_path) spec = _make_spec_dict() - def fake_run(*args, **kwargs): - # Garak got far enough to emit the jsonl but crashed before the html. - _plant_reports(ctx.storage.persistent, "run1", (".report.jsonl",)) - return subprocess.CompletedProcess(args=[], returncode=2, stdout="", stderr="garak exploded\n") - - with patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run): + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=2) result = AuditJob().run(spec, ctx=ctx) assert result["status"] == "failed" - assert result["returncode"] == 2 - assert "garak exploded" in result["stderr_tail"] - assert set(result["results"].keys()) == {"report-jsonl"} + assert "results" in result - def test_failed_run_with_no_artifacts_still_returns_envelope(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_failed_run_with_no_artifacts_still_returns_envelope( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: ctx = _make_ctx(tmp_path) spec = _make_spec_dict() with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1, stdout="", stderr="early death") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) result = AuditJob().run(spec, ctx=ctx) assert result["status"] == "failed" - assert result["returncode"] == 1 assert result["results"] == {} - def test_uses_custom_report_prefix_and_dir(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_uses_custom_report_prefix_and_dir( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: ctx = _make_ctx(tmp_path) cfg = _make_config(reporting=AuditReportData(report_prefix="custom-prefix", report_dir="custom_dir")) spec = _make_spec_dict(config=cfg) - def fake_run(*args, **kwargs): - d = ctx.storage.persistent / "garak" / "custom_dir" - d.mkdir(parents=True, exist_ok=True) - (d / "custom-prefix.report.jsonl").write_text("hi") - return subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") - - with patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run): + def fake_run(cmd, **kwargs): + xdg = kwargs["env"]["XDG_DATA_HOME"] + marker = Path(xdg) / "garak" / "custom_dir" / "custom-prefix.report.html" + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok") + return subprocess.CompletedProcess(args=[], returncode=0) + + with ( + patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run), + patch("nemo_auditor.jobs.audit._aggregate_reports", return_value=True), + ): + _plant_reports(ctx.storage.persistent, "custom-prefix", (".report.jsonl",), report_dir="custom_dir") result = AuditJob().run(spec, ctx=ctx) assert result["status"] == "completed" assert "report-jsonl" in result["results"] + # ----------------------------------------------------------------------- + # Scratch space and per-probe directory management + # ----------------------------------------------------------------------- + + def test_first_run_creates_scratch_directories( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + ctx = _make_ctx(tmp_path) + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + AuditJob().run(_make_spec_dict(), ctx=ctx) + + persistent = ctx.storage.persistent + for d in ("todo", "running", "complete", "failed", "failed_probe_logs"): + assert (persistent / d).exists(), f"Expected {d}/ to be created" + + def test_successful_probe_lands_in_complete( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + ctx = _make_ctx(tmp_path) + + def fake_run(cmd, **kwargs): + xdg = kwargs["env"]["XDG_DATA_HOME"] + marker = Path(xdg) / "garak" / "garak_runs" / "run1.report.html" + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok") + return subprocess.CompletedProcess(args=[], returncode=0) + + with ( + patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run), + patch("nemo_auditor.jobs.audit._aggregate_reports", return_value=False), + ): + AuditJob().run(_make_spec_dict(), ctx=ctx) + + assert (ctx.storage.persistent / "complete" / _PROBE_NAME).is_dir() + # todo YAML removed after probe completes. + assert not list((ctx.storage.persistent / "todo").glob("*.yaml")) + + def test_failed_probe_lands_in_failed( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + ctx = _make_ctx(tmp_path) + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + AuditJob().run(_make_spec_dict(), ctx=ctx) + + assert (ctx.storage.persistent / "failed" / _PROBE_NAME).is_dir() + + def test_probe_succeeds_on_retry(self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec) -> None: + ctx = _make_ctx(tmp_path) + call_count = 0 + + def fake_run(cmd, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 2: + # Second attempt plants the success marker. + xdg = kwargs["env"]["XDG_DATA_HOME"] + marker = Path(xdg) / "garak" / "garak_runs" / "run1.report.html" + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok") + return subprocess.CompletedProcess(args=[], returncode=0) + return subprocess.CompletedProcess(args=[], returncode=1) + + spec = _make_spec_dict(max_probe_retries=1) + with ( + patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run), + patch("nemo_auditor.jobs.audit._aggregate_reports", return_value=False), + ): + result = AuditJob().run(spec, ctx=ctx) + + assert result["probes_complete"] == 1 + assert (ctx.storage.persistent / "complete" / _PROBE_NAME).is_dir() + # The failed-probe log directory was created for the first (failed) attempt. + assert (ctx.storage.persistent / "failed_probe_logs" / _PROBE_NAME).is_dir() + + def test_retries_exhausted_fail_job_returns_failed( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + ctx = _make_ctx(tmp_path) + spec = _make_spec_dict(max_probe_retries=1, fail_job_on_retries_exhausted=True) + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + result = AuditJob().run(spec, ctx=ctx) + + assert result["status"] == "failed" + assert _PROBE_NAME in result["error"] + assert mock_run.call_count == 2 # 1 attempt + 1 retry + + def test_retries_exhausted_continue_gives_partial_result( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + """With fail_job_on_retries_exhausted=False and two probes, exhausted retries + on the first probe are skipped and the second probe still runs.""" + ctx = _make_ctx(tmp_path) + + with patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock_spec: + mock_spec.return_value = ( + ["probes.encoding.InjectAscii85", "probes.encoding.InjectBase16"], + [], + ) + call_order: list[str] = [] + + def fake_run(cmd, **kwargs): + xdg = kwargs["env"]["XDG_DATA_HOME"] + probe_dir = Path(xdg) + if "InjectBase16" in str(probe_dir): + # Second probe succeeds. + marker = probe_dir / "garak" / "garak_runs" / "run1.report.html" + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text("ok") + call_order.append("success") + return subprocess.CompletedProcess(args=[], returncode=0) + call_order.append("fail") + return subprocess.CompletedProcess(args=[], returncode=1) + + spec = _make_spec_dict(fail_job_on_retries_exhausted=False) + with ( + patch("nemo_auditor.jobs.audit.subprocess.run", side_effect=fake_run), + patch("nemo_auditor.jobs.audit._aggregate_reports", return_value=False), + ): + result = AuditJob().run(spec, ctx=ctx) + + assert result["status"] == "partial" + assert result["probes_complete"] == 1 + assert result["probes_failed"] == 1 + + def test_all_probes_fail_returns_failed( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + ctx = _make_ctx(tmp_path) + spec = _make_spec_dict(fail_job_on_retries_exhausted=False) + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + result = AuditJob().run(spec, ctx=ctx) + + assert result["status"] == "failed" + assert "All probes failed" in result["error"] + + # ----------------------------------------------------------------------- + # Pause / resume + # ----------------------------------------------------------------------- + + def test_resume_skips_initialization(self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec) -> None: + """When todo/ already exists, _divide_and_write_confs must not be called again.""" + ctx = _make_ctx(tmp_path) + persistent = ctx.storage.persistent + + # Simulate a prior run that left one probe in todo/. + (persistent / "todo").mkdir(parents=True) + (persistent / "running").mkdir() + (persistent / "complete").mkdir() + (persistent / "failed").mkdir() + (persistent / "failed_probe_logs").mkdir() + cfg_dict = _garak_config_dict(_make_config()) + (persistent / "todo" / f"{_PROBE_NAME}.yaml").write_text(yaml.safe_dump(cfg_dict)) + + with ( + patch("nemo_auditor.jobs.audit.garakapi.parse_plugin_spec") as mock_spec, + patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run, + ): + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + AuditJob().run(_make_spec_dict(), ctx=ctx) + # parse_plugin_spec must not be called — initialization was skipped. + mock_spec.assert_not_called() + + def test_resume_requeues_interrupted_probe( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + """A probe directory left in running/ is re-queued to todo/ on resume.""" + ctx = _make_ctx(tmp_path) + persistent = ctx.storage.persistent + + (persistent / "todo").mkdir(parents=True) + (persistent / "running").mkdir() + (persistent / "complete").mkdir() + (persistent / "failed").mkdir() + (persistent / "failed_probe_logs").mkdir() + # Simulate an interrupted probe: running/ has the probe dir with a config. + probe_dir = persistent / "running" / _PROBE_NAME + probe_dir.mkdir() + cfg_dict = _garak_config_dict(_make_config()) + (probe_dir / "config.yaml").write_text(yaml.safe_dump(cfg_dict)) + + with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + AuditJob().run(_make_spec_dict(), ctx=ctx) + + # The probe was re-queued and executed (appears in failed/ because returncode=1). + assert (persistent / "failed" / _PROBE_NAME).is_dir() + # running/ was cleaned up after re-queue and execution. + assert not (persistent / "running" / _PROBE_NAME).exists() + + # ----------------------------------------------------------------------- + # SIGTERM handler + # ----------------------------------------------------------------------- + + def test_sigterm_handler_aggregates_partial_results( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: + """Directly invoke the SIGTERM closure and verify it calls aggregation.""" + ctx = _make_ctx(tmp_path) + captured_handler: list = [] + + original_signal = signal.signal + + def capture_signal(signum, handler): + if signum == signal.SIGTERM: + captured_handler.append(handler) + return original_signal(signum, handler) + + with ( + patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run, + patch("nemo_auditor.jobs.audit.signal.signal", side_effect=capture_signal), + patch("nemo_auditor.jobs.audit._aggregate_reports", return_value=False) as mock_agg, + patch("nemo_auditor.jobs.audit.sys.exit") as mock_exit, + ): + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1) + AuditJob().run(_make_spec_dict(), ctx=ctx) + + assert captured_handler, "SIGTERM handler was not registered" + # Invoke the handler inside the patch context so sys.exit remains mocked. + captured_handler[0](signal.SIGTERM, None) + mock_agg.assert_called() + mock_exit.assert_called_with(0) + # --------------------------------------------------------------------------- # Schema-level tests @@ -299,6 +711,15 @@ def test_requires_config_and_target(self) -> None: with pytest.raises(ValueError): AuditSpec.model_validate({"config": _make_config().model_dump(mode="json")}) + def test_default_task_options(self) -> None: + spec = AuditSpec.model_validate(_make_spec_dict()) + assert spec.max_probe_retries == 0 + assert spec.fail_job_on_retries_exhausted is True + + def test_rejects_negative_max_probe_retries(self) -> None: + with pytest.raises(ValueError): + AuditSpec.model_validate({**_make_spec_dict(), "max_probe_retries": -1}) + # --------------------------------------------------------------------------- # Optional smoke: real ~/.auditor venv reachable @@ -518,7 +939,7 @@ def test_wraps_sdk_lookup_failure_in_runtimeerror(self) -> None: class TestAuditJobIGW: def test_run_writes_options_with_resolved_uri_and_drops_sentinel( - self, tmp_path: Path, fake_garak_python: Path + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec ) -> None: ctx = _make_ctx(tmp_path) target = _make_target( @@ -535,10 +956,11 @@ def test_run_writes_options_with_resolved_uri_and_drops_sentinel( sdk = _mock_sdk("https://igw-resolved.example/v1") with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) AuditJob().run(spec, ctx=ctx, sdk=sdk) - opts_path = ctx.storage.ephemeral / "target_options.json" + # Options are written to persistent storage (not ephemeral). + opts_path = ctx.storage.persistent / "target_options.json" assert opts_path.exists() on_disk = json.loads(opts_path.read_text()) assert on_disk == { @@ -550,19 +972,23 @@ def test_run_writes_options_with_resolved_uri_and_drops_sentinel( # And the original validated spec is untouched. assert "nmp_uri_spec" in target.options["nim"] - def test_run_without_sdk_when_no_sentinel_works(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_run_without_sdk_when_no_sentinel_works( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: """sdk=None is fine when options carry no nmp_uri_spec.""" ctx = _make_ctx(tmp_path) spec = _make_spec_dict(target=_make_target(options={"nim": {"max_tokens": 100}})) with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) AuditJob().run(spec, ctx=ctx) # no sdk kwarg - on_disk = json.loads((ctx.storage.ephemeral / "target_options.json").read_text()) + on_disk = json.loads((ctx.storage.persistent / "target_options.json").read_text()) assert on_disk == {"nim": {"max_tokens": 100}} - def test_run_resolves_nmp_uri_spec_via_async_sdk(self, tmp_path: Path, fake_garak_python: Path) -> None: + def test_run_resolves_nmp_uri_spec_via_async_sdk( + self, tmp_path: Path, fake_garak_python: Path, fake_parse_plugin_spec + ) -> None: """async_sdk path: nmp_uri_spec is rewritten when sdk=None but async_sdk is provided.""" ctx = _make_ctx(tmp_path) target = _make_target( @@ -582,10 +1008,10 @@ def test_run_resolves_nmp_uri_spec_via_async_sdk(self, tmp_path: Path, fake_gara async_sdk.models.get_provider_route_openai_url.return_value = "https://igw-async.example/v1" with patch("nemo_auditor.jobs.audit.subprocess.run") as mock_run: - mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr="") + mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=0) AuditJob().run(spec, ctx=ctx, sdk=None, async_sdk=async_sdk) - opts_path = ctx.storage.ephemeral / "target_options.json" + opts_path = ctx.storage.persistent / "target_options.json" assert opts_path.exists() on_disk = json.loads(opts_path.read_text()) assert on_disk == {"nim": {"max_tokens": 32, "uri": "https://igw-async.example/v1"}} @@ -653,6 +1079,18 @@ def test_requires_config_and_target(self) -> None: with pytest.raises(ValueError): AuditInputSpec.model_validate({"config": "my-cfg"}) + def test_default_task_options(self) -> None: + spec = AuditInputSpec.model_validate({"config": "my-cfg", "target": "my-tgt"}) + assert spec.max_probe_retries == 0 + assert spec.fail_job_on_retries_exhausted is True + + def test_custom_task_options(self) -> None: + spec = AuditInputSpec.model_validate( + {"config": "my-cfg", "target": "my-tgt", "max_probe_retries": 3, "fail_job_on_retries_exhausted": False} + ) + assert spec.max_probe_retries == 3 + assert spec.fail_job_on_retries_exhausted is False + # --------------------------------------------------------------------------- # AuditJob.to_spec — name resolution via entity_client @@ -832,3 +1270,22 @@ def test_no_client_required_when_both_inline(self) -> None: ) ) assert isinstance(out, AuditSpec) + + def test_task_options_pass_through_to_spec(self) -> None: + """max_probe_retries and fail_job_on_retries_exhausted are forwarded to AuditSpec.""" + out = asyncio.run( + AuditJob.to_spec( + AuditInputSpec( + config=_make_config(), + target=_make_target(), + max_probe_retries=3, + fail_job_on_retries_exhausted=False, + ), + workspace="default", + entity_client=None, + async_sdk=None, + is_local=True, + ) + ) + assert out.max_probe_retries == 3 + assert out.fail_job_on_retries_exhausted is False diff --git a/plugins/nemo-auditor/tests/test_service.py b/plugins/nemo-auditor/tests/test_service.py new file mode 100644 index 0000000000..501a06d7a4 --- /dev/null +++ b/plugins/nemo-auditor/tests/test_service.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the auditor plugin service wiring.""" + +from __future__ import annotations + +from fastapi.routing import APIRoute +from nemo_auditor.jobs.audit import AuditJob +from nemo_auditor.service import AuditorPluginService +from nemo_platform_plugin.scheduler import submit_path_for + + +def _mounted_post_paths() -> set[str]: + service = AuditorPluginService() + paths: set[str] = set() + for spec in service.get_routers(): + for route in spec.router.routes: + if isinstance(route, APIRoute) and "POST" in route.methods: + paths.add(f"/apis/auditor{spec.prefix}{route.path}") + return paths + + +def test_audit_job_submit_route_is_mounted() -> None: + assert submit_path_for(AuditJob, workspace="{workspace}") in _mounted_post_paths() diff --git a/services/core/jobs/tests/test_config.py b/services/core/jobs/tests/test_config.py index 2d9e5a0d21..d91efdc1bb 100644 --- a/services/core/jobs/tests/test_config.py +++ b/services/core/jobs/tests/test_config.py @@ -370,8 +370,12 @@ def test_merged_profiles(): ), ), ), - *get_default_executor_profiles_for_runtime(runtime=Runtime.DOCKER, defaults=DefaultExecutionProfileConfig())[ - 2: + *[ + p + for p in get_default_executor_profiles_for_runtime( + runtime=Runtime.DOCKER, defaults=DefaultExecutionProfileConfig() + ) + if p.provider == "subprocess" ], ] diff --git a/third_party/osv-licenses.json b/third_party/osv-licenses.json index d5ad053797..445952faf3 100644 --- a/third_party/osv-licenses.json +++ b/third_party/osv-licenses.json @@ -148,7 +148,7 @@ { "package": { "name": "anthropic", - "version": "0.101.0", + "version": "0.116.0", "ecosystem": "PyPI" }, "licenses": [ @@ -567,239 +567,9 @@ { "package": { "name": "cryptography", - "version": "46.0.7", + "version": "48.0.1", "ecosystem": "PyPI" }, - "vulnerabilities": [ - { - "modified": "2026-06-16T19:59:26Z", - "published": "2026-06-15T20:12:27Z", - "schema_version": "1.7.5", - "id": "GHSA-537c-gmf6-5ccf", - "related": [ - "CGA-48f2-3qh9-f9x4" - ], - "summary": "Vulnerable OpenSSL included in cryptography wheels", - "details": "pyca/cryptography's wheels include a statically linked copy of OpenSSL. The versions of OpenSSL included in wheels prior to cryptograph 48.01 are vulnerable to a security issue. More details about the vulnerability itself can be found in https://openssl-library.org/news/secadv/20260609.txt.\n\nIf you are building cryptography source (\"sdist\") then you are responsible for upgrading your copy of OpenSSL. Only users installing from wheels built by the cryptography project (i.e., those distributed on PyPI) need to update their cryptography versions.", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" - } - ], - "affected": [ - { - "package": { - "ecosystem": "PyPI", - "name": "cryptography", - "purl": "pkg:pypi/cryptography" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0.5.0" - }, - { - "fixed": "48.0.1" - } - ] - } - ], - "versions": [ - "0.5", - "0.5.1", - "0.5.2", - "0.5.3", - "0.5.4", - "0.6", - "0.6.1", - "0.7", - "0.7.1", - "0.7.2", - "0.8", - "0.8.1", - "0.8.2", - "0.9", - "0.9.1", - "0.9.2", - "0.9.3", - "1.0", - "1.0.1", - "1.0.2", - "1.1", - "1.1.1", - "1.1.2", - "1.2", - "1.2.1", - "1.2.2", - "1.2.3", - "1.3", - "1.3.1", - "1.3.2", - "1.3.3", - "1.3.4", - "1.4", - "1.5", - "1.5.1", - "1.5.2", - "1.5.3", - "1.6", - "1.7", - "1.7.1", - "1.7.2", - "1.8", - "1.8.1", - "1.8.2", - "1.9", - "2.0", - "2.0.1", - "2.0.2", - "2.0.3", - "2.1", - "2.1.1", - "2.1.2", - "2.1.3", - "2.1.4", - "2.2", - "2.2.1", - "2.2.2", - "2.3", - "2.3.1", - "2.4", - "2.4.1", - "2.4.2", - "2.5", - "2.6", - "2.6.1", - "2.7", - "2.8", - "2.9", - "2.9.1", - "2.9.2", - "3.0", - "3.1", - "3.1.1", - "3.2", - "3.2.1", - "3.3", - "3.3.1", - "3.3.2", - "3.4", - "3.4.1", - "3.4.2", - "3.4.3", - "3.4.4", - "3.4.5", - "3.4.6", - "3.4.7", - "3.4.8", - "35.0.0", - "36.0.0", - "36.0.1", - "36.0.2", - "37.0.0", - "37.0.1", - "37.0.2", - "37.0.3", - "37.0.4", - "38.0.0", - "38.0.1", - "38.0.2", - "38.0.3", - "38.0.4", - "39.0.0", - "39.0.1", - "39.0.2", - "40.0.0", - "40.0.1", - "40.0.2", - "41.0.0", - "41.0.1", - "41.0.2", - "41.0.3", - "41.0.4", - "41.0.5", - "41.0.6", - "41.0.7", - "42.0.0", - "42.0.1", - "42.0.2", - "42.0.3", - "42.0.4", - "42.0.5", - "42.0.6", - "42.0.7", - "42.0.8", - "43.0.0", - "43.0.1", - "43.0.3", - "44.0.0", - "44.0.1", - "44.0.2", - "44.0.3", - "45.0.0", - "45.0.1", - "45.0.2", - "45.0.3", - "45.0.4", - "45.0.5", - "45.0.6", - "45.0.7", - "46.0.0", - "46.0.1", - "46.0.2", - "46.0.3", - "46.0.4", - "46.0.5", - "46.0.6", - "46.0.7", - "47.0.0", - "48.0.0" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/06/GHSA-537c-gmf6-5ccf/GHSA-537c-gmf6-5ccf.json" - } - } - ], - "references": [ - { - "type": "WEB", - "url": "https://github.com/pyca/cryptography/security/advisories/GHSA-537c-gmf6-5ccf" - }, - { - "type": "PACKAGE", - "url": "https://github.com/pyca/cryptography" - }, - { - "type": "WEB", - "url": "https://openssl-library.org/news/secadv/20260609.txt" - } - ], - "database_specific": { - "cwe_ids": [ - "CWE-125", - "CWE-1395" - ], - "github_reviewed": true, - "github_reviewed_at": "2026-06-15T20:12:27Z", - "nvd_published_at": null, - "severity": "HIGH" - } - } - ], - "groups": [ - { - "ids": [ - "GHSA-537c-gmf6-5ccf" - ], - "aliases": [ - "GHSA-537c-gmf6-5ccf" - ], - "max_severity": "7.5" - } - ], "licenses": [ "Apache-2.0 OR BSD-3-Clause" ] @@ -1097,7 +867,7 @@ { "package": { "name": "filelock", - "version": "3.25.2", + "version": "3.29.7", "ecosystem": "PyPI" }, "licenses": [ @@ -1830,178 +1600,581 @@ "version": "1.26.0", "ecosystem": "PyPI" }, + "vulnerabilities": [ + { + "modified": "2026-07-16T20:00:21Z", + "published": "2026-07-16T19:56:12Z", + "schema_version": "1.7.5", + "id": "GHSA-hvrp-rf83-w775", + "aliases": [ + "CVE-2026-52870" + ], + "summary": "MCP Python SDK: Experimental task handlers allow any client to access and cancel other clients' tasks", + "details": "### Summary\nIn affected versions, the default request handlers installed by the experimental tasks feature (`server.experimental.enable_tasks()`) did not check which session created a task before acting on it. On a server with more than one connected client, any client could observe, read results from, and cancel tasks belonging to other clients.\n\n### Am I affected?\nOnly if the developer's application server calls `server.experimental.enable_tasks()`. If `grep -r enable_tasks` over their codebase finds nothing, the application is not affected.\n\n### Details\nWhen tasks support is enabled on the low-level server, default handlers are registered for `tasks/list`, `tasks/get`, `tasks/result`, and `tasks/cancel`. These handlers operated on the task identifier alone and kept no record of the session that created each task. Because `tasks/list` returned every task in the store, a connected client did not need to know any identifiers in advance: it could enumerate all tasks, read any task's status and result via `tasks/get` and `tasks/result`, retrieve queued task messages \u2014 such as elicitation requests intended for the task's creator, which are removed from the queue on delivery, so the intended recipient never receives them \u2014 and cancel any task via `tasks/cancel`.\n\n### Impact\nServers that call `server.experimental.enable_tasks()` and serve multiple clients are affected: one client can read other clients' task results and elicitation payloads, consume messages meant for them, and cancel their tasks. The feature is experimental and opt-in, so servers that never enable it are unaffected. Servers that registered their own task handlers instead of the defaults are affected only if those handlers have the same omission.\n\n### Mitigation\nUpgrade to version 1.27.2 or later, in which task IDs generated by `run_task()` embed an opaque per-session marker and the default handlers restrict each session to its own tasks: requests for another session's task receive \"task not found\", and `tasks/list` returns only the requesting session's tasks. Tasks created with explicitly chosen IDs or written directly through a `TaskStore` remain reachable by ID but are not listed. Alternatively, leave the experimental tasks feature disabled, or register task handlers that validate session ownership.", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:L/A:L" + } + ], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "mcp", + "purl": "pkg:pypi/mcp" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.23.0" + }, + { + "fixed": "1.27.2" + } + ] + } + ], + "versions": [ + "1.23.0", + "1.23.1", + "1.23.2", + "1.23.3", + "1.24.0", + "1.25.0", + "1.26.0", + "1.27.0", + "1.27.1" + ], + "database_specific": { + "last_known_affected_version_range": "<= 1.27.1", + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/07/GHSA-hvrp-rf83-w775/GHSA-hvrp-rf83-w775.json" + } + } + ], + "references": [ + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/security/advisories/GHSA-hvrp-rf83-w775" + }, + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2026-52870" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/pull/2720" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/commit/62137874ff26dd74d2fea80ff528a7fd9ca7a5e7" + }, + { + "type": "PACKAGE", + "url": "https://github.com/modelcontextprotocol/python-sdk" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/releases/tag/v1.27.2" + } + ], + "database_specific": { + "cwe_ids": [ + "CWE-862" + ], + "github_reviewed": true, + "github_reviewed_at": "2026-07-16T19:56:12Z", + "nvd_published_at": "2026-07-15T20:17:38Z", + "severity": "HIGH" + } + }, + { + "modified": "2026-07-16T20:15:16Z", + "published": "2026-07-16T19:58:53Z", + "schema_version": "1.7.5", + "id": "GHSA-jpw9-pfvf-9f58", + "aliases": [ + "CVE-2026-52869" + ], + "summary": "MCP Python SDK: HTTP transports serve session requests without verifying the authenticated principal", + "details": "### Summary\nIn affected versions, the SSE and Streamable HTTP server transports routed incoming requests to an existing session based only on the session identifier, without verifying that the request was authenticated as the same principal that created the session. Anyone who learned or guessed a session ID could send JSON-RPC messages on that session, regardless of which bearer token the request carried.\n\n### Am I affected?\nOnly if a developer's application server uses an HTTP transport (SSE, or Streamable HTTP in stateful mode) **and** authenticates requests. Servers on stdio, stateless Streamable HTTP, or with no authentication configured are not affected.\n\n### Details\nBoth transports look up the target session by its identifier alone \u2014 the `session_id` query parameter for SSE (`mcp.server.sse.SseServerTransport`) and the `Mcp-Session-Id` header for Streamable HTTP (`mcp.server.streamable_http_manager.StreamableHTTPSessionManager`). Once the lookup succeeded, the request was handled on that session without comparing its authentication context to the credentials presented when the session was created, so a request authenticated as a different OAuth client could inject messages into the session. On the SSE transport the response is delivered to the original client's event stream; on the Streamable HTTP transport it is returned on the injecting request, so the injecting client can also read the result. The SSE transport has been affected since the first release; the Streamable HTTP transport since version 1.8.0.\n\n### Impact\nServers using either HTTP transport together with the SDK's built-in bearer-token authentication are affected: the per-client isolation that authentication provides can be bypassed for any session whose ID is known. Session IDs are randomly generated UUIDs, so exploitation requires obtaining one out of band (logs, network observation). Servers that do not enable bearer-token authentication have no per-client isolation to bypass and are not addressed by this advisory, and stateless Streamable HTTP deployments do not maintain sessions and are unaffected.\n\n### Mitigation\nUpgrade to version 1.27.2 or later, which records the authenticated principal that created each session \u2014 the OAuth client ID together with the token's issuer and subject when the token verifier supplies them \u2014 and answers requests presenting a different principal with the same 404 response as for an unknown session.\n\nDeployments where many end users share a single OAuth client (hosted MCP clients, gateways) should ensure their token verifier populates `AccessToken.subject` (e.g. from the token's `sub` claim) so sessions are isolated per user rather than per client. Deployments using a custom authentication backend other than the built-in `BearerAuthBackend` should enforce an equivalent check themselves.", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:H/PR:L/UI:N/S:U/C:H/I:H/A:L" + } + ], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "mcp", + "purl": "pkg:pypi/mcp" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.27.2" + } + ] + } + ], + "versions": [ + "0.9.1", + "1.0.0", + "1.1.0", + "1.1.1", + "1.1.2", + "1.1.3", + "1.10.0", + "1.10.1", + "1.11.0", + "1.12.0", + "1.12.1", + "1.12.2", + "1.12.3", + "1.12.4", + "1.13.0", + "1.13.1", + "1.14.0", + "1.14.1", + "1.15.0", + "1.16.0", + "1.17.0", + "1.18.0", + "1.19.0", + "1.2.0", + "1.2.0rc1", + "1.2.1", + "1.20.0", + "1.21.0", + "1.21.1", + "1.21.2", + "1.22.0", + "1.23.0", + "1.23.1", + "1.23.2", + "1.23.3", + "1.24.0", + "1.25.0", + "1.26.0", + "1.27.0", + "1.27.1", + "1.3.0", + "1.3.0rc1", + "1.4.0", + "1.4.1", + "1.5.0", + "1.6.0", + "1.7.0", + "1.7.1", + "1.8.0", + "1.8.1", + "1.9.0", + "1.9.1", + "1.9.2", + "1.9.3", + "1.9.4" + ], + "database_specific": { + "last_known_affected_version_range": "<= 1.27.1", + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/07/GHSA-jpw9-pfvf-9f58/GHSA-jpw9-pfvf-9f58.json" + } + } + ], + "references": [ + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/security/advisories/GHSA-jpw9-pfvf-9f58" + }, + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2026-52869" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/pull/2690" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/pull/2719" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/commit/1abcca2408a6b50e10ec601181f63f9978705c00" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/commit/ce267b6fc515dc4efc1dc70b6975b16ff0feef0a" + }, + { + "type": "PACKAGE", + "url": "https://github.com/modelcontextprotocol/python-sdk" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/releases/tag/v1.27.2" + } + ], + "database_specific": { + "cwe_ids": [ + "CWE-639" + ], + "github_reviewed": true, + "github_reviewed_at": "2026-07-16T19:58:53Z", + "nvd_published_at": "2026-07-15T20:17:38Z", + "severity": "HIGH" + } + }, + { + "modified": "2026-07-16T20:30:09Z", + "published": "2026-07-16T20:14:34Z", + "schema_version": "1.7.5", + "id": "GHSA-vj7q-gjh5-988w", + "aliases": [ + "CVE-2026-59950" + ], + "summary": "MCP Python SDK: WebSocket server transport does not support Host/Origin validation", + "details": "### Summary\nIn affected versions, the deprecated WebSocket server transport (`mcp.server.websocket.websocket_server`) accepted the WebSocket handshake without applying any `Host` or `Origin` header validation. The `TransportSecuritySettings` mechanism that the SSE and Streamable HTTP transports use for this purpose was not wired into the WebSocket transport, so there was no SDK-level way to restrict which origins could connect.\n\n### Am I affected?\nOnly if a developer's application server exposes `mcp.server.websocket.websocket_server`. This transport has never been part of the MCP specification, is marked deprecated, and is not reachable through `FastMCP` \u2014 a developer must have wired it into an ASGI application themselves. Servers using stdio, SSE, or Streamable HTTP are not affected by this advisory.\n\n### Details\n`websocket_server()` constructed a Starlette `WebSocket` and called `accept(subprotocol=\"mcp\")` immediately, with no inspection of the connection's headers. By contrast, `SseServerTransport` and `StreamableHTTPServerTransport` accept an optional `security_settings: TransportSecuritySettings` and run `TransportSecurityMiddleware.validate_request()` against the incoming `Host` and `Origin` headers before establishing a session. Because browsers attach an `Origin` header to cross-origin WebSocket upgrade requests but do not enforce a same-origin policy on the response, a web page served from any origin could open a WebSocket to a reachable MCP server on this transport, complete the `initialize` handshake, and issue JSON-RPC requests on the resulting session.\n\n### Impact\nA user who runs an MCP server on this transport bound to localhost or a LAN address, without a separate authentication or origin gate in front of it, and visits a malicious web page, can have that page enumerate and invoke the server's tools and read its resources. The consequences depend entirely on what the server exposes. The transport itself requires no token or prior session. Some browsers prompt before allowing a public page to open a connection to a local-network address, which adds a user-interaction step but is not a substitute for server-side validation.\n\n### Mitigation\nUpgrade to version 1.28.1 or later, in which `websocket_server()` accepts the same optional `security_settings: TransportSecuritySettings` argument as the other HTTP-based transports and validates the `Host` and `Origin` headers before accepting the handshake; a request that fails validation is rejected with HTTP 403 and `ValueError(\"Request validation failed\")` is raised to the caller. As with the other transports the parameter defaults to `None`, which leaves validation disabled, so upgrading alone does not change behaviour: pass a `TransportSecuritySettings` with `enable_dns_rebinding_protection=True` and appropriate `allowed_hosts` / `allowed_origins` to receive the protection. The recommended path remains to migrate off this deprecated transport to Streamable HTTP, where `FastMCP` enables this protection automatically for localhost binds. The WebSocket transport has been removed entirely in v2.", + "severity": [ + { + "type": "CVSS_V4", + "score": "CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:P/VC:H/VI:H/VA:N/SC:N/SI:N/SA:N" + } + ], + "affected": [ + { + "package": { + "ecosystem": "PyPI", + "name": "mcp", + "purl": "pkg:pypi/mcp" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.28.1" + } + ] + } + ], + "versions": [ + "0.9.1", + "1.0.0", + "1.1.0", + "1.1.1", + "1.1.2", + "1.1.3", + "1.10.0", + "1.10.1", + "1.11.0", + "1.12.0", + "1.12.1", + "1.12.2", + "1.12.3", + "1.12.4", + "1.13.0", + "1.13.1", + "1.14.0", + "1.14.1", + "1.15.0", + "1.16.0", + "1.17.0", + "1.18.0", + "1.19.0", + "1.2.0", + "1.2.0rc1", + "1.2.1", + "1.20.0", + "1.21.0", + "1.21.1", + "1.21.2", + "1.22.0", + "1.23.0", + "1.23.1", + "1.23.2", + "1.23.3", + "1.24.0", + "1.25.0", + "1.26.0", + "1.27.0", + "1.27.1", + "1.27.2", + "1.28.0", + "1.3.0", + "1.3.0rc1", + "1.4.0", + "1.4.1", + "1.5.0", + "1.6.0", + "1.7.0", + "1.7.1", + "1.8.0", + "1.8.1", + "1.9.0", + "1.9.1", + "1.9.2", + "1.9.3", + "1.9.4" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/07/GHSA-vj7q-gjh5-988w/GHSA-vj7q-gjh5-988w.json" + } + } + ], + "references": [ + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/security/advisories/GHSA-vj7q-gjh5-988w" + }, + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2026-59950" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/pull/2992" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/commit/777b8d06710c140e3606b0d4598e2aa48546c266" + }, + { + "type": "PACKAGE", + "url": "https://github.com/modelcontextprotocol/python-sdk" + }, + { + "type": "WEB", + "url": "https://github.com/modelcontextprotocol/python-sdk/releases/tag/v1.28.1" + } + ], + "database_specific": { + "cwe_ids": [ + "CWE-1385", + "CWE-346" + ], + "github_reviewed": true, + "github_reviewed_at": "2026-07-16T20:14:34Z", + "nvd_published_at": "2026-07-15T21:16:55Z", + "severity": "HIGH" + } + } + ], + "groups": [ + { + "ids": [ + "GHSA-hvrp-rf83-w775" + ], + "aliases": [ + "CVE-2026-52870", + "GHSA-hvrp-rf83-w775" + ], + "max_severity": "7.6" + }, + { + "ids": [ + "GHSA-jpw9-pfvf-9f58" + ], + "aliases": [ + "CVE-2026-52869", + "GHSA-jpw9-pfvf-9f58" + ], + "max_severity": "7.1" + }, + { + "ids": [ + "GHSA-vj7q-gjh5-988w" + ], + "aliases": [ + "CVE-2026-59950", + "GHSA-vj7q-gjh5-988w" + ], + "max_severity": "7.6" + } + ], "licenses": [ - "MIT" - ] - }, - { - "package": { - "name": "mdurl", - "version": "0.1.2", - "ecosystem": "PyPI" - }, - "licenses": [ - "MIT" - ] - }, - { - "package": { - "name": "mlflow-skinny", - "version": "3.11.1", - "ecosystem": "PyPI" - }, - "licenses": [ - "non-standard" - ] - }, - { - "package": { - "name": "mmh3", - "version": "5.2.1", - "ecosystem": "PyPI" - }, - "licenses": [ - "non-standard" - ] - }, - { - "package": { - "name": "more-itertools", - "version": "10.8.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "MIT" - ] - }, - { - "package": { - "name": "mpmath", - "version": "1.3.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "non-standard" - ] - }, - { - "package": { - "name": "multidict", - "version": "6.7.1", - "ecosystem": "PyPI" - }, - "licenses": [ - "Apache-2.0" - ] - }, - { - "package": { - "name": "multiprocess", - "version": "0.70.16", - "ecosystem": "PyPI" - }, - "licenses": [ - "BSD-3-Clause" - ] - }, - { - "package": { - "name": "mypy-extensions", - "version": "1.0.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "MIT" - ] - }, - { - "package": { - "name": "nemo-anonymizer", - "version": "0.3.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "Apache-2.0" - ] - }, - { - "package": { - "name": "nemo-relay", - "version": "0.4.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "Apache-2.0" - ] - }, - { - "package": { - "name": "nemo-safe-synthesizer", - "version": "0.1.7", - "ecosystem": "PyPI" - }, - "licenses": [ - "Apache-2.0" - ] - }, - { - "package": { - "name": "nemoguardrails", - "version": "0.23.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "non-standard" - ] - }, - { - "package": { - "name": "nest-asyncio", - "version": "1.6.0", - "ecosystem": "PyPI" - }, - "licenses": [ - "non-standard" + "MIT" ] }, { "package": { - "name": "nest-asyncio2", - "version": "1.7.2", + "name": "mdurl", + "version": "0.1.2", "ecosystem": "PyPI" }, "licenses": [ - "non-standard" + "MIT" ] }, { "package": { - "name": "networkx", - "version": "3.6.1", + "name": "mlflow-skinny", + "version": "3.11.1", "ecosystem": "PyPI" }, "licenses": [ - "BSD-3-Clause" + "non-standard" ] }, { "package": { - "name": "ngcsdk", - "version": "4.16.0", + "name": "mmh3", + "version": "5.2.1", "ecosystem": "PyPI" }, "licenses": [ - "Apache-2.0" + "non-standard" ] }, { "package": { - "name": "nltk", - "version": "3.10.0", + "name": "more-itertools", + "version": "10.8.0", "ecosystem": "PyPI" }, "licenses": [ - "Apache-2.0" + "MIT" + ] + }, + { + "package": { + "name": "mpmath", + "version": "1.3.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "non-standard" + ] + }, + { + "package": { + "name": "multidict", + "version": "6.7.1", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" + ] + }, + { + "package": { + "name": "multiprocess", + "version": "0.70.16", + "ecosystem": "PyPI" + }, + "licenses": [ + "BSD-3-Clause" + ] + }, + { + "package": { + "name": "mypy-extensions", + "version": "1.0.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "MIT" + ] + }, + { + "package": { + "name": "nemo-anonymizer", + "version": "0.3.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" + ] + }, + { + "package": { + "name": "nemo-relay", + "version": "0.4.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" + ] + }, + { + "package": { + "name": "nemo-safe-synthesizer", + "version": "0.1.7", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" + ] + }, + { + "package": { + "name": "nemoguardrails", + "version": "0.23.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "non-standard" + ] + }, + { + "package": { + "name": "nest-asyncio", + "version": "1.6.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "non-standard" + ] + }, + { + "package": { + "name": "nest-asyncio2", + "version": "1.7.2", + "ecosystem": "PyPI" + }, + "licenses": [ + "non-standard" + ] + }, + { + "package": { + "name": "networkx", + "version": "3.6.1", + "ecosystem": "PyPI" + }, + "licenses": [ + "BSD-3-Clause" + ] + }, + { + "package": { + "name": "ngcsdk", + "version": "4.16.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" + ] + }, + { + "package": { + "name": "nltk", + "version": "3.10.0", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" ] }, { @@ -3064,7 +3237,7 @@ { "package": { "name": "platformdirs", - "version": "4.9.4", + "version": "4.10.0", "ecosystem": "PyPI" }, "licenses": [ @@ -4832,513 +5005,9 @@ { "package": { "name": "soupsieve", - "version": "2.8.3", + "version": "2.8.4", "ecosystem": "PyPI" }, - "vulnerabilities": [ - { - "modified": "2026-07-13T16:31:33Z", - "published": "2026-07-13T15:46:30Z", - "schema_version": "1.7.5", - "id": "PYSEC-2026-3071", - "aliases": [ - "CVE-2026-49476", - "GHSA-2wc2-fm75-p42x" - ], - "summary": "Soup Sieve has Memory Exhaustion via Large Comma-Separated Selector Lists", - "details": "### Summary\n\nThe CSS selector parser in soupsieve (the CSS selector engine for Beautiful Soup 4) allocates unbounded memory when compiling large comma-separated selector lists. An attacker who can supply a crafted CSS selector string to `soupsieve.compile()` or Beautiful Soup's `.select()` / `.select_one()` can cause the application to allocate hundreds of megabytes of heap memory from a relatively small input, leading to memory exhaustion and denial of service.\n\nTo be completely transparent, AI tools helped surface this issue. However, it was independently reproduced and carefully validated. Researchers follow responsible disclosure practices and originally shared this report privately.\n\nA **500 KB** selector string triggers allocation of approximately **244 MB** of heap memory - a 488x\u2014 amplification ratio**.\n\n### Details\n\n**Affected code:** `soupsieve/css_parser.py`, lines ~204, 925, 1106\n\nThe soupsieve CSS parser splits comma-separated selector lists and creates one `CSSSelector` object per list item. Each `CSSSelector` object contains parsed selector data structures including `SelectorList`, `Selector`, and associated tag/attribute/pseudo-class metadata.\n\nWhen a selector string such as `a,a,a,...` (with 250,000 comma-separated items) is passed to `sv.compile()`, the parser:\n\n1. Tokenises the entire string and identifies each comma-delimited segment (line ~1106)\n2. Parses each segment into a full `Selector` object with all associated metadata (line ~925)\n3. Stores all parsed selectors in a `SelectorList` (line ~204)\n\n**Root cause:** No limit is enforced on the number of selectors in a comma-separated list. The parser will attempt to parse and store an arbitrary number of selectors, with each selector object consuming approximately **976 bytes** of heap memory. The total allocation scales linearly with the number of list items, but the amplification ratio (output memory / input bytes) is extremely high because each single-character selector like `a` expands into a complex object graph.\n\n**Attack surface:** Any application that passes user-supplied CSS selectors to `soupsieve.compile()` or Beautiful Soup's `.select()` / `.select_one()`.\n\n### Proof of Concept\n\n```python\nimport tracemalloc\nimport soupsieve as sv\n\ntracemalloc.start()\n\n# Build a 500 KB selector string: \"a,a,a,...,a\" (250,000 items)\ncount = 250_000\nselector = \",\".join(\"a\" for _ in range(count))\nprint(f\"Selector string size: {len(selector):,} bytes ({len(selector) / 1024:.0f} KB)\")\n\n# Compile the selector \u00e2\u20ac\u201d this allocates ~244 MB\ncompiled = sv.compile(selector)\n\ncurrent, peak = tracemalloc.get_traced_memory()\ntracemalloc.stop()\n\nprint(f\"Compiled selector count: {len(compiled.selectors):,}\")\nprint(f\"Current memory: {current / 1024 / 1024:.1f} MB\")\nprint(f\"Peak memory: {peak / 1024 / 1024:.1f} MB\")\nprint(f\"Amplification ratio: {peak / len(selector):.0f}x\")\n\n# Expected output:\n# Selector string size: 499,999 bytes (488 KB)\n# Compiled selector count: 250,000\n# Current memory: ~244 MB\n# Peak memory: ~244 MB\n# Amplification ratio: ~488x\n```\n\n### Impact\n\n**Severity: High**\n\nAn attacker can exhaust available memory on any server-side Python application that compiles user-supplied CSS selectors via soupsieve. This can cause:\n\n- **OOM kills** in containerised deployments (Kubernetes pods, Docker containers) with memory limits\n- **Swap thrashing** on bare-metal servers, degrading performance for all co-located processes\n- **Process termination** via Python's `MemoryError` exception if the system runs out of addressable memory\n\n| Parameter | Value |\n|---|---|\n| Input size | ~500 KB selector string |\n| Memory allocated | ~244 MB |\n| Amplification ratio | ~488\u00c3\u2014 |\n| Per-object overhead | ~976 bytes per selector |\n| Authentication required | None |\n| User interaction required | None |\n\n**Scalability of attack:** The memory allocation scales linearly - doubling the selector count doubles memory usage. An attacker can tune the payload to exactly exhaust a target's memory limits. Multiple concurrent requests multiply the effect.\n\n**Downstream exposure:** soupsieve is an automatic dependency of `beautifulsoup4`, one of the most widely installed Python packages. Any web application accepting CSS selectors from users (e.g., web scraping APIs, content filtering tools, CMS preview features) is potentially affected.\n\n---\n### Credit\n\nDiscovered by a security research team from the University of Sydney, focused on detecting open source software vulnerabilities.\nLiyi Zhou: https://lzhou1110.github.io/\nZiyue Wang: https://zyy0530.github.io/\nStrick: https://str1ckl4nd.github.io/\nMaurice: https://maurice.busystar.org/\nChenchen Yu: https://7thparkk.github.io/", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" - } - ], - "affected": [ - { - "package": { - "ecosystem": "PyPI", - "name": "soupsieve", - "purl": "pkg:pypi/soupsieve" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "2.8.4" - } - ] - } - ], - "versions": [ - "0.4", - "0.5", - "0.5.1", - "0.5.2", - "0.5.3", - "0.6", - "1.0", - "1.0.1", - "1.0.2", - "1.0b1", - "1.0b2", - "1.1", - "1.2", - "1.2.1", - "1.3", - "1.3.1", - "1.4", - "1.5", - "1.6", - "1.6.1", - "1.6.2", - "1.7", - "1.7.1", - "1.7.2", - "1.7.3", - "1.8", - "1.9", - "1.9.1", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "2.0", - "2.0.1", - "2.1", - "2.2", - "2.2.1", - "2.3", - "2.3.1", - "2.3.2", - "2.3.2.post1", - "2.4", - "2.4.1", - "2.5", - "2.6", - "2.7", - "2.8", - "2.8.1", - "2.8.2", - "2.8.3" - ], - "database_specific": { - "source": "https://github.com/pypa/advisory-database/blob/main/vulns/soupsieve/PYSEC-2026-3071.yaml" - } - } - ], - "references": [ - { - "type": "WEB", - "url": "https://github.com/facelessuser/soupsieve/security/advisories/GHSA-2wc2-fm75-p42x" - }, - { - "type": "PACKAGE", - "url": "https://github.com/facelessuser/soupsieve" - }, - { - "type": "PACKAGE", - "url": "https://pypi.org/project/soupsieve" - }, - { - "type": "ADVISORY", - "url": "https://github.com/advisories/GHSA-2wc2-fm75-p42x" - }, - { - "type": "ADVISORY", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2026-49476" - } - ] - }, - { - "modified": "2026-07-13T16:31:33Z", - "published": "2026-07-13T15:46:30Z", - "schema_version": "1.7.5", - "id": "PYSEC-2026-3072", - "aliases": [ - "CVE-2026-49477", - "GHSA-836r-79rf-4m37" - ], - "summary": "Soup Sieve: Regular Expression Denial of Service (ReDoS) via Selector Parser", - "details": "### Summary\n\nThe CSS selector parser in soupsieve (the CSS selector engine for Beautiful Soup 4) contains a regular expression vulnerable to catastrophic backtracking. When processing an attribute selector with an unterminated quoted value, the `VALUE` regex pattern in `css_parser.py` enters exponential backtracking. A payload of only **300 bytes** causes the regex engine to hang for **over 3 seconds**, enabling a trivial Regular Expression Denial of Service (ReDoS) attack.\n\nTo be completely transparent, AI tools helped surface this issue. However, this was independently reproduced and carefully validated.\n\nAny application that passes untrusted CSS selector strings to `soupsieve.compile()` or Beautiful Soup's `.select()` / `.select_one()` is affected.\n\n### Details\n\n**Affected code:** `soupsieve/css_parser.py`, line ~121 - `RE_VALUES` / `VALUE` regex pattern\n\nThe soupsieve CSS parser uses a compiled regular expression to tokenise attribute selector values. This pattern matches both quoted strings (`\"value\"` or `'value'`) and unquoted identifiers. The regex contains alternation branches for:\n\n1. Double-quoted strings: `\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\"`\n2. Single-quoted strings: `'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'`\n3. Unquoted identifiers\n\nWhen an attribute selector contains an **unterminated quoted value** - e.g., `[a=\"xxxx...` (opening `\"` but no closing `\"`) -\u201d the regex engine attempts to match the quoted-string branch. After that branch fails (no closing quote), the engine backtracks and attempts to match the remaining input against subsequent alternation branches and parent patterns. The structure of the pattern causes **catastrophic backtracking** where the number of backtracking steps grows exponentially with the length of the content between the opening quote and the end of the string.\n\n**Root cause:** The regex pattern does not anchor or guard against the case where a quoted string is never terminated. The overlapping character classes across alternation branches create exponential backtracking when the quoted-string branch fails on long input.\n\n**Key characteristics:**\n- **Input size:** Only 300 bytes are needed to trigger a >3 second hang\n- **Amplification:** Each additional character approximately doubles the backtracking time\n- **No memory impact:** The attack consumes CPU only (regex backtracking is compute-bound)\n\n### Proof of Concept\n\n```python\nimport time\nimport soupsieve as sv\n\nPAYLOAD_LEN = 300\n\n# Control: well-formed selector with terminated quote (completes instantly)\nwell_formed = '[a=\"' + ('x' * PAYLOAD_LEN) + '\"]'\nstart = time.perf_counter()\ntry:\n sv.compile(well_formed)\nexcept Exception:\n pass\ncontrol_time = time.perf_counter() - start\nprint(f\"Well-formed selector ({len(well_formed)} bytes): {control_time:.4f}s\")\n\n# Exploit: unterminated quote triggers catastrophic regex backtracking\nmalformed = '[a=\"' + ('x' * PAYLOAD_LEN)\nstart = time.perf_counter()\ntry:\n sv.compile(malformed) # WARNING: This will hang for >3 seconds\nexcept Exception:\n pass\nexploit_time = time.perf_counter() - start\nprint(f\"Malformed selector ({len(malformed)} bytes): {exploit_time:.4f}s\")\n\nslowdown = exploit_time / max(control_time, 1e-9)\nprint(f\"Slowdown: {slowdown:.0f}x\")\n\n# Expected output:\n# Well-formed selector (306 bytes): ~0.001s\n# Malformed selector (304 bytes): >3.0s (may need to be killed)\n# Slowdown: >3000x\n#\n# NOTE: On some systems the malformed selector may hang indefinitely.\n# Use a timeout mechanism (signal.alarm, threading.Timer) when testing.\n```\n\n**Safe testing variant with timeout:**\n\n```python\nimport signal\nimport soupsieve as sv\n\ndef timeout_handler(signum, frame):\n raise TimeoutError(\"ReDoS confirmed: regex backtracking exceeded timeout\")\n\nPAYLOAD_LEN = 300\nmalformed = '[a=\"' + ('x' * PAYLOAD_LEN)\n\nsignal.signal(signal.SIGALRM, timeout_handler)\nsignal.alarm(3) # 3-second timeout\n\ntry:\n sv.compile(malformed)\n print(\"Selector compiled (not vulnerable)\")\nexcept TimeoutError as e:\n print(f\"VULNERABLE: {e}\")\nexcept Exception as e:\n print(f\"Other error: {e}\")\nfinally:\n signal.alarm(0) # Cancel the alarm\n```\n\n### Impact\n\n**Severity: High**\n\nAn attacker can cause CPU exhaustion on any server-side Python application that compiles user-supplied CSS selectors via soupsieve. The attack is particularly dangerous because:\n\n1. **Tiny payload:** Only 300 bytes are needed - well within typical URL parameter, form field, or API request limits\n2. **No special characters:** The payload consists entirely of printable ASCII characters (`[a=\"xxx...`)\n3. **Exponential scaling:** Each additional byte approximately doubles the backtracking time, making the attack easily tuneable\n4. **Thread blocking:** The regex engine blocks the calling thread with no opportunity for interruption (except via OS signals)\n\n| Parameter | Value |\n|---|---|\n| Input size | 300 bytes |\n| CPU time consumed | >3 seconds (exponential with payload length) |\n| Memory consumed | Negligible (CPU-only attack) |\n| Authentication required | None |\n| User interaction required | None |\n\n**Deployment impact:** In threaded or async web applications, a single malicious request blocks a worker thread for the duration of the backtracking. An attacker can submit multiple concurrent requests to exhaust all available workers, causing complete service denial. The small payload size makes the attack easy to deliver and difficult to detect via request size limits.\n\n**Downstream exposure:** soupsieve is an automatic dependency of `beautifulsoup4`, one of the most widely installed Python packages. Any web application, API, or service that accepts CSS selectors from users is potentially affected.\n\n---\n\n### Credit\n\nThe vulnerability was discovered by a security research team from the University of Sydney, whose focus is detecting open source software vulnerabilities.\nLiyi Zhou: https://lzhou1110.github.io/\nZiyue Wang: https://zyy0530.github.io/\nStrick: https://str1ckl4nd.github.io/\nMaurice: https://maurice.busystar.org/\nChenchen Yu: https://7thparkk.github.io/", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" - } - ], - "affected": [ - { - "package": { - "ecosystem": "PyPI", - "name": "soupsieve", - "purl": "pkg:pypi/soupsieve" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "2.8.4" - } - ] - } - ], - "versions": [ - "0.4", - "0.5", - "0.5.1", - "0.5.2", - "0.5.3", - "0.6", - "1.0", - "1.0.1", - "1.0.2", - "1.0b1", - "1.0b2", - "1.1", - "1.2", - "1.2.1", - "1.3", - "1.3.1", - "1.4", - "1.5", - "1.6", - "1.6.1", - "1.6.2", - "1.7", - "1.7.1", - "1.7.2", - "1.7.3", - "1.8", - "1.9", - "1.9.1", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "2.0", - "2.0.1", - "2.1", - "2.2", - "2.2.1", - "2.3", - "2.3.1", - "2.3.2", - "2.3.2.post1", - "2.4", - "2.4.1", - "2.5", - "2.6", - "2.7", - "2.8", - "2.8.1", - "2.8.2", - "2.8.3" - ], - "database_specific": { - "source": "https://github.com/pypa/advisory-database/blob/main/vulns/soupsieve/PYSEC-2026-3072.yaml" - } - } - ], - "references": [ - { - "type": "WEB", - "url": "https://github.com/facelessuser/soupsieve/security/advisories/GHSA-836r-79rf-4m37" - }, - { - "type": "PACKAGE", - "url": "https://github.com/facelessuser/soupsieve" - }, - { - "type": "PACKAGE", - "url": "https://pypi.org/project/soupsieve" - }, - { - "type": "ADVISORY", - "url": "https://github.com/advisories/GHSA-836r-79rf-4m37" - }, - { - "type": "ADVISORY", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2026-49477" - } - ] - }, - { - "modified": "2026-07-13T16:43:32Z", - "published": "2026-07-09T13:37:40Z", - "schema_version": "1.7.5", - "id": "GHSA-2wc2-fm75-p42x", - "aliases": [ - "CVE-2026-49476", - "PYSEC-2026-3071" - ], - "related": [ - "CGA-fqw6-g84h-prc6" - ], - "summary": "Soup Sieve has Memory Exhaustion via Large Comma-Separated Selector Lists", - "details": "### Summary\n\nThe CSS selector parser in soupsieve (the CSS selector engine for Beautiful Soup 4) allocates unbounded memory when compiling large comma-separated selector lists. An attacker who can supply a crafted CSS selector string to `soupsieve.compile()` or Beautiful Soup's `.select()` / `.select_one()` can cause the application to allocate hundreds of megabytes of heap memory from a relatively small input, leading to memory exhaustion and denial of service.\n\nTo be completely transparent, AI tools helped surface this issue. However, it was independently reproduced and carefully validated. Researchers follow responsible disclosure practices and originally shared this report privately.\n\nA **500 KB** selector string triggers allocation of approximately **244 MB** of heap memory - a 488x\u2014 amplification ratio**.\n\n### Details\n\n**Affected code:** `soupsieve/css_parser.py`, lines ~204, 925, 1106\n\nThe soupsieve CSS parser splits comma-separated selector lists and creates one `CSSSelector` object per list item. Each `CSSSelector` object contains parsed selector data structures including `SelectorList`, `Selector`, and associated tag/attribute/pseudo-class metadata.\n\nWhen a selector string such as `a,a,a,...` (with 250,000 comma-separated items) is passed to `sv.compile()`, the parser:\n\n1. Tokenises the entire string and identifies each comma-delimited segment (line ~1106)\n2. Parses each segment into a full `Selector` object with all associated metadata (line ~925)\n3. Stores all parsed selectors in a `SelectorList` (line ~204)\n\n**Root cause:** No limit is enforced on the number of selectors in a comma-separated list. The parser will attempt to parse and store an arbitrary number of selectors, with each selector object consuming approximately **976 bytes** of heap memory. The total allocation scales linearly with the number of list items, but the amplification ratio (output memory / input bytes) is extremely high because each single-character selector like `a` expands into a complex object graph.\n\n**Attack surface:** Any application that passes user-supplied CSS selectors to `soupsieve.compile()` or Beautiful Soup's `.select()` / `.select_one()`.\n\n### Proof of Concept\n\n```python\nimport tracemalloc\nimport soupsieve as sv\n\ntracemalloc.start()\n\n# Build a 500 KB selector string: \"a,a,a,...,a\" (250,000 items)\ncount = 250_000\nselector = \",\".join(\"a\" for _ in range(count))\nprint(f\"Selector string size: {len(selector):,} bytes ({len(selector) / 1024:.0f} KB)\")\n\n# Compile the selector \u00e2\u20ac\u201d this allocates ~244 MB\ncompiled = sv.compile(selector)\n\ncurrent, peak = tracemalloc.get_traced_memory()\ntracemalloc.stop()\n\nprint(f\"Compiled selector count: {len(compiled.selectors):,}\")\nprint(f\"Current memory: {current / 1024 / 1024:.1f} MB\")\nprint(f\"Peak memory: {peak / 1024 / 1024:.1f} MB\")\nprint(f\"Amplification ratio: {peak / len(selector):.0f}x\")\n\n# Expected output:\n# Selector string size: 499,999 bytes (488 KB)\n# Compiled selector count: 250,000\n# Current memory: ~244 MB\n# Peak memory: ~244 MB\n# Amplification ratio: ~488x\n```\n\n### Impact\n\n**Severity: High**\n\nAn attacker can exhaust available memory on any server-side Python application that compiles user-supplied CSS selectors via soupsieve. This can cause:\n\n- **OOM kills** in containerised deployments (Kubernetes pods, Docker containers) with memory limits\n- **Swap thrashing** on bare-metal servers, degrading performance for all co-located processes\n- **Process termination** via Python's `MemoryError` exception if the system runs out of addressable memory\n\n| Parameter | Value |\n|---|---|\n| Input size | ~500 KB selector string |\n| Memory allocated | ~244 MB |\n| Amplification ratio | ~488\u00c3\u2014 |\n| Per-object overhead | ~976 bytes per selector |\n| Authentication required | None |\n| User interaction required | None |\n\n**Scalability of attack:** The memory allocation scales linearly - doubling the selector count doubles memory usage. An attacker can tune the payload to exactly exhaust a target's memory limits. Multiple concurrent requests multiply the effect.\n\n**Downstream exposure:** soupsieve is an automatic dependency of `beautifulsoup4`, one of the most widely installed Python packages. Any web application accepting CSS selectors from users (e.g., web scraping APIs, content filtering tools, CMS preview features) is potentially affected.\n\n---\n### Credit\n\nDiscovered by a security research team from the University of Sydney, focused on detecting open source software vulnerabilities.\nLiyi Zhou: https://lzhou1110.github.io/\nZiyue Wang: https://zyy0530.github.io/\nStrick: https://str1ckl4nd.github.io/\nMaurice: https://maurice.busystar.org/\nChenchen Yu: https://7thparkk.github.io/", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" - } - ], - "affected": [ - { - "package": { - "ecosystem": "PyPI", - "name": "soupsieve", - "purl": "pkg:pypi/soupsieve" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "2.8.4" - } - ] - } - ], - "versions": [ - "0.4", - "0.5", - "0.5.1", - "0.5.2", - "0.5.3", - "0.6", - "1.0", - "1.0.1", - "1.0.2", - "1.0b1", - "1.0b2", - "1.1", - "1.2", - "1.2.1", - "1.3", - "1.3.1", - "1.4", - "1.5", - "1.6", - "1.6.1", - "1.6.2", - "1.7", - "1.7.1", - "1.7.2", - "1.7.3", - "1.8", - "1.9", - "1.9.1", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "2.0", - "2.0.1", - "2.1", - "2.2", - "2.2.1", - "2.3", - "2.3.1", - "2.3.2", - "2.3.2.post1", - "2.4", - "2.4.1", - "2.5", - "2.6", - "2.7", - "2.8", - "2.8.1", - "2.8.2", - "2.8.3" - ], - "database_specific": { - "last_known_affected_version_range": "<= 2.8.3", - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/07/GHSA-2wc2-fm75-p42x/GHSA-2wc2-fm75-p42x.json" - } - } - ], - "references": [ - { - "type": "WEB", - "url": "https://github.com/facelessuser/soupsieve/security/advisories/GHSA-2wc2-fm75-p42x" - }, - { - "type": "PACKAGE", - "url": "https://github.com/facelessuser/soupsieve" - } - ], - "database_specific": { - "cwe_ids": [ - "CWE-400", - "CWE-770" - ], - "github_reviewed": true, - "github_reviewed_at": "2026-07-09T13:37:40Z", - "nvd_published_at": null, - "severity": "HIGH" - } - }, - { - "modified": "2026-07-13T16:42:45Z", - "published": "2026-07-09T13:37:46Z", - "schema_version": "1.7.5", - "id": "GHSA-836r-79rf-4m37", - "aliases": [ - "CVE-2026-49477", - "PYSEC-2026-3072" - ], - "related": [ - "CGA-9cx6-r7p8-hvrm" - ], - "summary": "Soup Sieve: Regular Expression Denial of Service (ReDoS) via Selector Parser", - "details": "### Summary\n\nThe CSS selector parser in soupsieve (the CSS selector engine for Beautiful Soup 4) contains a regular expression vulnerable to catastrophic backtracking. When processing an attribute selector with an unterminated quoted value, the `VALUE` regex pattern in `css_parser.py` enters exponential backtracking. A payload of only **300 bytes** causes the regex engine to hang for **over 3 seconds**, enabling a trivial Regular Expression Denial of Service (ReDoS) attack.\n\nTo be completely transparent, AI tools helped surface this issue. However, this was independently reproduced and carefully validated.\n\nAny application that passes untrusted CSS selector strings to `soupsieve.compile()` or Beautiful Soup's `.select()` / `.select_one()` is affected.\n\n### Details\n\n**Affected code:** `soupsieve/css_parser.py`, line ~121 - `RE_VALUES` / `VALUE` regex pattern\n\nThe soupsieve CSS parser uses a compiled regular expression to tokenise attribute selector values. This pattern matches both quoted strings (`\"value\"` or `'value'`) and unquoted identifiers. The regex contains alternation branches for:\n\n1. Double-quoted strings: `\"[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*\"`\n2. Single-quoted strings: `'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'`\n3. Unquoted identifiers\n\nWhen an attribute selector contains an **unterminated quoted value** - e.g., `[a=\"xxxx...` (opening `\"` but no closing `\"`) -\u201d the regex engine attempts to match the quoted-string branch. After that branch fails (no closing quote), the engine backtracks and attempts to match the remaining input against subsequent alternation branches and parent patterns. The structure of the pattern causes **catastrophic backtracking** where the number of backtracking steps grows exponentially with the length of the content between the opening quote and the end of the string.\n\n**Root cause:** The regex pattern does not anchor or guard against the case where a quoted string is never terminated. The overlapping character classes across alternation branches create exponential backtracking when the quoted-string branch fails on long input.\n\n**Key characteristics:**\n- **Input size:** Only 300 bytes are needed to trigger a >3 second hang\n- **Amplification:** Each additional character approximately doubles the backtracking time\n- **No memory impact:** The attack consumes CPU only (regex backtracking is compute-bound)\n\n### Proof of Concept\n\n```python\nimport time\nimport soupsieve as sv\n\nPAYLOAD_LEN = 300\n\n# Control: well-formed selector with terminated quote (completes instantly)\nwell_formed = '[a=\"' + ('x' * PAYLOAD_LEN) + '\"]'\nstart = time.perf_counter()\ntry:\n sv.compile(well_formed)\nexcept Exception:\n pass\ncontrol_time = time.perf_counter() - start\nprint(f\"Well-formed selector ({len(well_formed)} bytes): {control_time:.4f}s\")\n\n# Exploit: unterminated quote triggers catastrophic regex backtracking\nmalformed = '[a=\"' + ('x' * PAYLOAD_LEN)\nstart = time.perf_counter()\ntry:\n sv.compile(malformed) # WARNING: This will hang for >3 seconds\nexcept Exception:\n pass\nexploit_time = time.perf_counter() - start\nprint(f\"Malformed selector ({len(malformed)} bytes): {exploit_time:.4f}s\")\n\nslowdown = exploit_time / max(control_time, 1e-9)\nprint(f\"Slowdown: {slowdown:.0f}x\")\n\n# Expected output:\n# Well-formed selector (306 bytes): ~0.001s\n# Malformed selector (304 bytes): >3.0s (may need to be killed)\n# Slowdown: >3000x\n#\n# NOTE: On some systems the malformed selector may hang indefinitely.\n# Use a timeout mechanism (signal.alarm, threading.Timer) when testing.\n```\n\n**Safe testing variant with timeout:**\n\n```python\nimport signal\nimport soupsieve as sv\n\ndef timeout_handler(signum, frame):\n raise TimeoutError(\"ReDoS confirmed: regex backtracking exceeded timeout\")\n\nPAYLOAD_LEN = 300\nmalformed = '[a=\"' + ('x' * PAYLOAD_LEN)\n\nsignal.signal(signal.SIGALRM, timeout_handler)\nsignal.alarm(3) # 3-second timeout\n\ntry:\n sv.compile(malformed)\n print(\"Selector compiled (not vulnerable)\")\nexcept TimeoutError as e:\n print(f\"VULNERABLE: {e}\")\nexcept Exception as e:\n print(f\"Other error: {e}\")\nfinally:\n signal.alarm(0) # Cancel the alarm\n```\n\n### Impact\n\n**Severity: High**\n\nAn attacker can cause CPU exhaustion on any server-side Python application that compiles user-supplied CSS selectors via soupsieve. The attack is particularly dangerous because:\n\n1. **Tiny payload:** Only 300 bytes are needed - well within typical URL parameter, form field, or API request limits\n2. **No special characters:** The payload consists entirely of printable ASCII characters (`[a=\"xxx...`)\n3. **Exponential scaling:** Each additional byte approximately doubles the backtracking time, making the attack easily tuneable\n4. **Thread blocking:** The regex engine blocks the calling thread with no opportunity for interruption (except via OS signals)\n\n| Parameter | Value |\n|---|---|\n| Input size | 300 bytes |\n| CPU time consumed | >3 seconds (exponential with payload length) |\n| Memory consumed | Negligible (CPU-only attack) |\n| Authentication required | None |\n| User interaction required | None |\n\n**Deployment impact:** In threaded or async web applications, a single malicious request blocks a worker thread for the duration of the backtracking. An attacker can submit multiple concurrent requests to exhaust all available workers, causing complete service denial. The small payload size makes the attack easy to deliver and difficult to detect via request size limits.\n\n**Downstream exposure:** soupsieve is an automatic dependency of `beautifulsoup4`, one of the most widely installed Python packages. Any web application, API, or service that accepts CSS selectors from users is potentially affected.\n\n---\n\n### Credit\n\nThe vulnerability was discovered by a security research team from the University of Sydney, whose focus is detecting open source software vulnerabilities.\nLiyi Zhou: https://lzhou1110.github.io/\nZiyue Wang: https://zyy0530.github.io/\nStrick: https://str1ckl4nd.github.io/\nMaurice: https://maurice.busystar.org/\nChenchen Yu: https://7thparkk.github.io/", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" - } - ], - "affected": [ - { - "package": { - "ecosystem": "PyPI", - "name": "soupsieve", - "purl": "pkg:pypi/soupsieve" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "2.8.4" - } - ] - } - ], - "versions": [ - "0.4", - "0.5", - "0.5.1", - "0.5.2", - "0.5.3", - "0.6", - "1.0", - "1.0.1", - "1.0.2", - "1.0b1", - "1.0b2", - "1.1", - "1.2", - "1.2.1", - "1.3", - "1.3.1", - "1.4", - "1.5", - "1.6", - "1.6.1", - "1.6.2", - "1.7", - "1.7.1", - "1.7.2", - "1.7.3", - "1.8", - "1.9", - "1.9.1", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "2.0", - "2.0.1", - "2.1", - "2.2", - "2.2.1", - "2.3", - "2.3.1", - "2.3.2", - "2.3.2.post1", - "2.4", - "2.4.1", - "2.5", - "2.6", - "2.7", - "2.8", - "2.8.1", - "2.8.2", - "2.8.3" - ], - "database_specific": { - "last_known_affected_version_range": "<= 2.8.3", - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2026/07/GHSA-836r-79rf-4m37/GHSA-836r-79rf-4m37.json" - } - } - ], - "references": [ - { - "type": "WEB", - "url": "https://github.com/facelessuser/soupsieve/security/advisories/GHSA-836r-79rf-4m37" - }, - { - "type": "PACKAGE", - "url": "https://github.com/facelessuser/soupsieve" - } - ], - "database_specific": { - "cwe_ids": [ - "CWE-1333", - "CWE-400" - ], - "github_reviewed": true, - "github_reviewed_at": "2026-07-09T13:37:46Z", - "nvd_published_at": null, - "severity": "HIGH" - } - } - ], - "groups": [ - { - "ids": [ - "PYSEC-2026-3071", - "GHSA-2wc2-fm75-p42x" - ], - "aliases": [ - "CVE-2026-49476", - "GHSA-2wc2-fm75-p42x", - "PYSEC-2026-3071" - ], - "max_severity": "7.5" - }, - { - "ids": [ - "PYSEC-2026-3072", - "GHSA-836r-79rf-4m37" - ], - "aliases": [ - "CVE-2026-49477", - "GHSA-836r-79rf-4m37", - "PYSEC-2026-3072" - ], - "max_severity": "7.5" - } - ], "licenses": [ "MIT" ] @@ -5616,7 +5285,7 @@ { "package": { "name": "tzdata", - "version": "2025.3", + "version": "2026.2", "ecosystem": "PyPI" }, "licenses": [ diff --git a/third_party/requirements-main.txt b/third_party/requirements-main.txt index 11138213bc..925533b070 100644 --- a/third_party/requirements-main.txt +++ b/third_party/requirements-main.txt @@ -26,7 +26,6 @@ # nemo-automodel-plugin # nemo-customizer-plugin # nemo-data-designer-plugin - # nemo-deployments-plugin # nemo-guardrails-plugin # nemo-rl-plugin # nemo-safe-synthesizer-plugin @@ -56,6 +55,7 @@ # nmp-common # nmp-customization-common # nmp-inference-gateway + # nmp-models # nmp-platform # nmp-platform-runner # nmp-rl @@ -121,6 +121,7 @@ -e ./sdk/python/nemo-platform ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') # via # models + # nemo-deployments-plugin # nemo-evaluator-plugin # nemo-platform # nemo-platform-ext @@ -283,9 +284,9 @@ annotated-types==0.7.0 ; (platform_machine == 'arm64' and sys_platform == 'darwi --hash=sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 \ --hash=sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89 # via pydantic -anthropic==0.101.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:1116a6a87c55757e0fbe3e1ba40804fbd04de7963601a6dd6b539a889f18de3e \ - --hash=sha256:cc3cc6576989471e2aa9132258034ad0ff0d8fe500b04ac499e4e46ed68c5ed0 +anthropic==0.116.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:5fc248fbb9fe03ef686f8a774f81586bca31a043260aab88b387ea3660f4a396 \ + --hash=sha256:6c0a7698e8d652455da3499978279bb2588c7264d0a35be3666009a4258c8256 # via # nemo-agents-plugin # nemo-platform-plugin @@ -526,31 +527,31 @@ colorlog==6.10.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or --hash=sha256:2d7e8348291948af66122cff006c9f8da6255d224e7cf8e37d8de2df3bad8c9c \ --hash=sha256:eb4ae5cb65fe7fec7773c2306061a8e63e02efc2c72eba9d27b0fa23c94f1321 # via optuna -cryptography==46.0.7 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65 \ - --hash=sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832 \ - --hash=sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067 \ - --hash=sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0 \ - --hash=sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968 \ - --hash=sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef \ - --hash=sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b \ - --hash=sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4 \ - --hash=sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308 \ - --hash=sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e \ - --hash=sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163 \ - --hash=sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77 \ - --hash=sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85 \ - --hash=sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7 \ - --hash=sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83 \ - --hash=sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85 \ - --hash=sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e \ - --hash=sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba \ - --hash=sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325 \ - --hash=sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455 \ - --hash=sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15 \ - --hash=sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5 \ - --hash=sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4 \ - --hash=sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f +cryptography==48.0.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a \ + --hash=sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f \ + --hash=sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a \ + --hash=sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1 \ + --hash=sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225 \ + --hash=sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92 \ + --hash=sha256:3752f2dbc8f07a30aad2932c986cea495b03bb554887828225da104f732852b6 \ + --hash=sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1 \ + --hash=sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8 \ + --hash=sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577 \ + --hash=sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67 \ + --hash=sha256:66fd0771e7b9c6dcd44cf1120690d2338d16d72795cf40cae2786a39eba65429 \ + --hash=sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd \ + --hash=sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9 \ + --hash=sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a \ + --hash=sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475 \ + --hash=sha256:88c852a0ae366e262e5a1744b685e6a433dc8788dd2a277e418bf4904203609d \ + --hash=sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c \ + --hash=sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6 \ + --hash=sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b \ + --hash=sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242 \ + --hash=sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691 \ + --hash=sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41 \ + --hash=sha256:fe0180af5bf9236518a087e35bf2d9a347d5f5f51e63c579d683ddff424e3d46 # via # authlib # data-designer-engine @@ -635,6 +636,7 @@ docker==7.1.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (p --hash=sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c \ --hash=sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0 # via + # nemo-deployments-plugin # nemo-platform-ext # nemo-platform-sdk # ngcsdk @@ -790,9 +792,9 @@ fastuuid==0.14.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or --hash=sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022 \ --hash=sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070 # via litellm -filelock==3.25.2 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694 \ - --hash=sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70 +filelock==3.29.7 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d \ + --hash=sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51 # via # datasets # huggingface-hub @@ -1188,6 +1190,7 @@ kubernetes==35.0.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') --hash=sha256:39e2b33b46e5834ef6c3985ebfe2047ab39135d41de51ce7641a7ca5b372a13d \ --hash=sha256:3d00d344944239821458b9efd484d6df9f011da367ecb155dadf9513f05f09ee # via + # nemo-deployments-plugin # nmp-common # nmp-jobs # nmp-models @@ -1636,7 +1639,9 @@ nvidia-nat-eval==1.8.0 ; (platform_machine == 'arm64' and sys_platform == 'darwi # nvidia-nat-langchain nvidia-nat-langchain==1.8.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:8120ad2b972ce2d90fae1e7f95b3daf6b463310e0105fc330896bfb069a5d403 - # via nemo-agents-plugin + # via + # nemo-agents-example-calculator + # nemo-agents-plugin nvidia-nat-opentelemetry==1.8.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:2ef2de28a1d07029126ae1c13ff39b9dd61a6985e43199e6fe80968b465b1eec # via nvidia-nat-langchain @@ -1988,9 +1993,9 @@ pkginfo==1.12.1.2 ; (platform_machine == 'arm64' and sys_platform == 'darwin') o --hash=sha256:5cd957824ac36f140260964eba3c6be6442a8359b8c48f4adf90210f33a04b7b \ --hash=sha256:c783ac885519cab2c34927ccfa6bf64b5a704d7c69afaea583dd9b7afe969343 # via nvidia-nat-core -platformdirs==4.9.4 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934 \ - --hash=sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868 +platformdirs==4.10.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7 \ + --hash=sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a # via # fastmcp # nvidia-nat-core @@ -2726,9 +2731,9 @@ sniffio==1.3.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or ( # nemo-platform-sdk # openai # pyleak -soupsieve==2.8.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349 \ - --hash=sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95 +soupsieve==2.8.4 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e \ + --hash=sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65 # via beautifulsoup4 sqlalchemy==2.0.48 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:07edba08061bc277bfdc772dd2a1a43978f5a45994dd3ede26391b405c15221e \ @@ -2999,9 +3004,9 @@ typing-inspection==0.4.2 ; (platform_machine == 'arm64' and sys_platform == 'dar # mcp # pydantic # pydantic-settings -tzdata==2025.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 \ - --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 +tzdata==2026.2 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ + --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 # via pandas tzlocal==5.3.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd \