diff --git a/e2e/k8s/values/default.yaml b/e2e/k8s/values/default.yaml index 8a63cf1493..e10e8bdb08 100644 --- a/e2e/k8s/values/default.yaml +++ b/e2e/k8s/values/default.yaml @@ -13,6 +13,24 @@ platformConfig: # This is used to enable mock provider mode for E2E tests. inference_gateway: mock_provider_prefix: igw-mock- + jobs: + executors: + - provider: cpu + profile: additional-volume + backend: kubernetes_job + config: + service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}' + launcher_image: '{{ include "nmp-core.image" . }}' + storage: + pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}' + additional_volumes: + - name: nmp-additional-volume + persistent_volume_claim: + claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + additional_volume_mounts: + - name: nmp-additional-volume + mount_path: /mnt/additional_storage core: storage: diff --git a/e2e/k8s/values/kind.yaml b/e2e/k8s/values/kind.yaml index 734c77515a..ecc99ce2c3 100644 --- a/e2e/k8s/values/kind.yaml +++ b/e2e/k8s/values/kind.yaml @@ -24,6 +24,24 @@ httpRoute: platformConfig: inference_gateway: mock_provider_prefix: igw-mock- + jobs: + executors: + - provider: cpu + profile: additional-volume + backend: kubernetes_job + config: + service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}' + launcher_image: '{{ include "nmp-core.image" . }}' + storage: + pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}' + additional_volumes: + - name: nmp-additional-volume + persistent_volume_claim: + claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + additional_volume_mounts: + - name: nmp-additional-volume + mount_path: /mnt/additional_storage models: controller: backends: diff --git a/e2e/k8s/values/minikube-auth-portforward.yaml b/e2e/k8s/values/minikube-auth-portforward.yaml index a84228dbc0..28cd5398cb 100644 --- a/e2e/k8s/values/minikube-auth-portforward.yaml +++ b/e2e/k8s/values/minikube-auth-portforward.yaml @@ -32,6 +32,24 @@ platformConfig: admin_email: "admin@example.com" inference_gateway: mock_provider_prefix: igw-mock- + jobs: + executors: + - provider: cpu + profile: additional-volume + backend: kubernetes_job + config: + service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}' + launcher_image: '{{ include "nmp-core.image" . }}' + storage: + pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}' + additional_volumes: + - name: nmp-additional-volume + persistent_volume_claim: + claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + additional_volume_mounts: + - name: nmp-additional-volume + mount_path: /mnt/additional_storage models: controller: backends: diff --git a/e2e/k8s/values/minikube-auth.yaml b/e2e/k8s/values/minikube-auth.yaml index 713fc3a23e..798997dc08 100644 --- a/e2e/k8s/values/minikube-auth.yaml +++ b/e2e/k8s/values/minikube-auth.yaml @@ -38,6 +38,24 @@ platformConfig: admin_email: "admin@example.com" inference_gateway: mock_provider_prefix: igw-mock- + jobs: + executors: + - provider: cpu + profile: additional-volume + backend: kubernetes_job + config: + service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}' + launcher_image: '{{ include "nmp-core.image" . }}' + storage: + pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}' + additional_volumes: + - name: nmp-additional-volume + persistent_volume_claim: + claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + additional_volume_mounts: + - name: nmp-additional-volume + mount_path: /mnt/additional_storage models: controller: backends: diff --git a/e2e/k8s/values/minikube.yaml b/e2e/k8s/values/minikube.yaml index 0f48a4f718..33c97acc1f 100644 --- a/e2e/k8s/values/minikube.yaml +++ b/e2e/k8s/values/minikube.yaml @@ -27,5 +27,23 @@ ingress: platformConfig: inference_gateway: mock_provider_prefix: igw-mock- + jobs: + executors: + - provider: cpu + profile: additional-volume + backend: kubernetes_job + config: + service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}' + launcher_image: '{{ include "nmp-core.image" . }}' + storage: + pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}' + additional_volumes: + - name: nmp-additional-volume + persistent_volume_claim: + claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}' + additional_volume_mounts: + - name: nmp-additional-volume + mount_path: /mnt/additional_storage safe_synthesizer: job_mode: container diff --git a/e2e/test_evaluator_plugin.py b/e2e/test_evaluator_plugin.py index d8f92758be..3c1de977dc 100644 --- a/e2e/test_evaluator_plugin.py +++ b/e2e/test_evaluator_plugin.py @@ -97,6 +97,7 @@ def _add_mock_provider_or_skip( workspace: str, name: str, mock_response_body: dict[str, object], + should_autoprovision_virtual_model: bool = True, ) -> ModelProvider: """Create an IGW mock provider or skip when the deployment does not support one.""" try: @@ -105,6 +106,7 @@ def _add_mock_provider_or_skip( workspace=workspace, name=name, mock_response_body=mock_response_body, + should_autoprovision_virtual_model=should_autoprovision_virtual_model, ) except RuntimeError as exc: if "mock_provider_prefix is not configured" in str(exc): @@ -250,6 +252,7 @@ def _create_ready_mock_model( workspace=workspace, name=name, mock_response_body=mock_response_body, + should_autoprovision_virtual_model=False, ) sdk.models.create( workspace=workspace, @@ -258,8 +261,20 @@ def _create_ready_mock_model( model_providers=[f"{workspace}/{provider.name}"], exist_ok=True, ) - wait_for_model_entity(sdk, workspace, name, ensure_virtual_model=True) - ensure_passthrough_virtual_model(sdk, workspace, name, timeout=IGW_ROUTE_TIMEOUT_SECONDS) + wait_for_model_entity( + sdk, + workspace, + name, + ensure_virtual_model=True, + should_autoprovision_virtual_model=False, + ) + ensure_passthrough_virtual_model( + sdk, + workspace, + name, + timeout=IGW_ROUTE_TIMEOUT_SECONDS, + autoprovisioned=False, + ) _wait_for_stable_model_chat_route(sdk, workspace, name) diff --git a/e2e/test_jobs.py b/e2e/test_jobs.py index c14bca3df2..46c401d24c 100644 --- a/e2e/test_jobs.py +++ b/e2e/test_jobs.py @@ -16,7 +16,10 @@ from nemo_platform_plugin.jobs.constants import DEFAULT_JOB_STORAGE_PATH from nmp.testing.e2e import wait_for_job_logs, wait_for_platform_job +from e2e.services_pool import RunningServices + JOB_SOURCE = "e2e-test-jobs" +ADDITIONAL_VOLUME_PROFILE = "additional-volume" pytestmark = [ pytest.mark.timeout(600), @@ -449,9 +452,15 @@ def test_job_pause_and_cancel(sdk: NeMoPlatform, workspace: str): assert cancelled_job.status == "cancelled", f"Job should have been cancelled but has status: {cancelled_job.status}" -@pytest.mark.skip(reason="Requires additional_volumes configured in Helm chart storage config") -def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str): +def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str, _services_instance: RunningServices): """Test that a job can use an additional volume to store data between steps.""" + if _services_instance.config_path is not None and _services_instance.docker_network_name is None: + pytest.skip("Requires a container-backed platform with /mnt/additional_storage mounted") + + # Kubernetes e2e runs use a dedicated profile so the extra PVC mount does + # not affect unrelated jobs that also request persistent job storage. + profile = ADDITIONAL_VOLUME_PROFILE if _services_instance.config_path is None else "default" + job = sdk.jobs.create( workspace=workspace, source=JOB_SOURCE, @@ -462,6 +471,7 @@ def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str): "name": "write-data", "executor": { "provider": "cpu", + "profile": profile, "container": { "command": [ "sh", @@ -476,6 +486,7 @@ def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str): "name": "read-data", "executor": { "provider": "cpu", + "profile": profile, "container": { "command": [ "sh", diff --git a/e2e/test_safe_synthesizer.py b/e2e/test_safe_synthesizer.py index d16eb04969..ad563f5a7b 100644 --- a/e2e/test_safe_synthesizer.py +++ b/e2e/test_safe_synthesizer.py @@ -17,17 +17,18 @@ from __future__ import annotations import csv +import importlib.util import io import json import os -import subprocess import sys import time import uuid from collections.abc import Callable, Iterator from contextlib import suppress from pathlib import Path -from typing import Any +from types import ModuleType +from typing import Any, cast import pytest from nemo_platform import NeMoPlatform @@ -51,7 +52,6 @@ K8S_JOB_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_JOB_TIMEOUT_SECONDS", "5400")) POLL_INTERVAL_SECONDS = float(os.environ.get("NSS_E2E_POLL_INTERVAL_SECONDS", "10")) RESULT_DOWNLOAD_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_RESULT_DOWNLOAD_TIMEOUT_SECONDS", "600")) -MODEL_FILESETS_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_MODEL_FILESETS_TIMEOUT_SECONDS", "300")) DELETE_VERIFY_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_DELETE_VERIFY_TIMEOUT_SECONDS", "60")) DEFAULT_INPUT_ROWS = int(os.environ.get("NSS_E2E_INPUT_ROWS", "250")) DEFAULT_NUM_RECORDS = int(os.environ.get("NSS_E2E_NUM_RECORDS", "250")) @@ -344,14 +344,6 @@ def _assert_known_pii_replaced(content: bytes) -> None: assert source_value not in text -def _process_output_text(output: str | bytes | None) -> str: - if output is None: - return "" - if isinstance(output, bytes): - return output.decode("utf-8", errors="replace") - return output - - def _platform_root() -> Path: candidates: list[Path] = [] if os.environ.get("NMP_PLATFORM_ROOT"): @@ -376,39 +368,31 @@ def nss_model_filesets(sdk: NeMoPlatform) -> None: return platform_root = _platform_root() - script = platform_root / "plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py" - try: - result = subprocess.run( - [ - "uv", - "run", - "--project", - str(platform_root), - "python", - str(script), - "--files-api-url", - str(sdk.base_url).rstrip("/"), - "--workspace", - "default", - ], - cwd=platform_root, - timeout=MODEL_FILESETS_TIMEOUT_SECONDS, - check=False, - capture_output=True, - text=True, - ) - except subprocess.TimeoutExpired as exc: - pytest.fail( - f"Timed out after {MODEL_FILESETS_TIMEOUT_SECONDS:g}s registering Safe Synthesizer model filesets\n" - f"stdout:\n{_process_output_text(exc.stdout)}\n" - f"stderr:\n{_process_output_text(exc.stderr)}" - ) - if result.returncode != 0: + setup_module = _load_model_filesets_setup_module(platform_root) + create_filesets = cast( + Callable[[NeMoPlatform, str, bool], list[str]], + getattr(setup_module, "create_filesets"), + ) + model_filesets = cast(list[dict[str, object]], getattr(setup_module, "MODEL_FILESETS")) + + created = create_filesets(sdk, "default", False) + if len(created) != len(model_filesets): pytest.fail( - f"Failed to register Safe Synthesizer model filesets\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + "Failed to register all Safe Synthesizer model filesets: " + f"registered {len(created)} of {len(model_filesets)} ({', '.join(created)})" ) +def _load_model_filesets_setup_module(platform_root: Path) -> ModuleType: + setup_script = platform_root / "plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py" + spec = importlib.util.spec_from_file_location("nss_setup_model_filesets", setup_script) + if spec is None or spec.loader is None: + pytest.fail(f"Failed to load Safe Synthesizer model fileset setup script: {setup_script}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + @pytest.fixture def nss_dataset(sdk: NeMoPlatform, workspace: str) -> Iterator[tuple[str, str]]: fileset, data_source = _upload_dataset(sdk, workspace)