chore: enable volume extra test

e2e/k8s/values/default.yaml

@@ -13,6 +13,24 @@ platformConfig:
   # This is used to enable mock provider mode for E2E tests.
   inference_gateway:
     mock_provider_prefix: igw-mock-
+  jobs:
+    executors:
+      - provider: cpu
+        profile: additional-volume
+        backend: kubernetes_job
+        config:
+          service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}'
+          launcher_image: '{{ include "nmp-core.image" . }}'
+          storage:
+            pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}'
+            additional_volumes:
+              - name: nmp-additional-volume
+                persistent_volume_claim:
+                  claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            additional_volume_mounts:
+              - name: nmp-additional-volume
+                mount_path: /mnt/additional_storage
 
 core:
   storage:

e2e/k8s/values/kind.yaml

@@ -24,6 +24,24 @@ httpRoute:
 platformConfig:
   inference_gateway:
     mock_provider_prefix: igw-mock-
+  jobs:
+    executors:
+      - provider: cpu
+        profile: additional-volume
+        backend: kubernetes_job
+        config:
+          service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}'
+          launcher_image: '{{ include "nmp-core.image" . }}'
+          storage:
+            pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}'
+            additional_volumes:
+              - name: nmp-additional-volume
+                persistent_volume_claim:
+                  claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            additional_volume_mounts:
+              - name: nmp-additional-volume
+                mount_path: /mnt/additional_storage
   models:
     controller:
       backends:

e2e/k8s/values/minikube-auth-portforward.yaml

@@ -32,6 +32,24 @@ platformConfig:
     admin_email: "admin@example.com"
   inference_gateway:
     mock_provider_prefix: igw-mock-
+  jobs:
+    executors:
+      - provider: cpu
+        profile: additional-volume
+        backend: kubernetes_job
+        config:
+          service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}'
+          launcher_image: '{{ include "nmp-core.image" . }}'
+          storage:
+            pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}'
+            additional_volumes:
+              - name: nmp-additional-volume
+                persistent_volume_claim:
+                  claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            additional_volume_mounts:
+              - name: nmp-additional-volume
+                mount_path: /mnt/additional_storage
   models:
     controller:
       backends:

e2e/k8s/values/minikube-auth.yaml

@@ -38,6 +38,24 @@ platformConfig:
     admin_email: "admin@example.com"
   inference_gateway:
     mock_provider_prefix: igw-mock-
+  jobs:
+    executors:
+      - provider: cpu
+        profile: additional-volume
+        backend: kubernetes_job
+        config:
+          service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}'
+          launcher_image: '{{ include "nmp-core.image" . }}'
+          storage:
+            pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}'
+            additional_volumes:
+              - name: nmp-additional-volume
+                persistent_volume_claim:
+                  claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            additional_volume_mounts:
+              - name: nmp-additional-volume
+                mount_path: /mnt/additional_storage
   models:
     controller:
       backends:

e2e/k8s/values/minikube.yaml

@@ -27,5 +27,23 @@ ingress:
 platformConfig:
   inference_gateway:
     mock_provider_prefix: igw-mock-
+  jobs:
+    executors:
+      - provider: cpu
+        profile: additional-volume
+        backend: kubernetes_job
+        config:
+          service_account_name: '{{ include "nmp-core.jobsServiceAccountName" . }}'
+          launcher_image: '{{ include "nmp-core.image" . }}'
+          storage:
+            pvc_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            volume_permissions_image: '{{ .Values.core.storage.volumePermissionsImage }}'
+            additional_volumes:
+              - name: nmp-additional-volume
+                persistent_volume_claim:
+                  claim_name: '{{ include "nmp-core.persistentVolumeClaim" . }}'
+            additional_volume_mounts:
+              - name: nmp-additional-volume
+                mount_path: /mnt/additional_storage
   safe_synthesizer:
     job_mode: container

e2e/test_evaluator_plugin.py

@@ -97,6 +97,7 @@ def _add_mock_provider_or_skip(
     workspace: str,
     name: str,
     mock_response_body: dict[str, object],
+    should_autoprovision_virtual_model: bool = True,
 ) -> ModelProvider:
     """Create an IGW mock provider or skip when the deployment does not support one."""
     try:
@@ -105,6 +106,7 @@ def _add_mock_provider_or_skip(
             workspace=workspace,
             name=name,
             mock_response_body=mock_response_body,
+            should_autoprovision_virtual_model=should_autoprovision_virtual_model,
         )
     except RuntimeError as exc:
         if "mock_provider_prefix is not configured" in str(exc):
@@ -250,6 +252,7 @@ def _create_ready_mock_model(
         workspace=workspace,
         name=name,
         mock_response_body=mock_response_body,
+        should_autoprovision_virtual_model=False,
     )
     sdk.models.create(
         workspace=workspace,
@@ -258,8 +261,20 @@ def _create_ready_mock_model(
         model_providers=[f"{workspace}/{provider.name}"],
         exist_ok=True,
     )
-    wait_for_model_entity(sdk, workspace, name, ensure_virtual_model=True)
-    ensure_passthrough_virtual_model(sdk, workspace, name, timeout=IGW_ROUTE_TIMEOUT_SECONDS)
+    wait_for_model_entity(
+        sdk,
+        workspace,
+        name,
+        ensure_virtual_model=True,
+        should_autoprovision_virtual_model=False,
+    )
+    ensure_passthrough_virtual_model(
+        sdk,
+        workspace,
+        name,
+        timeout=IGW_ROUTE_TIMEOUT_SECONDS,
+        autoprovisioned=False,
+    )
     _wait_for_stable_model_chat_route(sdk, workspace, name)
 
 

e2e/test_jobs.py

@@ -16,7 +16,10 @@
 from nemo_platform_plugin.jobs.constants import DEFAULT_JOB_STORAGE_PATH
 from nmp.testing.e2e import wait_for_job_logs, wait_for_platform_job
 
+from e2e.services_pool import RunningServices
+
 JOB_SOURCE = "e2e-test-jobs"
+ADDITIONAL_VOLUME_PROFILE = "additional-volume"
 
 pytestmark = [
     pytest.mark.timeout(600),
@@ -449,9 +452,15 @@ def test_job_pause_and_cancel(sdk: NeMoPlatform, workspace: str):
     assert cancelled_job.status == "cancelled", f"Job should have been cancelled but has status: {cancelled_job.status}"
 
 
-@pytest.mark.skip(reason="Requires additional_volumes configured in Helm chart storage config")
-def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str):
+def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str, _services_instance: RunningServices):
     """Test that a job can use an additional volume to store data between steps."""
+    if _services_instance.config_path is not None and _services_instance.docker_network_name is None:
+        pytest.skip("Requires a container-backed platform with /mnt/additional_storage mounted")
+
+    # Kubernetes e2e runs use a dedicated profile so the extra PVC mount does
+    # not affect unrelated jobs that also request persistent job storage.
+    profile = ADDITIONAL_VOLUME_PROFILE if _services_instance.config_path is None else "default"
+
     job = sdk.jobs.create(
         workspace=workspace,
         source=JOB_SOURCE,
@@ -462,6 +471,7 @@ def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str):
                     "name": "write-data",
                     "executor": {
                         "provider": "cpu",
+                        "profile": profile,
                         "container": {
                             "command": [
                                 "sh",
@@ -476,6 +486,7 @@ def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str):
                     "name": "read-data",
                     "executor": {
                         "provider": "cpu",
+                        "profile": profile,
                         "container": {
                             "command": [
                                 "sh",

e2e/test_safe_synthesizer.py

@@ -17,17 +17,18 @@
 from __future__ import annotations
 
 import csv
+import importlib.util
 import io
 import json
 import os
-import subprocess
 import sys
 import time
 import uuid
 from collections.abc import Callable, Iterator
 from contextlib import suppress
 from pathlib import Path
-from typing import Any
+from types import ModuleType
+from typing import Any, cast
 
 import pytest
 from nemo_platform import NeMoPlatform
@@ -51,7 +52,6 @@
 K8S_JOB_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_JOB_TIMEOUT_SECONDS", "5400"))
 POLL_INTERVAL_SECONDS = float(os.environ.get("NSS_E2E_POLL_INTERVAL_SECONDS", "10"))
 RESULT_DOWNLOAD_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_RESULT_DOWNLOAD_TIMEOUT_SECONDS", "600"))
-MODEL_FILESETS_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_MODEL_FILESETS_TIMEOUT_SECONDS", "300"))
 DELETE_VERIFY_TIMEOUT_SECONDS = float(os.environ.get("NSS_E2E_DELETE_VERIFY_TIMEOUT_SECONDS", "60"))
 DEFAULT_INPUT_ROWS = int(os.environ.get("NSS_E2E_INPUT_ROWS", "250"))
 DEFAULT_NUM_RECORDS = int(os.environ.get("NSS_E2E_NUM_RECORDS", "250"))
@@ -344,14 +344,6 @@ def _assert_known_pii_replaced(content: bytes) -> None:
         assert source_value not in text
 
 
-def _process_output_text(output: str | bytes | None) -> str:
-    if output is None:
-        return ""
-    if isinstance(output, bytes):
-        return output.decode("utf-8", errors="replace")
-    return output
-
-
 def _platform_root() -> Path:
     candidates: list[Path] = []
     if os.environ.get("NMP_PLATFORM_ROOT"):
@@ -376,39 +368,31 @@ def nss_model_filesets(sdk: NeMoPlatform) -> None:
         return
 
     platform_root = _platform_root()
-    script = platform_root / "plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py"
-    try:
-        result = subprocess.run(
-            [
-                "uv",
-                "run",
-                "--project",
-                str(platform_root),
-                "python",
-                str(script),
-                "--files-api-url",
-                str(sdk.base_url).rstrip("/"),
-                "--workspace",
-                "default",
-            ],
-            cwd=platform_root,
-            timeout=MODEL_FILESETS_TIMEOUT_SECONDS,
-            check=False,
-            capture_output=True,
-            text=True,
-        )
-    except subprocess.TimeoutExpired as exc:
-        pytest.fail(
-            f"Timed out after {MODEL_FILESETS_TIMEOUT_SECONDS:g}s registering Safe Synthesizer model filesets\n"
-            f"stdout:\n{_process_output_text(exc.stdout)}\n"
-            f"stderr:\n{_process_output_text(exc.stderr)}"
-        )
-    if result.returncode != 0:
+    setup_module = _load_model_filesets_setup_module(platform_root)
+    create_filesets = cast(
+        Callable[[NeMoPlatform, str, bool], list[str]],
+        getattr(setup_module, "create_filesets"),
+    )
+    model_filesets = cast(list[dict[str, object]], getattr(setup_module, "MODEL_FILESETS"))
+
+    created = create_filesets(sdk, "default", False)
+    if len(created) != len(model_filesets):
         pytest.fail(
-            f"Failed to register Safe Synthesizer model filesets\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+            "Failed to register all Safe Synthesizer model filesets: "
+            f"registered {len(created)} of {len(model_filesets)} ({', '.join(created)})"
         )
 
 
+def _load_model_filesets_setup_module(platform_root: Path) -> ModuleType:
+    setup_script = platform_root / "plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py"
+    spec = importlib.util.spec_from_file_location("nss_setup_model_filesets", setup_script)
+    if spec is None or spec.loader is None:
+        pytest.fail(f"Failed to load Safe Synthesizer model fileset setup script: {setup_script}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 @pytest.fixture
 def nss_dataset(sdk: NeMoPlatform, workspace: str) -> Iterator[tuple[str, str]]:
     fileset, data_source = _upload_dataset(sdk, workspace)