From 21623ed390e081298afa67a5f6a4dd50cd34ab4a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 8 Jun 2026 15:15:32 -0700 Subject: [PATCH 1/5] feat(e2e): Add files external provider tests Signed-off-by: Matthew Grossman --- .github/workflows/ci.yaml | 2 + e2e/{ => files}/test_files.py | 0 e2e/files/test_storage_backends.py | 225 +++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+) rename e2e/{ => files}/test_files.py (100%) create mode 100644 e2e/files/test_storage_backends.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9d0534c805c..6b29c75710c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -338,6 +338,8 @@ jobs: env: _TYPER_FORCE_DISABLE_TERMINAL: "1" E2E_SERVICES_LOG: ${{ runner.temp }}/services.log + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} - name: Dump server logs if: always() run: | diff --git a/e2e/test_files.py b/e2e/files/test_files.py similarity index 100% rename from e2e/test_files.py rename to e2e/files/test_files.py diff --git a/e2e/files/test_storage_backends.py b/e2e/files/test_storage_backends.py new file mode 100644 index 00000000000..b55c238aebb --- /dev/null +++ b/e2e/files/test_storage_backends.py @@ -0,0 +1,225 @@ +"""E2E tests for external storage backends (NGC, Hugging Face). + +These tests verify that the files service can create filesets backed by +external storage providers and read files from them via the SDK. + +NGC tests require ``NGC_API_KEY`` in the environment and are skipped +otherwise. Hugging Face tests use a small public repo; when ``HF_TOKEN`` +is set the request is authenticated (avoids rate-limits in CI). +""" + +import os +import tempfile +import uuid +from collections.abc import Iterator +from pathlib import Path + +import pytest +from nemo_platform import NeMoPlatform +from nemo_platform.types.files import HuggingfaceStorageConfigParam, NGCStorageConfigParam + +# --------------------------------------------------------------------------- +# NGC configuration +# --------------------------------------------------------------------------- +NGC_API_KEY_ENV = "NGC_API_KEY" + +NGC_ORG = "nvidian" +NGC_TEAM = "nemo-llm" +NGC_TARGET = "nemo-platform-quickstart" +NGC_TARGET_TYPE = "resource" + +# --------------------------------------------------------------------------- +# Hugging Face configuration — small public model +# --------------------------------------------------------------------------- +HF_TOKEN_ENV = "HF_TOKEN" + +HF_REPO_ID = "hf-internal-testing/tiny-random-bert" +HF_REPO_TYPE = "model" + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def ngc_api_key() -> str: + """Return the NGC API key from the environment.""" + key = os.environ.get(NGC_API_KEY_ENV) + assert key, f"{NGC_API_KEY_ENV} must be set" + return key + + +@pytest.fixture +def ngc_secret(sdk: NeMoPlatform, workspace: str, ngc_api_key: str) -> Iterator[str]: + """Create a secret containing the NGC API key, cleaned up after test.""" + secret_name = f"e2e-ngc-key-{uuid.uuid4().hex[:8]}" + sdk.secrets.create(workspace=workspace, name=secret_name, value=ngc_api_key) + yield secret_name + try: + sdk.secrets.delete(workspace=workspace, name=secret_name) + except Exception: + pass + + +@pytest.fixture +def ngc_fileset(sdk: NeMoPlatform, workspace: str, ngc_secret: str) -> Iterator[str]: + """Create an NGC-backed fileset, cleaned up after test.""" + fileset_name = f"e2e-ngc-fs-{uuid.uuid4().hex[:8]}" + sdk.files.filesets.create( + workspace=workspace, + name=fileset_name, + description="E2E test NGC-backed fileset", + storage=NGCStorageConfigParam( + api_key_secret=ngc_secret, + org=NGC_ORG, + team=NGC_TEAM, + target=NGC_TARGET, + target_type=NGC_TARGET_TYPE, + ), + ) + yield fileset_name + try: + sdk.files.filesets.delete(fileset_name, workspace=workspace) + except Exception: + pass + + +@pytest.fixture +def hf_secret(sdk: NeMoPlatform, workspace: str) -> Iterator[str | None]: + """Create a secret for the HF token if available, otherwise yield None.""" + token = os.environ.get(HF_TOKEN_ENV) + if not token: + yield None + return + + secret_name = f"e2e-hf-tok-{uuid.uuid4().hex[:8]}" + sdk.secrets.create(workspace=workspace, name=secret_name, value=token) + yield secret_name + try: + sdk.secrets.delete(workspace=workspace, name=secret_name) + except Exception: + pass + + +@pytest.fixture +def hf_fileset(sdk: NeMoPlatform, workspace: str, hf_secret: str | None) -> Iterator[str]: + """Create a Hugging Face-backed fileset, cleaned up after test.""" + fileset_name = f"e2e-hf-fs-{uuid.uuid4().hex[:8]}" + + storage = HuggingfaceStorageConfigParam( + repo_id=HF_REPO_ID, + repo_type=HF_REPO_TYPE, + ) + if hf_secret is not None: + storage["token_secret"] = hf_secret + + sdk.files.filesets.create( + workspace=workspace, + name=fileset_name, + description="E2E test HF-backed fileset", + storage=storage, + ) + yield fileset_name + try: + sdk.files.filesets.delete(fileset_name, workspace=workspace) + except Exception: + pass + + +# =================================================================== +# NGC tests +# =================================================================== + + +@pytest.mark.skipif(not os.environ.get(NGC_API_KEY_ENV), reason=f"{NGC_API_KEY_ENV} not set") +class TestNGCFileset: + """Tests for NGC-backed filesets.""" + + def test_list_files(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str): + """Listing an NGC-backed fileset returns files with paths and sizes.""" + files = sdk.files.list(fileset=ngc_fileset, workspace=workspace) + assert len(files.data) > 0, "NGC fileset should contain at least one file" + + for f in files.data: + assert f.path, "Each file should have a path" + assert f.size > 0, "Each file should have a non-zero size" + + def test_download_file(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str): + """Downloading the smallest file from an NGC fileset succeeds and size matches.""" + files = sdk.files.list(fileset=ngc_fileset, workspace=workspace) + assert len(files.data) > 0 + + target = min(files.data, key=lambda f: f.size) + + with tempfile.TemporaryDirectory() as tmpdir: + local_path = Path(tmpdir) / target.path.replace("/", "_") + sdk.files.download( + fileset=ngc_fileset, + workspace=workspace, + remote_path=target.path, + local_path=str(local_path), + ) + assert local_path.exists() + assert local_path.stat().st_size == target.size + + def test_cache_status(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str): + """NGC-backed files report a cacheable status.""" + files = sdk.files.list( + fileset=ngc_fileset, + workspace=workspace, + include_cache_status=True, + ) + assert len(files.data) > 0 + + for f in files.data: + assert f.cache_status is not None + assert f.cache_status != "not_cacheable" + + +# =================================================================== +# Hugging Face tests +# =================================================================== + + +class TestHuggingFaceFileset: + """Tests for Hugging Face-backed filesets.""" + + def test_list_files(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str): + """Listing an HF-backed fileset returns files with paths and sizes.""" + files = sdk.files.list(fileset=hf_fileset, workspace=workspace) + assert len(files.data) > 0, "HF fileset should contain at least one file" + + for f in files.data: + assert f.path, "Each file should have a path" + assert f.size > 0, "Each file should have a non-zero size" + + def test_download_file(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str): + """Downloading the smallest file from an HF fileset succeeds and size matches.""" + files = sdk.files.list(fileset=hf_fileset, workspace=workspace) + assert len(files.data) > 0 + + target = min(files.data, key=lambda f: f.size) + + with tempfile.TemporaryDirectory() as tmpdir: + local_path = Path(tmpdir) / target.path.replace("/", "_") + sdk.files.download( + fileset=hf_fileset, + workspace=workspace, + remote_path=target.path, + local_path=str(local_path), + ) + assert local_path.exists() + assert local_path.stat().st_size == target.size + + def test_cache_status(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str): + """HF-backed files report a cacheable status.""" + files = sdk.files.list( + fileset=hf_fileset, + workspace=workspace, + include_cache_status=True, + ) + assert len(files.data) > 0 + + for f in files.data: + assert f.cache_status is not None + assert f.cache_status != "not_cacheable" From f06aac988410dc056261ed75b18f17c1d023aa0e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 8 Jun 2026 16:40:29 -0700 Subject: [PATCH 2/5] fix tests Signed-off-by: Matthew Grossman --- .github/workflows/ci.yaml | 2 +- e2e/conftest.py | 54 ++++++++++- e2e/files/test_storage_backends.py | 92 ++++++++++++++++++- .../src/nmp/core/files/app/backends/ngc.py | 15 +-- 4 files changed, 149 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b29c75710c..109b1f70d3c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -338,7 +338,7 @@ jobs: env: _TYPER_FORCE_DISABLE_TERMINAL: "1" E2E_SERVICES_LOG: ${{ runner.temp }}/services.log - NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + NGC_API_KEY: ${{ secrets.NGC_REGISTRY_READ_TOKEN }} HF_TOKEN: ${{ secrets.HF_TOKEN }} - name: Dump server logs if: always() diff --git a/e2e/conftest.py b/e2e/conftest.py index f9de40bfc24..e4b96b3046e 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -54,7 +54,26 @@ def pytest_configure(config: pytest.Config) -> None: _HEALTH_TIMEOUT = 60 _HEALTH_POLL_INTERVAL = 1.0 -_SERVICES_LOG = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log"))) + +# Number of log lines to dump from the services log on test failure. +_TAIL_LINES_ON_FAILURE = 100 + +_services_log_key = pytest.StashKey[Path]() + + +@pytest.fixture(scope="session") +def services_log_path(request: pytest.FixtureRequest) -> Path: + """Return a unique services log path for this session. + + Appends a UUID so parallel pytest processes never clobber each + other's logs. The path is stashed on the session so the + ``pytest_runtest_makereport`` hook can read it without requesting + the fixture. + """ + base = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log"))) + path = base.with_stem(f"{base.stem}-{uuid.uuid4().hex[:8]}") + request.session.stash[_services_log_key] = path + return path def _find_free_port() -> int: @@ -99,8 +118,37 @@ def background_process(args: list[str], stdout: IO[Any] | None = None) -> Iterat proc.wait(timeout=5) +# ---- Services log tail on failure ------------------------------------------ + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): # noqa: ARG001 + """Append the services log tail to the report when a test fails. + + This hook is the pytest-sanctioned way to add extra sections to test + reports (``report.sections``). Fixtures cannot do this because they + don't have access to the report object. + """ + outcome = yield + report = outcome.get_result() + + if not report.failed: + return + + log_path = item.session.stash.get(_services_log_key, None) + if log_path and log_path.exists(): + lines = log_path.read_text().splitlines(keepends=True) + tail = lines[-_TAIL_LINES_ON_FAILURE:] + if tail: + header = f"--- services log (last {len(tail)} lines) [{log_path}] ---" + report.sections.append(("Services Log", f"{header}\n{''.join(tail)}")) + + +# ---- Fixtures -------------------------------------------------------------- + + @pytest.fixture(scope="session") -def _services() -> Iterator[str]: +def _services(services_log_path: Path) -> Iterator[str]: """Spawn ``nemo services run`` and yield the base URL. Skipped when ``NMP_BASE_URL`` is already set (external services). @@ -124,7 +172,7 @@ def _services() -> Iterator[str]: logger.info("Starting nemo services on port %d", port) - log_path = _SERVICES_LOG + log_path = services_log_path with open(log_path, "w") as log_file, background_process(args, stdout=log_file) as proc: if not _wait_for_healthy(url): pytest.fail( diff --git a/e2e/files/test_storage_backends.py b/e2e/files/test_storage_backends.py index b55c238aebb..3a679f381e3 100644 --- a/e2e/files/test_storage_backends.py +++ b/e2e/files/test_storage_backends.py @@ -15,7 +15,7 @@ from pathlib import Path import pytest -from nemo_platform import NeMoPlatform +from nemo_platform import BadRequestError, NeMoPlatform from nemo_platform.types.files import HuggingfaceStorageConfigParam, NGCStorageConfigParam # --------------------------------------------------------------------------- @@ -23,9 +23,9 @@ # --------------------------------------------------------------------------- NGC_API_KEY_ENV = "NGC_API_KEY" -NGC_ORG = "nvidian" -NGC_TEAM = "nemo-llm" -NGC_TARGET = "nemo-platform-quickstart" +NGC_ORG = "nvidia" +NGC_TEAM = "nemo-microservices" +NGC_TARGET = "nemo-microservices-quickstart" NGC_TARGET_TYPE = "resource" # --------------------------------------------------------------------------- @@ -175,6 +175,76 @@ def test_cache_status(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str) assert f.cache_status is not None assert f.cache_status != "not_cacheable" + # -- error cases -- + + @pytest.mark.parametrize( + ("secret_value", "storage_overrides", "match"), + [ + pytest.param( + "not-a-real-key", + {}, + "Invalid API key. Legacy NGC keys are not supported.", + id="invalid-key-prefix", + ), + pytest.param( + None, + {"org": "nvidian", "team": "nemo-llm", "target": "nemo-platform-quickstart"}, + "Error creating NGC storage backend:", + id="wrong-org", + ), + pytest.param( + None, + {"target": "this-resource-does-not-exist-12345"}, + "Failed to access NGC resource this-resource-does-not-exist-12345", + id="nonexistent-resource", + ), + ], + ) + def test_create_error( + self, + sdk: NeMoPlatform, + workspace: str, + ngc_api_key: str, + secret_value: str | None, + storage_overrides: dict, + match: str, + ): + """Bad NGC configurations are rejected with 400.""" + value = secret_value if secret_value is not None else ngc_api_key + secret_name = f"e2e-ngc-err-{uuid.uuid4().hex[:8]}" + sdk.secrets.create(workspace=workspace, name=secret_name, value=value) + try: + storage = NGCStorageConfigParam( + api_key_secret=secret_name, + org=storage_overrides.get("org", NGC_ORG), + team=storage_overrides.get("team", NGC_TEAM), + target=storage_overrides.get("target", NGC_TARGET), + target_type=NGC_TARGET_TYPE, + ) + with pytest.raises(BadRequestError, match=match): + sdk.files.filesets.create( + workspace=workspace, + name=f"e2e-ngc-err-{uuid.uuid4().hex[:8]}", + storage=storage, + ) + finally: + sdk.secrets.delete(workspace=workspace, name=secret_name) + + def test_create_error_nonexistent_secret(self, sdk: NeMoPlatform, workspace: str): + """Referencing a secret that doesn't exist is rejected with 400.""" + with pytest.raises(BadRequestError, match="Secret not found:"): + sdk.files.filesets.create( + workspace=workspace, + name=f"e2e-ngc-err-{uuid.uuid4().hex[:8]}", + storage=NGCStorageConfigParam( + api_key_secret="no-such-secret-99999", + org=NGC_ORG, + team=NGC_TEAM, + target=NGC_TARGET, + target_type=NGC_TARGET_TYPE, + ), + ) + # =================================================================== # Hugging Face tests @@ -223,3 +293,17 @@ def test_cache_status(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str): for f in files.data: assert f.cache_status is not None assert f.cache_status != "not_cacheable" + + # -- error cases -- + + def test_error_nonexistent_repo(self, sdk: NeMoPlatform, workspace: str): + """Pointing at a repo that doesn't exist is rejected with 400.""" + with pytest.raises(BadRequestError): + sdk.files.filesets.create( + workspace=workspace, + name=f"e2e-hf-err-{uuid.uuid4().hex[:8]}", + storage=HuggingfaceStorageConfigParam( + repo_id="this-org-does-not-exist/this-repo-does-not-exist-12345", + repo_type="model", + ), + ) diff --git a/services/core/files/src/nmp/core/files/app/backends/ngc.py b/services/core/files/src/nmp/core/files/app/backends/ngc.py index 06e6bcad26c..f01c1f138c9 100644 --- a/services/core/files/src/nmp/core/files/app/backends/ngc.py +++ b/services/core/files/src/nmp/core/files/app/backends/ngc.py @@ -15,7 +15,7 @@ import aiohttp from anyio import to_thread from ngcbase.constants import SCOPED_KEY_PREFIX -from ngcbase.errors import ResourceNotFoundException +from ngcbase.errors import NgcException, ResourceNotFoundException from ngcsdk import Client from nmp.common.files.storage_config import NGCStorageConfig as NGCStorageConfig from nmp.core.files.app.backends.base import ( @@ -91,7 +91,7 @@ def _configure() -> Client: try: self._client = await to_thread.run_sync(_configure) - except ValueError as exc: + except (ValueError, NgcException) as exc: raise NGCBackendError(f"Error creating NGC storage backend: [{str(exc)}]") from exc return self._client @@ -264,15 +264,18 @@ async def validate_storage(self): """Validate that we can access the NGC asset.""" validate_external_host(self.config.host) - registry_api = await self._get_registry_api() - target = await self._get_target() - target_with_version = await self._get_target_with_version() - try: + registry_api = await self._get_registry_api() + target = await self._get_target() + target_with_version = await self._get_target_with_version() await to_thread.run_sync(registry_api.info, target) + except NGCBackendError: + raise except ResourceNotFoundException as exc: + target_with_version = f"{self.config.target}" raise NGCBackendError(f"NGC {self.config.target_type} not found: {target_with_version}") from exc except Exception as exc: + target_with_version = f"{self.config.target}" raise NGCBackendError( f"Failed to access NGC {self.config.target_type} {target_with_version} [{str(exc)}]" ) from exc From 8c12da8320f371966d753f894ed6929178dc9911 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 8 Jun 2026 16:46:35 -0700 Subject: [PATCH 3/5] fix tests Signed-off-by: Matthew Grossman --- e2e/files/test_storage_backends.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/files/test_storage_backends.py b/e2e/files/test_storage_backends.py index 3a679f381e3..a49f14dcfad 100644 --- a/e2e/files/test_storage_backends.py +++ b/e2e/files/test_storage_backends.py @@ -58,7 +58,7 @@ def ngc_secret(sdk: NeMoPlatform, workspace: str, ngc_api_key: str) -> Iterator[ try: sdk.secrets.delete(workspace=workspace, name=secret_name) except Exception: - pass + pass # Best-effort cleanup; the workspace is deleted anyway @pytest.fixture @@ -81,7 +81,7 @@ def ngc_fileset(sdk: NeMoPlatform, workspace: str, ngc_secret: str) -> Iterator[ try: sdk.files.filesets.delete(fileset_name, workspace=workspace) except Exception: - pass + pass # Best-effort cleanup; the workspace is deleted anyway @pytest.fixture @@ -98,7 +98,7 @@ def hf_secret(sdk: NeMoPlatform, workspace: str) -> Iterator[str | None]: try: sdk.secrets.delete(workspace=workspace, name=secret_name) except Exception: - pass + pass # Best-effort cleanup; the workspace is deleted anyway @pytest.fixture @@ -123,7 +123,7 @@ def hf_fileset(sdk: NeMoPlatform, workspace: str, hf_secret: str | None) -> Iter try: sdk.files.filesets.delete(fileset_name, workspace=workspace) except Exception: - pass + pass # Best-effort cleanup; the workspace is deleted anyway # =================================================================== From 63a6b8947801c1ec7d7345de2e5d82b249b63bea Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 8 Jun 2026 17:09:24 -0700 Subject: [PATCH 4/5] fix Signed-off-by: Matthew Grossman --- services/core/files/src/nmp/core/files/app/backends/ngc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/core/files/src/nmp/core/files/app/backends/ngc.py b/services/core/files/src/nmp/core/files/app/backends/ngc.py index f01c1f138c9..a081f6affdf 100644 --- a/services/core/files/src/nmp/core/files/app/backends/ngc.py +++ b/services/core/files/src/nmp/core/files/app/backends/ngc.py @@ -267,17 +267,14 @@ async def validate_storage(self): try: registry_api = await self._get_registry_api() target = await self._get_target() - target_with_version = await self._get_target_with_version() await to_thread.run_sync(registry_api.info, target) except NGCBackendError: raise except ResourceNotFoundException as exc: - target_with_version = f"{self.config.target}" - raise NGCBackendError(f"NGC {self.config.target_type} not found: {target_with_version}") from exc + raise NGCBackendError(f"NGC {self.config.target_type} not found: {self.config.target}") from exc except Exception as exc: - target_with_version = f"{self.config.target}" raise NGCBackendError( - f"Failed to access NGC {self.config.target_type} {target_with_version} [{str(exc)}]" + f"Failed to access NGC {self.config.target_type} {self.config.target} [{exc}]" ) from exc async def upload( From ab229a020f1b5b659d4ea58c0060765d3dc9f706 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 9 Jun 2026 08:59:55 -0700 Subject: [PATCH 5/5] fix logging dir Signed-off-by: Matthew Grossman --- .github/workflows/ci.yaml | 10 ++++++---- e2e/conftest.py | 21 +++++++++++++++------ e2e/files/test_storage_backends.py | 21 ++++++++++++--------- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 109b1f70d3c..066fe33aacc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -337,14 +337,16 @@ jobs: run: make test-e2e env: _TYPER_FORCE_DISABLE_TERMINAL: "1" - E2E_SERVICES_LOG: ${{ runner.temp }}/services.log + E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs NGC_API_KEY: ${{ secrets.NGC_REGISTRY_READ_TOKEN }} HF_TOKEN: ${{ secrets.HF_TOKEN }} - name: Dump server logs if: always() run: | - echo "::group::Server log" - cat "${{ runner.temp }}/services.log" 2>/dev/null || echo "No server log found" + echo "::group::Server logs" + for f in "${{ runner.temp }}/e2e-services-logs"/*.log; do + [ -f "$f" ] && echo "--- $(basename "$f") ---" && cat "$f" || echo "No server logs found" + done echo "::endgroup::" - name: Upload test artifacts if: always() @@ -354,7 +356,7 @@ jobs: retention-days: 30 path: | report.xml - ${{ runner.temp }}/services.log + ${{ runner.temp }}/e2e-services-logs/ # Required-check pin: branch protection should reference this aggregator # rather than the per-row matrix jobs, so the matrix can grow or shrink diff --git a/e2e/conftest.py b/e2e/conftest.py index e4b96b3046e..a42a9741fd0 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -23,7 +23,6 @@ import socket import subprocess import sys -import tempfile import time import uuid from collections.abc import Iterator @@ -62,16 +61,26 @@ def pytest_configure(config: pytest.Config) -> None: @pytest.fixture(scope="session") -def services_log_path(request: pytest.FixtureRequest) -> Path: +def services_log_path(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Path: """Return a unique services log path for this session. - Appends a UUID so parallel pytest processes never clobber each - other's logs. The path is stashed on the session so the + ``E2E_SERVICES_LOG_DIR`` (if set) is treated as a **directory**; in CI + the job uploads everything under it as artifacts. When unset we + fall back to a pytest-managed temp directory. Either way, each + session writes to a UUID-named file inside the directory so + parallel workers never clobber each other. + + The path is stashed on the session so the ``pytest_runtest_makereport`` hook can read it without requesting the fixture. """ - base = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log"))) - path = base.with_stem(f"{base.stem}-{uuid.uuid4().hex[:8]}") + log_dir = os.environ.get("E2E_SERVICES_LOG_DIR") + if log_dir: + directory = Path(log_dir) + directory.mkdir(parents=True, exist_ok=True) + else: + directory = tmp_path_factory.mktemp("e2e-services-logs") + path = directory / f"services-{uuid.uuid4().hex[:8]}.log" request.session.stash[_services_log_key] = path return path diff --git a/e2e/files/test_storage_backends.py b/e2e/files/test_storage_backends.py index a49f14dcfad..39f9125a0ef 100644 --- a/e2e/files/test_storage_backends.py +++ b/e2e/files/test_storage_backends.py @@ -85,15 +85,18 @@ def ngc_fileset(sdk: NeMoPlatform, workspace: str, ngc_secret: str) -> Iterator[ @pytest.fixture -def hf_secret(sdk: NeMoPlatform, workspace: str) -> Iterator[str | None]: - """Create a secret for the HF token if available, otherwise yield None.""" +def hf_token() -> str: + """Return the HF token from the environment.""" token = os.environ.get(HF_TOKEN_ENV) - if not token: - yield None - return + assert token, f"{HF_TOKEN_ENV} must be set" + return token + +@pytest.fixture +def hf_secret(sdk: NeMoPlatform, workspace: str, hf_token: str) -> Iterator[str]: + """Create a secret containing the HF token, cleaned up after test.""" secret_name = f"e2e-hf-tok-{uuid.uuid4().hex[:8]}" - sdk.secrets.create(workspace=workspace, name=secret_name, value=token) + sdk.secrets.create(workspace=workspace, name=secret_name, value=hf_token) yield secret_name try: sdk.secrets.delete(workspace=workspace, name=secret_name) @@ -102,16 +105,15 @@ def hf_secret(sdk: NeMoPlatform, workspace: str) -> Iterator[str | None]: @pytest.fixture -def hf_fileset(sdk: NeMoPlatform, workspace: str, hf_secret: str | None) -> Iterator[str]: +def hf_fileset(sdk: NeMoPlatform, workspace: str, hf_secret: str) -> Iterator[str]: """Create a Hugging Face-backed fileset, cleaned up after test.""" fileset_name = f"e2e-hf-fs-{uuid.uuid4().hex[:8]}" storage = HuggingfaceStorageConfigParam( repo_id=HF_REPO_ID, repo_type=HF_REPO_TYPE, + token_secret=hf_secret, ) - if hf_secret is not None: - storage["token_secret"] = hf_secret sdk.files.filesets.create( workspace=workspace, @@ -251,6 +253,7 @@ def test_create_error_nonexistent_secret(self, sdk: NeMoPlatform, workspace: str # =================================================================== +@pytest.mark.skipif(not os.environ.get(HF_TOKEN_ENV), reason=f"{HF_TOKEN_ENV} not set") class TestHuggingFaceFileset: """Tests for Hugging Face-backed filesets."""