Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,8 @@ jobs:
env:
_TYPER_FORCE_DISABLE_TERMINAL: "1"
E2E_SERVICES_LOG: ${{ runner.temp }}/services.log
NGC_API_KEY: ${{ secrets.NGC_REGISTRY_READ_TOKEN }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Dump server logs
if: always()
run: |
Expand Down
54 changes: 51 additions & 3 deletions e2e/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,26 @@ def pytest_configure(config: pytest.Config) -> None:

_HEALTH_TIMEOUT = 60
_HEALTH_POLL_INTERVAL = 1.0
_SERVICES_LOG = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log")))

# Number of log lines to dump from the services log on test failure.
_TAIL_LINES_ON_FAILURE = 100

_services_log_key = pytest.StashKey[Path]()


@pytest.fixture(scope="session")
def services_log_path(request: pytest.FixtureRequest) -> Path:
"""Return a unique services log path for this session.

Appends a UUID so parallel pytest processes never clobber each
other's logs. The path is stashed on the session so the
``pytest_runtest_makereport`` hook can read it without requesting
the fixture.
"""
base = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log")))
path = base.with_stem(f"{base.stem}-{uuid.uuid4().hex[:8]}")
request.session.stash[_services_log_key] = path
Comment thread
matthewgrossman marked this conversation as resolved.
Resolved
return path


def _find_free_port() -> int:
Expand Down Expand Up @@ -99,8 +118,37 @@ def background_process(args: list[str], stdout: IO[Any] | None = None) -> Iterat
proc.wait(timeout=5)


# ---- Services log tail on failure ------------------------------------------


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): # noqa: ARG001
"""Append the services log tail to the report when a test fails.

This hook is the pytest-sanctioned way to add extra sections to test
reports (``report.sections``). Fixtures cannot do this because they
don't have access to the report object.
"""
outcome = yield
Comment thread
matthewgrossman marked this conversation as resolved.
Resolved
report = outcome.get_result()

if not report.failed:
return

log_path = item.session.stash.get(_services_log_key, None)
if log_path and log_path.exists():
lines = log_path.read_text().splitlines(keepends=True)
tail = lines[-_TAIL_LINES_ON_FAILURE:]
if tail:
header = f"--- services log (last {len(tail)} lines) [{log_path}] ---"
report.sections.append(("Services Log", f"{header}\n{''.join(tail)}"))


# ---- Fixtures --------------------------------------------------------------


@pytest.fixture(scope="session")
def _services() -> Iterator[str]:
def _services(services_log_path: Path) -> Iterator[str]:
"""Spawn ``nemo services run`` and yield the base URL.

Skipped when ``NMP_BASE_URL`` is already set (external services).
Expand All @@ -124,7 +172,7 @@ def _services() -> Iterator[str]:

logger.info("Starting nemo services on port %d", port)

log_path = _SERVICES_LOG
log_path = services_log_path
with open(log_path, "w") as log_file, background_process(args, stdout=log_file) as proc:
if not _wait_for_healthy(url):
pytest.fail(
Expand Down
File renamed without changes.
309 changes: 309 additions & 0 deletions e2e/files/test_storage_backends.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,309 @@
"""E2E tests for external storage backends (NGC, Hugging Face).

These tests verify that the files service can create filesets backed by
external storage providers and read files from them via the SDK.

NGC tests require ``NGC_API_KEY`` in the environment and are skipped
otherwise. Hugging Face tests use a small public repo; when ``HF_TOKEN``
is set the request is authenticated (avoids rate-limits in CI).
"""

import os
import tempfile
import uuid
from collections.abc import Iterator
from pathlib import Path

import pytest
from nemo_platform import BadRequestError, NeMoPlatform
from nemo_platform.types.files import HuggingfaceStorageConfigParam, NGCStorageConfigParam

# ---------------------------------------------------------------------------
# NGC configuration
# ---------------------------------------------------------------------------
NGC_API_KEY_ENV = "NGC_API_KEY"

NGC_ORG = "nvidia"
NGC_TEAM = "nemo-microservices"
NGC_TARGET = "nemo-microservices-quickstart"
NGC_TARGET_TYPE = "resource"

# ---------------------------------------------------------------------------
# Hugging Face configuration — small public model
# ---------------------------------------------------------------------------
HF_TOKEN_ENV = "HF_TOKEN"

HF_REPO_ID = "hf-internal-testing/tiny-random-bert"
HF_REPO_TYPE = "model"

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture
def ngc_api_key() -> str:
"""Return the NGC API key from the environment."""
key = os.environ.get(NGC_API_KEY_ENV)
assert key, f"{NGC_API_KEY_ENV} must be set"
return key


@pytest.fixture
def ngc_secret(sdk: NeMoPlatform, workspace: str, ngc_api_key: str) -> Iterator[str]:
"""Create a secret containing the NGC API key, cleaned up after test."""
secret_name = f"e2e-ngc-key-{uuid.uuid4().hex[:8]}"
sdk.secrets.create(workspace=workspace, name=secret_name, value=ngc_api_key)
yield secret_name
try:
sdk.secrets.delete(workspace=workspace, name=secret_name)
except Exception:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
pass


@pytest.fixture
def ngc_fileset(sdk: NeMoPlatform, workspace: str, ngc_secret: str) -> Iterator[str]:
"""Create an NGC-backed fileset, cleaned up after test."""
fileset_name = f"e2e-ngc-fs-{uuid.uuid4().hex[:8]}"
sdk.files.filesets.create(
workspace=workspace,
name=fileset_name,
description="E2E test NGC-backed fileset",
storage=NGCStorageConfigParam(
api_key_secret=ngc_secret,
org=NGC_ORG,
team=NGC_TEAM,
target=NGC_TARGET,
target_type=NGC_TARGET_TYPE,
),
)
yield fileset_name
try:
sdk.files.filesets.delete(fileset_name, workspace=workspace)
except Exception:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
pass


@pytest.fixture
def hf_secret(sdk: NeMoPlatform, workspace: str) -> Iterator[str | None]:
Comment thread
matthewgrossman marked this conversation as resolved.
Resolved
"""Create a secret for the HF token if available, otherwise yield None."""
token = os.environ.get(HF_TOKEN_ENV)
if not token:
yield None
return

secret_name = f"e2e-hf-tok-{uuid.uuid4().hex[:8]}"
sdk.secrets.create(workspace=workspace, name=secret_name, value=token)
yield secret_name
try:
sdk.secrets.delete(workspace=workspace, name=secret_name)
except Exception:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
pass


@pytest.fixture
def hf_fileset(sdk: NeMoPlatform, workspace: str, hf_secret: str | None) -> Iterator[str]:
"""Create a Hugging Face-backed fileset, cleaned up after test."""
fileset_name = f"e2e-hf-fs-{uuid.uuid4().hex[:8]}"

storage = HuggingfaceStorageConfigParam(
repo_id=HF_REPO_ID,
repo_type=HF_REPO_TYPE,
)
if hf_secret is not None:
storage["token_secret"] = hf_secret

sdk.files.filesets.create(
workspace=workspace,
name=fileset_name,
description="E2E test HF-backed fileset",
storage=storage,
)
yield fileset_name
try:
sdk.files.filesets.delete(fileset_name, workspace=workspace)
except Exception:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
pass


# ===================================================================
# NGC tests
# ===================================================================


@pytest.mark.skipif(not os.environ.get(NGC_API_KEY_ENV), reason=f"{NGC_API_KEY_ENV} not set")
class TestNGCFileset:
"""Tests for NGC-backed filesets."""

def test_list_files(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str):
"""Listing an NGC-backed fileset returns files with paths and sizes."""
files = sdk.files.list(fileset=ngc_fileset, workspace=workspace)
assert len(files.data) > 0, "NGC fileset should contain at least one file"

for f in files.data:
assert f.path, "Each file should have a path"
assert f.size > 0, "Each file should have a non-zero size"

def test_download_file(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str):
"""Downloading the smallest file from an NGC fileset succeeds and size matches."""
files = sdk.files.list(fileset=ngc_fileset, workspace=workspace)
assert len(files.data) > 0

target = min(files.data, key=lambda f: f.size)

with tempfile.TemporaryDirectory() as tmpdir:
local_path = Path(tmpdir) / target.path.replace("/", "_")
sdk.files.download(
fileset=ngc_fileset,
workspace=workspace,
remote_path=target.path,
local_path=str(local_path),
)
assert local_path.exists()
assert local_path.stat().st_size == target.size

def test_cache_status(self, sdk: NeMoPlatform, workspace: str, ngc_fileset: str):
"""NGC-backed files report a cacheable status."""
files = sdk.files.list(
fileset=ngc_fileset,
workspace=workspace,
include_cache_status=True,
)
assert len(files.data) > 0

for f in files.data:
assert f.cache_status is not None
assert f.cache_status != "not_cacheable"

# -- error cases --

@pytest.mark.parametrize(
("secret_value", "storage_overrides", "match"),
[
pytest.param(
"not-a-real-key",
{},
"Invalid API key. Legacy NGC keys are not supported.",
id="invalid-key-prefix",
),
pytest.param(
None,
{"org": "nvidian", "team": "nemo-llm", "target": "nemo-platform-quickstart"},
"Error creating NGC storage backend:",
id="wrong-org",
),
pytest.param(
None,
{"target": "this-resource-does-not-exist-12345"},
"Failed to access NGC resource this-resource-does-not-exist-12345",
id="nonexistent-resource",
Comment thread
matthewgrossman marked this conversation as resolved.
Resolved
),
],
)
def test_create_error(
self,
sdk: NeMoPlatform,
workspace: str,
ngc_api_key: str,
secret_value: str | None,
storage_overrides: dict,
match: str,
):
"""Bad NGC configurations are rejected with 400."""
value = secret_value if secret_value is not None else ngc_api_key
secret_name = f"e2e-ngc-err-{uuid.uuid4().hex[:8]}"
sdk.secrets.create(workspace=workspace, name=secret_name, value=value)
try:
storage = NGCStorageConfigParam(
api_key_secret=secret_name,
org=storage_overrides.get("org", NGC_ORG),
team=storage_overrides.get("team", NGC_TEAM),
target=storage_overrides.get("target", NGC_TARGET),
target_type=NGC_TARGET_TYPE,
)
with pytest.raises(BadRequestError, match=match):
sdk.files.filesets.create(
workspace=workspace,
name=f"e2e-ngc-err-{uuid.uuid4().hex[:8]}",
storage=storage,
)
finally:
sdk.secrets.delete(workspace=workspace, name=secret_name)

def test_create_error_nonexistent_secret(self, sdk: NeMoPlatform, workspace: str):
"""Referencing a secret that doesn't exist is rejected with 400."""
with pytest.raises(BadRequestError, match="Secret not found:"):
sdk.files.filesets.create(
workspace=workspace,
name=f"e2e-ngc-err-{uuid.uuid4().hex[:8]}",
storage=NGCStorageConfigParam(
api_key_secret="no-such-secret-99999",
org=NGC_ORG,
team=NGC_TEAM,
target=NGC_TARGET,
target_type=NGC_TARGET_TYPE,
),
)


# ===================================================================
# Hugging Face tests
# ===================================================================


class TestHuggingFaceFileset:
"""Tests for Hugging Face-backed filesets."""

def test_list_files(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str):
"""Listing an HF-backed fileset returns files with paths and sizes."""
files = sdk.files.list(fileset=hf_fileset, workspace=workspace)
assert len(files.data) > 0, "HF fileset should contain at least one file"

for f in files.data:
assert f.path, "Each file should have a path"
assert f.size > 0, "Each file should have a non-zero size"

def test_download_file(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str):
"""Downloading the smallest file from an HF fileset succeeds and size matches."""
files = sdk.files.list(fileset=hf_fileset, workspace=workspace)
assert len(files.data) > 0

target = min(files.data, key=lambda f: f.size)

with tempfile.TemporaryDirectory() as tmpdir:
local_path = Path(tmpdir) / target.path.replace("/", "_")
sdk.files.download(
fileset=hf_fileset,
workspace=workspace,
remote_path=target.path,
local_path=str(local_path),
)
assert local_path.exists()
assert local_path.stat().st_size == target.size

def test_cache_status(self, sdk: NeMoPlatform, workspace: str, hf_fileset: str):
"""HF-backed files report a cacheable status."""
files = sdk.files.list(
fileset=hf_fileset,
workspace=workspace,
include_cache_status=True,
)
assert len(files.data) > 0

for f in files.data:
assert f.cache_status is not None
assert f.cache_status != "not_cacheable"

# -- error cases --

def test_error_nonexistent_repo(self, sdk: NeMoPlatform, workspace: str):
"""Pointing at a repo that doesn't exist is rejected with 400."""
with pytest.raises(BadRequestError):
sdk.files.filesets.create(
workspace=workspace,
name=f"e2e-hf-err-{uuid.uuid4().hex[:8]}",
storage=HuggingfaceStorageConfigParam(
repo_id="this-org-does-not-exist/this-repo-does-not-exist-12345",
repo_type="model",
),
)
Loading
Loading