Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -337,12 +337,16 @@ jobs:
run: make test-e2e
env:
_TYPER_FORCE_DISABLE_TERMINAL: "1"
E2E_SERVICES_LOG: ${{ runner.temp }}/services.log
E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
NGC_API_KEY: ${{ secrets.NGC_REGISTRY_READ_TOKEN }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Dump server logs
if: always()
run: |
echo "::group::Server log"
cat "${{ runner.temp }}/services.log" 2>/dev/null || echo "No server log found"
echo "::group::Server logs"
for f in "${{ runner.temp }}/e2e-services-logs"/*.log; do
[ -f "$f" ] && echo "--- $(basename "$f") ---" && cat "$f" || echo "No server logs found"
done
echo "::endgroup::"
- name: Upload test artifacts
if: always()
Expand All @@ -352,7 +356,7 @@ jobs:
retention-days: 30
path: |
report.xml
${{ runner.temp }}/services.log
${{ runner.temp }}/e2e-services-logs/

# Required-check pin: branch protection should reference this aggregator
# rather than the per-row matrix jobs, so the matrix can grow or shrink
Expand Down
65 changes: 61 additions & 4 deletions e2e/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import socket
import subprocess
import sys
import tempfile
import time
import uuid
from collections.abc import Iterator
Expand Down Expand Up @@ -54,7 +53,36 @@ def pytest_configure(config: pytest.Config) -> None:

_HEALTH_TIMEOUT = 60
_HEALTH_POLL_INTERVAL = 1.0
_SERVICES_LOG = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log")))

# Number of log lines to dump from the services log on test failure.
_TAIL_LINES_ON_FAILURE = 100

_services_log_key = pytest.StashKey[Path]()


@pytest.fixture(scope="session")
def services_log_path(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Path:
"""Return a unique services log path for this session.

``E2E_SERVICES_LOG_DIR`` (if set) is treated as a **directory**; in CI
the job uploads everything under it as artifacts. When unset we
fall back to a pytest-managed temp directory. Either way, each
session writes to a UUID-named file inside the directory so
parallel workers never clobber each other.

The path is stashed on the session so the
``pytest_runtest_makereport`` hook can read it without requesting
the fixture.
"""
log_dir = os.environ.get("E2E_SERVICES_LOG_DIR")
if log_dir:
directory = Path(log_dir)
directory.mkdir(parents=True, exist_ok=True)
else:
directory = tmp_path_factory.mktemp("e2e-services-logs")
path = directory / f"services-{uuid.uuid4().hex[:8]}.log"
request.session.stash[_services_log_key] = path
return path


def _find_free_port() -> int:
Expand Down Expand Up @@ -99,8 +127,37 @@ def background_process(args: list[str], stdout: IO[Any] | None = None) -> Iterat
proc.wait(timeout=5)


# ---- Services log tail on failure ------------------------------------------


@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo): # noqa: ARG001
"""Append the services log tail to the report when a test fails.

This hook is the pytest-sanctioned way to add extra sections to test
reports (``report.sections``). Fixtures cannot do this because they
don't have access to the report object.
"""
outcome = yield
Comment thread
matthewgrossman marked this conversation as resolved.
report = outcome.get_result()

if not report.failed:
return

log_path = item.session.stash.get(_services_log_key, None)
if log_path and log_path.exists():
lines = log_path.read_text().splitlines(keepends=True)
tail = lines[-_TAIL_LINES_ON_FAILURE:]
if tail:
header = f"--- services log (last {len(tail)} lines) [{log_path}] ---"
report.sections.append(("Services Log", f"{header}\n{''.join(tail)}"))


# ---- Fixtures --------------------------------------------------------------


@pytest.fixture(scope="session")
def _services() -> Iterator[str]:
def _services(services_log_path: Path) -> Iterator[str]:
"""Spawn ``nemo services run`` and yield the base URL.

Skipped when ``NMP_BASE_URL`` is already set (external services).
Expand All @@ -124,7 +181,7 @@ def _services() -> Iterator[str]:

logger.info("Starting nemo services on port %d", port)

log_path = _SERVICES_LOG
log_path = services_log_path
with open(log_path, "w") as log_file, background_process(args, stdout=log_file) as proc:
if not _wait_for_healthy(url):
pytest.fail(
Expand Down
File renamed without changes.
Loading
Loading