diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a8913d3218..0973fc23e9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -202,14 +202,11 @@ jobs: coverage.xml coverage.json - # One job, one (package x python) matrix — kept as a single job rather - # than split per-package because the build path is identical and the - # matrix definition is the natural place to change what's covered. - # nemo-platform's wheel force-includes auth's policy.wasm at build time; - # nemo-platform-plugin doesn't, but plugin rows still wait on policy-wasm - # because needs: is per-job. ~5s of harmless wait per plugin row. - wheel-test: - name: ${{ matrix.package }} wheel build + test (py${{ matrix.python-version }}) + # Build wheels for all packages × python versions. Downstream jobs + # (wheel-test, python-e2e-test) download these artifacts instead + # of rebuilding. + wheel-build: + name: ${{ matrix.package }} wheel build (py${{ matrix.python-version }}) needs: [policy-wasm] runs-on: ubuntu-latest timeout-minutes: 25 @@ -240,25 +237,53 @@ jobs: # build, but stamp_sdk_version.py requires \d{14}. nightly-timestamp: "19700101000000" python-version: ${{ matrix.python-version }} + - name: Upload wheel + uses: actions/upload-artifact@v6 + with: + name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }} + path: ${{ steps.build.outputs.wheel-path }} + retention-days: 7 + if-no-files-found: error + + # Test each wheel: install from the artifact and run basic CLI / + # import checks. + wheel-test: + name: ${{ matrix.package }} wheel build + test (py${{ matrix.python-version }}) + needs: [wheel-build] + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + package: [nemo-platform, nemo-platform-plugin] + python-version: ["3.11", "3.12", "3.13"] + steps: + - name: Checkout code + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + - name: Download wheel + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }} + path: ${{ runner.temp }}/wheelcheck - # The two install+test steps below intentionally share a small preamble - # (mkdir + cp). Hoisting it into a separate step would cost more in - # workflow indirection than the 3 duplicated lines save. - name: Install + test nemo-platform CLI if: matrix.package == 'nemo-platform' shell: bash env: - WHEEL: ${{ steps.build.outputs.wheel-path }} PYTHON_VERSION: ${{ matrix.python-version }} NMP_DATA_DIR: ${{ runner.temp }}/nemo-data + _TYPER_FORCE_DISABLE_TERMINAL: "1" run: | set -euo pipefail - # Copy the wheel out of $GITHUB_WORKSPACE so the test cd's away - # from the source tree and Python can't reach back into it. - mkdir -p "${RUNNER_TEMP}/wheelcheck" - cp "${WHEEL}" "${RUNNER_TEMP}/wheelcheck/" - uv tool install --force --python "${PYTHON_VERSION}" \ - "${RUNNER_TEMP}/wheelcheck/$(basename "${WHEEL}")[services]" + WHEEL="$(ls ${RUNNER_TEMP}/wheelcheck/*.whl)" + uv tool install --force --python "${PYTHON_VERSION}" "${WHEEL}[services]" cd "${RUNNER_TEMP}/wheelcheck" unset PYTHONPATH VIRTUAL_ENV bash "${GITHUB_WORKSPACE}/script/test-nemo-cli.sh" @@ -267,22 +292,18 @@ jobs: if: matrix.package == 'nemo-platform-plugin' shell: bash env: - WHEEL: ${{ steps.build.outputs.wheel-path }} PYTHON_VERSION: ${{ matrix.python-version }} run: | set -euo pipefail - mkdir -p "${RUNNER_TEMP}/wheelcheck" - cp "${WHEEL}" "${RUNNER_TEMP}/wheelcheck/" + WHEEL="$(ls ${RUNNER_TEMP}/wheelcheck/*.whl)" cd "${RUNNER_TEMP}/wheelcheck" uv venv .venv --python "${PYTHON_VERSION}" - uv pip install --python .venv/bin/python "$(basename "${WHEEL}")" + uv pip install --python .venv/bin/python "${WHEEL}" unset PYTHONPATH VIRTUAL_ENV # Import the package + a couple of representative submodules # (cli, commands). These are the surfaces a plugin author would # touch first; if any of them fail to import, the wheel is - # broken in a way that surfaces immediately on day one. If - # the plugin's public API surface changes substantially, this - # list should be revisited. + # broken in a way that surfaces immediately on day one. .venv/bin/python -c " import nemo_platform_plugin import nemo_platform_plugin.cli @@ -290,14 +311,46 @@ jobs: print('nemo_platform_plugin', getattr(nemo_platform_plugin, '__version__', '')) " - - name: Upload wheel + # E2E tests: start services from the workspace venv and run the e2e + # suite against the real process. + python-e2e-test: + name: Python e2e tests + needs: [policy-wasm] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v6 + - name: Download policy WASM + uses: actions/download-artifact@v8 + with: + name: policy-wasm + path: services/core/auth/src/nmp/core/auth/assets + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + - name: Run e2e tests + run: make test-e2e + env: + _TYPER_FORCE_DISABLE_TERMINAL: "1" + E2E_SERVICES_LOG: ${{ runner.temp }}/services.log + - name: Dump server logs + if: always() + run: | + echo "::group::Server log" + cat "${{ runner.temp }}/services.log" 2>/dev/null || echo "No server log found" + echo "::endgroup::" + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v6 with: - name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }} - path: ${{ steps.build.outputs.wheel-path }} - retention-days: 7 - if-no-files-found: error + name: python-e2e-test-results + retention-days: 30 + path: | + report.xml + ${{ runner.temp }}/services.log # Required-check pin: branch protection should reference this aggregator # rather than the per-row matrix jobs, so the matrix can grow or shrink @@ -314,10 +367,6 @@ jobs: shell: bash env: MATRIX_RESULT: ${{ needs.wheel-test.result }} - # Per-row results, useful for debugging which (package, python) - # combination failed. GitHub doesn't expose individual matrix - # results by name; the workflow run UI is the canonical place - # to look. We surface a pointer in the failure message. RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail diff --git a/Makefile b/Makefile index bf23349a64..7709b62d2d 100644 --- a/Makefile +++ b/Makefile @@ -266,11 +266,10 @@ test-all-script: ## Run all unit tests using the helper script (with summary) @echo "Running all unit tests with summary..." uv run --frozen python tools/run_all_tests.py -# NOTE: disabled this in favour of the other test-e2e target (see below) .PHONY: test-e2e -# test-e2e: ## Run Python end-to-end tests (customer workflows and blueprints) -# @echo "Running Python end-to-end tests..." -# uv run --frozen pytest -v -m e2e +test-e2e: ## Run e2e tests against nemo services (starts/stops services automatically) + @echo "Running e2e tests..." + uv run --frozen pytest e2e -v --run-e2e --junitxml=report.xml $(PYTEST_EXTRA) .PHONY: test-regression test-regression: ## Run Python regression tests (functional microservice baseline tests) diff --git a/TESTING.md b/TESTING.md index 306a5c59c9..34fcd2150c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -113,25 +113,38 @@ def test_create_and_fetch_entity(client, db_session): ### 3. End-to-End (E2E) Tests -**Objective**: Ensure that customers can orchestrate services together for common workflows and blueprints on real deployed infrastructure. +**Objective**: Ensure that services work together correctly when running as a real platform process. **Characteristics**: -- Test complete customer workflows on actual deployments -- Uses testcontainers with Docker or Kubernetes backends -- Multiple services working together with real infrastructure -- Slowest tests (minutes to hours) -- Requires deployed infrastructure (no mocking of Jobs or Inference) +- Start the platform via `nemo services run` (real process, real ports) +- Hit services with an external HTTP client (the NeMoPlatform SDK) +- Test startup machinery, port binding, config resolution, and cross-service workflows +- Slower than integration tests (tens of seconds for startup) but faster than Docker/K8s e2e + +**How to run**: + +```bash +# Start services, run tests, stop services (all automatic) +make test-e2e + +# Or manually +uv run --frozen pytest e2e -v --run-e2e + +# If you already have services running +NMP_BASE_URL=http://localhost:8080 uv run --frozen pytest e2e -v --run-e2e +``` + +**Prerequisites**: `make bootstrap` must have been run. The harness spawns `nemo services run` +on a free port, so it won't conflict with your dev instance. **When to Write E2E Tests**: Write E2E tests when you need to: - **Verify cross-service workflows**: Test operations that span multiple services (e.g., create workspace → upload file → run job → get results) -- **Test real infrastructure**: Validate jobs, inference, or storage backends that cannot be mocked -- **Validate customer scenarios**: Test complete workflows as users would experience them +- **Validate the real startup path**: Ensure config resolution, service discovery, and health checks work - **Test authentication/authorization**: Verify role-based access control across multiple services -- **Test async workflows**: Validate long-running operations, job lifecycles, and event propagation -- **Ensure service integration**: Verify that services work together correctly in production-like setups +- **Ensure service integration**: Verify that services work together correctly end-to-end **What NOT to E2E Test**: - Single service APIs (use integration tests) @@ -139,9 +152,7 @@ Write E2E tests when you need to: - Every permutation of inputs (E2E should focus on critical paths) - Implementation details (test user-visible behavior) -For detailed E2E test documentation, configuration options, and best practices, see: **[e2e/README.md](e2e/README.md)** - -**Location**: `e2e/` (root-level for deployed infrastructure tests) +**Location**: `e2e/` (root-level) ### 4. Infrastructure Tests @@ -395,19 +406,12 @@ uv run python tools/run_all_tests.py make test-integration uv run pytest -v -m integration -# End-to-end tests (Docker backend - recommended for local dev) -make test-e2e-docker -uv run pytest e2e --docker -v - -# End-to-end tests (Kubernetes backend: local minikube or custom cluster) -make test-e2e-minikube -uv run pytest e2e --kubernetes --cluster-url=https://my-cluster.example.com -v - -# E2E with custom registry and tag -uv run pytest e2e --docker --registry=my-registry --tag=v1.0.0 -v +# End-to-end tests (starts nemo services automatically) +make test-e2e +uv run --frozen pytest e2e -v --run-e2e -# E2E with custom config -uv run pytest e2e --docker --config=e2e/quickstart/custom.yaml -v +# E2E against an already-running instance +NMP_BASE_URL=http://localhost:8080 uv run --frozen pytest e2e -v --run-e2e # Regression tests make test-regression diff --git a/conftest.py b/conftest.py index 5d2c6809d8..87351128fc 100644 --- a/conftest.py +++ b/conftest.py @@ -285,6 +285,9 @@ def pytest_runtest_setup(item): if "slow" in [marker.name for marker in item.iter_markers()]: if not item.config.getoption("--run-slow"): skip_test("Skipping slow test (use --run-slow to run)") + if "e2e" in [marker.name for marker in item.iter_markers()]: + if not item.config.getoption("--run-e2e"): + skip_test("Skipping e2e test (use --run-e2e to run)") from xdist.scheduler.loadscope import LoadScopeScheduling # noqa: E402 diff --git a/e2e/conftest.py b/e2e/conftest.py new file mode 100644 index 0000000000..49fbffb06c --- /dev/null +++ b/e2e/conftest.py @@ -0,0 +1,135 @@ +"""E2E test fixtures that run against a real ``nemo services`` process. + +Usage:: + + # Start services, run e2e tests, stop services + make test-e2e + + # Or manually + uv run --frozen pytest e2e -v --run-e2e + + # If you already have services running + NMP_BASE_URL=http://localhost:9090 uv run --frozen pytest e2e -v --run-e2e + +When ``NMP_BASE_URL`` is set the harness skips service startup/shutdown and +connects to the given URL. Otherwise it spawns ``nemo services run`` as a +child process on a free port, polls ``/health/ready`` until ready, and +terminates the process after the session. +""" + +import contextlib +import logging +import os +import socket +import subprocess +import sys +import tempfile +import time +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import IO, Any + +import httpx +import pytest +from nemo_platform import NeMoPlatform + +logger = logging.getLogger(__name__) + +_HEALTH_TIMEOUT = 60 +_HEALTH_POLL_INTERVAL = 1.0 +_SERVICES_LOG = Path(os.environ.get("E2E_SERVICES_LOG", os.path.join(tempfile.gettempdir(), "services.log"))) + + +def _find_free_port() -> int: + """Bind to port 0 and let the OS assign a free port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_for_healthy(url: str, timeout: float = _HEALTH_TIMEOUT) -> bool: + """Poll /health/ready until it returns 200 or timeout expires.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + resp = httpx.get(f"{url}/health/ready", timeout=2.0) + if resp.status_code == 200: + return True + except httpx.RequestError: + pass # Server not up yet, keep polling + time.sleep(_HEALTH_POLL_INTERVAL) + return False + + +@contextlib.contextmanager +def background_process(args: list[str], stdout: IO[Any] | None = None) -> Iterator[subprocess.Popen]: + """Run a subprocess, yield the ``Popen``, and terminate on exit. + + Unlike ``Popen``'s built-in context manager (which only waits for the + process), this sends SIGTERM/SIGKILL so long-running servers are + cleaned up. + """ + proc = subprocess.Popen(args, stdout=stdout, stderr=subprocess.STDOUT) + try: + yield proc + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + logger.warning("Process %d did not exit after SIGTERM, sending SIGKILL", proc.pid) + proc.kill() + proc.wait(timeout=5) + + +@pytest.fixture(scope="session") +def _services() -> Iterator[str]: + """Spawn ``nemo services run`` and yield the base URL. + + Skipped when ``NMP_BASE_URL`` is already set (external services). + + This is the "subprocess" backend. When we add Docker and Kubernetes + backends, this fixture should be replaced by a backend-selection layer + (e.g. ``--docker`` / ``--kubernetes`` CLI flags) that dispatches to the + appropriate setup while yielding the same base URL interface. Tests + should remain agnostic to the backend. + """ + external_url = os.environ.get("NMP_BASE_URL") + if external_url: + yield external_url + return + + port = _find_free_port() + url = f"http://127.0.0.1:{port}" + + nemo_bin = str(Path(sys.executable).parent / "nemo") + args = [nemo_bin, "services", "run", "--service-group", "all", "--port", str(port)] + + logger.info("Starting nemo services on port %d", port) + + log_path = _SERVICES_LOG + with open(log_path, "w") as log_file, background_process(args, stdout=log_file) as proc: + if not _wait_for_healthy(url): + pytest.fail( + f"nemo services run did not become healthy within {_HEALTH_TIMEOUT}s.\nlog:\n{log_path.read_text()}" + ) + + logger.info("Platform services ready on port %d (pid %d)", port, proc.pid) + yield url + logger.info("Terminating nemo services (pid %d)", proc.pid) + + +@pytest.fixture(scope="session") +def sdk(_services: str) -> NeMoPlatform: + """Provide an SDK client connected to the running platform.""" + return NeMoPlatform(base_url=_services, max_retries=2) + + +@pytest.fixture(scope="function") +def workspace(sdk: NeMoPlatform) -> Iterator[str]: + """Create a unique workspace for each test, deleted on teardown.""" + name = f"e2e-{uuid.uuid4().hex[:8]}" + sdk.workspaces.create(name=name) + yield name + sdk.workspaces.delete(name) diff --git a/e2e/test_smoke.py b/e2e/test_smoke.py new file mode 100644 index 0000000000..322bd1efd3 --- /dev/null +++ b/e2e/test_smoke.py @@ -0,0 +1,38 @@ +"""Smoke tests that verify the platform is reachable and core APIs respond. + +These are intentionally minimal — they validate the e2e harness works and +that services are up. Add more substantive tests in separate files. +""" + +import uuid + +from nemo_platform import NeMoPlatform + + +def test_health_ready(sdk: NeMoPlatform): + """GET /health/ready returns 200 when all services are up.""" + resp = sdk._client.get("/health/ready") + assert resp.status_code == 200 + + +def test_health_live(sdk: NeMoPlatform): + """GET /health/live returns 200 (liveness probe).""" + resp = sdk._client.get("/health/live") + assert resp.status_code == 200 + + +def test_create_and_delete_workspace(sdk: NeMoPlatform): + """Workspace create and delete round-trips through the platform.""" + name = f"e2e-smoke-{uuid.uuid4().hex[:8]}" + ws = sdk.workspaces.create(name=name) + try: + assert ws.name == name + finally: + sdk.workspaces.delete(name) + + +def test_list_workspaces(sdk: NeMoPlatform, workspace: str): + """Listing workspaces returns at least the test workspace.""" + page = sdk.workspaces.list() + names = [w.name for w in page.data] + assert workspace in names