feat(ci): Add e2e test suite, including barebones test_smoke.py

.github/workflows/ci.yaml

@@ -202,14 +202,11 @@ jobs:
             coverage.xml
             coverage.json
 
-  # One job, one (package x python) matrix — kept as a single job rather
-  # than split per-package because the build path is identical and the
-  # matrix definition is the natural place to change what's covered.
-  # nemo-platform's wheel force-includes auth's policy.wasm at build time;
-  # nemo-platform-plugin doesn't, but plugin rows still wait on policy-wasm
-  # because needs: is per-job. ~5s of harmless wait per plugin row.
-  wheel-test:
-    name: ${{ matrix.package }} wheel build + test (py${{ matrix.python-version }})
+  # Build wheels for all packages × python versions. Downstream jobs
+  # (wheel-test, python-e2e-test) download these artifacts instead
+  # of rebuilding.
+  wheel-build:
+    name: ${{ matrix.package }} wheel build (py${{ matrix.python-version }})
     needs: [policy-wasm]
     runs-on: ubuntu-latest
     timeout-minutes: 25
@@ -240,25 +237,53 @@ jobs:
           # build, but stamp_sdk_version.py requires \d{14}.
           nightly-timestamp: "19700101000000"
           python-version: ${{ matrix.python-version }}
+      - name: Upload wheel
+        uses: actions/upload-artifact@v6
+        with:
+          name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }}
+          path: ${{ steps.build.outputs.wheel-path }}
+          retention-days: 7
+          if-no-files-found: error
+
+  # Test each wheel: install from the artifact and run basic CLI /
+  # import checks.
+  wheel-test:
+    name: ${{ matrix.package }} wheel build + test (py${{ matrix.python-version }})
+    needs: [wheel-build]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        package: [nemo-platform, nemo-platform-plugin]
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+      - name: Download wheel
+        uses: actions/download-artifact@v8
+        with:
+          name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }}
+          path: ${{ runner.temp }}/wheelcheck
 
-      # The two install+test steps below intentionally share a small preamble
-      # (mkdir + cp). Hoisting it into a separate step would cost more in
-      # workflow indirection than the 3 duplicated lines save.
       - name: Install + test nemo-platform CLI
         if: matrix.package == 'nemo-platform'
         shell: bash
         env:
-          WHEEL: ${{ steps.build.outputs.wheel-path }}
           PYTHON_VERSION: ${{ matrix.python-version }}
           NMP_DATA_DIR: ${{ runner.temp }}/nemo-data
+          _TYPER_FORCE_DISABLE_TERMINAL: "1"
         run: |
           set -euo pipefail
-          # Copy the wheel out of $GITHUB_WORKSPACE so the test cd's away
-          # from the source tree and Python can't reach back into it.
-          mkdir -p "${RUNNER_TEMP}/wheelcheck"
-          cp "${WHEEL}" "${RUNNER_TEMP}/wheelcheck/"
-          uv tool install --force --python "${PYTHON_VERSION}" \
-            "${RUNNER_TEMP}/wheelcheck/$(basename "${WHEEL}")[services]"
+          WHEEL="$(ls ${RUNNER_TEMP}/wheelcheck/*.whl)"
+          uv tool install --force --python "${PYTHON_VERSION}" "${WHEEL}[services]"
           cd "${RUNNER_TEMP}/wheelcheck"
           unset PYTHONPATH VIRTUAL_ENV
           bash "${GITHUB_WORKSPACE}/script/test-nemo-cli.sh"
@@ -267,37 +292,65 @@ jobs:
         if: matrix.package == 'nemo-platform-plugin'
         shell: bash
         env:
-          WHEEL: ${{ steps.build.outputs.wheel-path }}
           PYTHON_VERSION: ${{ matrix.python-version }}
         run: |
           set -euo pipefail
-          mkdir -p "${RUNNER_TEMP}/wheelcheck"
-          cp "${WHEEL}" "${RUNNER_TEMP}/wheelcheck/"
+          WHEEL="$(ls ${RUNNER_TEMP}/wheelcheck/*.whl)"
           cd "${RUNNER_TEMP}/wheelcheck"
           uv venv .venv --python "${PYTHON_VERSION}"
-          uv pip install --python .venv/bin/python "$(basename "${WHEEL}")"
+          uv pip install --python .venv/bin/python "${WHEEL}"
           unset PYTHONPATH VIRTUAL_ENV
           # Import the package + a couple of representative submodules
           # (cli, commands). These are the surfaces a plugin author would
           # touch first; if any of them fail to import, the wheel is
-          # broken in a way that surfaces immediately on day one. If
-          # the plugin's public API surface changes substantially, this
-          # list should be revisited.
+          # broken in a way that surfaces immediately on day one.
           .venv/bin/python -c "
           import nemo_platform_plugin
           import nemo_platform_plugin.cli
           import nemo_platform_plugin.commands
           print('nemo_platform_plugin', getattr(nemo_platform_plugin, '__version__', '<no __version__>'))
           "
 
-      - name: Upload wheel
+  # E2E tests: start services from the workspace venv and run the e2e
+  # suite against the real process.
+  python-e2e-test:
+    name: Python e2e tests
+    needs: [policy-wasm]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Download policy WASM
+        uses: actions/download-artifact@v8
+        with:
+          name: policy-wasm
+          path: services/core/auth/src/nmp/core/auth/assets
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.13"
+          enable-cache: true
+      - name: Run e2e tests
+        run: make test-e2e
+        env:
+          _TYPER_FORCE_DISABLE_TERMINAL: "1"
+          E2E_SERVICES_LOG: ${{ runner.temp }}/services.log
+      - name: Dump server logs
+        if: always()
+        run: |
+          echo "::group::Server log"
+          cat "${{ runner.temp }}/services.log" 2>/dev/null || echo "No server log found"
+          echo "::endgroup::"
+      - name: Upload test artifacts
         if: always()
         uses: actions/upload-artifact@v6
         with:
-          name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }}
-          path: ${{ steps.build.outputs.wheel-path }}
-          retention-days: 7
-          if-no-files-found: error
+          name: python-e2e-test-results
+          retention-days: 30
+          path: |
+            report.xml
+            ${{ runner.temp }}/services.log
 
   # Required-check pin: branch protection should reference this aggregator
   # rather than the per-row matrix jobs, so the matrix can grow or shrink
@@ -314,10 +367,6 @@ jobs:
         shell: bash
         env:
           MATRIX_RESULT: ${{ needs.wheel-test.result }}
-          # Per-row results, useful for debugging which (package, python)
-          # combination failed. GitHub doesn't expose individual matrix
-          # results by name; the workflow run UI is the canonical place
-          # to look. We surface a pointer in the failure message.
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -euo pipefail

Makefile

@@ -266,11 +266,10 @@ test-all-script: ## Run all unit tests using the helper script (with summary)
 	@echo "Running all unit tests with summary..."
 	uv run --frozen python tools/run_all_tests.py
 
-# NOTE: disabled this in favour of the other test-e2e target (see below)
 .PHONY: test-e2e
-# test-e2e: ## Run Python end-to-end tests (customer workflows and blueprints)
-# 	@echo "Running Python end-to-end tests..."
-# 	uv run --frozen pytest -v -m e2e
+test-e2e: ## Run e2e tests against nemo services (starts/stops services automatically)
+	@echo "Running e2e tests..."
+	uv run --frozen pytest e2e -v --run-e2e --junitxml=report.xml $(PYTEST_EXTRA)
 
 .PHONY: test-regression
 test-regression: ## Run Python regression tests (functional microservice baseline tests)

TESTING.md

@@ -113,35 +113,46 @@ def test_create_and_fetch_entity(client, db_session):
 
 ### 3. End-to-End (E2E) Tests
 
-**Objective**: Ensure that customers can orchestrate services together for common workflows and blueprints on real deployed infrastructure.
+**Objective**: Ensure that services work together correctly when running as a real platform process.
 
 **Characteristics**:
 
-- Test complete customer workflows on actual deployments
-- Uses testcontainers with Docker or Kubernetes backends
-- Multiple services working together with real infrastructure
-- Slowest tests (minutes to hours)
-- Requires deployed infrastructure (no mocking of Jobs or Inference)
+- Start the platform via `nemo services run` (real process, real ports)
+- Hit services with an external HTTP client (the NeMoPlatform SDK)
+- Test startup machinery, port binding, config resolution, and cross-service workflows
+- Slower than integration tests (tens of seconds for startup) but faster than Docker/K8s e2e
+
+**How to run**:
+
+```bash
+# Start services, run tests, stop services (all automatic)
+make test-e2e
+
+# Or manually
+uv run --frozen pytest e2e -v --run-e2e
+
+# If you already have services running
+NMP_BASE_URL=http://localhost:8080 uv run --frozen pytest e2e -v --run-e2e
+```
+
+**Prerequisites**: `make bootstrap` must have been run. The harness spawns `nemo services run`
+on a free port, so it won't conflict with your dev instance.
 
 **When to Write E2E Tests**:
 
 Write E2E tests when you need to:
 - **Verify cross-service workflows**: Test operations that span multiple services (e.g., create workspace → upload file → run job → get results)
-- **Test real infrastructure**: Validate jobs, inference, or storage backends that cannot be mocked
-- **Validate customer scenarios**: Test complete workflows as users would experience them
+- **Validate the real startup path**: Ensure config resolution, service discovery, and health checks work
 - **Test authentication/authorization**: Verify role-based access control across multiple services
-- **Test async workflows**: Validate long-running operations, job lifecycles, and event propagation
-- **Ensure service integration**: Verify that services work together correctly in production-like setups
+- **Ensure service integration**: Verify that services work together correctly end-to-end
 
 **What NOT to E2E Test**:
 - Single service APIs (use integration tests)
 - Business logic within a service (use unit/integration tests)
 - Every permutation of inputs (E2E should focus on critical paths)
 - Implementation details (test user-visible behavior)
 
-For detailed E2E test documentation, configuration options, and best practices, see: **[e2e/README.md](e2e/README.md)**
-
-**Location**: `e2e/` (root-level for deployed infrastructure tests)
+**Location**: `e2e/` (root-level)
 
 ### 4. Infrastructure Tests
 
@@ -395,19 +406,12 @@ uv run python tools/run_all_tests.py
 make test-integration
 uv run pytest -v -m integration
 
-# End-to-end tests (Docker backend - recommended for local dev)
-make test-e2e-docker
-uv run pytest e2e --docker -v
-
-# End-to-end tests (Kubernetes backend: local minikube or custom cluster)
-make test-e2e-minikube
-uv run pytest e2e --kubernetes --cluster-url=https://my-cluster.example.com -v
-
-# E2E with custom registry and tag
-uv run pytest e2e --docker --registry=my-registry --tag=v1.0.0 -v
+# End-to-end tests (starts nemo services automatically)
+make test-e2e
+uv run --frozen pytest e2e -v --run-e2e
 
-# E2E with custom config
-uv run pytest e2e --docker --config=e2e/quickstart/custom.yaml -v
+# E2E against an already-running instance
+NMP_BASE_URL=http://localhost:8080 uv run --frozen pytest e2e -v --run-e2e
 
 # Regression tests
 make test-regression

conftest.py

@@ -285,6 +285,9 @@ def pytest_runtest_setup(item):
     if "slow" in [marker.name for marker in item.iter_markers()]:
         if not item.config.getoption("--run-slow"):
             skip_test("Skipping slow test (use --run-slow to run)")
+    if "e2e" in [marker.name for marker in item.iter_markers()]:
+        if not item.config.getoption("--run-e2e"):
+            skip_test("Skipping e2e test (use --run-e2e to run)")
 
 
 from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa: E402