feat(ci): Add e2e test suite, including barebones test_smoke.py

Makefile

@@ -275,11 +275,10 @@ test-all-script: ## Run all unit tests using the helper script (with summary)
 	@echo "Running all unit tests with summary..."
 	uv run --frozen python tools/run_all_tests.py
 
-# NOTE: disabled this in favour of the other test-e2e target (see below)
 .PHONY: test-e2e
-# test-e2e: ## Run Python end-to-end tests (customer workflows and blueprints)
-# 	@echo "Running Python end-to-end tests..."
-# 	uv run --frozen pytest -v -m e2e
+test-e2e: ## Run e2e tests against nemo services (starts/stops services automatically)
+	@echo "Running e2e tests..."
+	uv run --frozen pytest e2e -v --run-e2e $(PYTEST_EXTRA)
 
 .PHONY: test-regression
 test-regression: ## Run Python regression tests (functional microservice baseline tests)

TESTING.md

@@ -113,35 +113,46 @@ def test_create_and_fetch_entity(client, db_session):
 
 ### 3. End-to-End (E2E) Tests
 
-**Objective**: Ensure that customers can orchestrate services together for common workflows and blueprints on real deployed infrastructure.
+**Objective**: Ensure that services work together correctly when running as a real platform process.
 
 **Characteristics**:
 
-- Test complete customer workflows on actual deployments
-- Uses testcontainers with Docker or Kubernetes backends
-- Multiple services working together with real infrastructure
-- Slowest tests (minutes to hours)
-- Requires deployed infrastructure (no mocking of Jobs or Inference)
+- Start the platform via `nemo services start` (real process, real ports)
+- Hit services with an external HTTP client (the NeMoPlatform SDK)
+- Test startup machinery, port binding, config resolution, and cross-service workflows
+- Slower than integration tests (tens of seconds for startup) but faster than Docker/K8s e2e
+
+**How to run**:
+
+```bash
+# Start services, run tests, stop services (all automatic)
+make test-e2e
+
+# Or manually
+uv run --frozen pytest e2e -v --run-e2e
+
+# If you already have services running
+NMP_BASE_URL=http://localhost:8080 uv run --frozen pytest e2e -v --run-e2e
+```
+
+**Prerequisites**: `make bootstrap` must have been run. The harness runs `nemo services start`
+on a free port with a unique `--instance` name, so it won't conflict with your dev instance.
 
 **When to Write E2E Tests**:
 
 Write E2E tests when you need to:
 - **Verify cross-service workflows**: Test operations that span multiple services (e.g., create workspace → upload file → run job → get results)
-- **Test real infrastructure**: Validate jobs, inference, or storage backends that cannot be mocked
-- **Validate customer scenarios**: Test complete workflows as users would experience them
+- **Validate the real startup path**: Ensure config resolution, service discovery, and health checks work
 - **Test authentication/authorization**: Verify role-based access control across multiple services
-- **Test async workflows**: Validate long-running operations, job lifecycles, and event propagation
-- **Ensure service integration**: Verify that services work together correctly in production-like setups
+- **Ensure service integration**: Verify that services work together correctly end-to-end
 
 **What NOT to E2E Test**:
 - Single service APIs (use integration tests)
 - Business logic within a service (use unit/integration tests)
 - Every permutation of inputs (E2E should focus on critical paths)
 - Implementation details (test user-visible behavior)
 
-For detailed E2E test documentation, configuration options, and best practices, see: **[e2e/README.md](e2e/README.md)**
-
-**Location**: `e2e/` (root-level for deployed infrastructure tests)
+**Location**: `e2e/` (root-level)
 
 ### 4. Infrastructure Tests
 
@@ -395,19 +406,12 @@ uv run python tools/run_all_tests.py
 make test-integration
 uv run pytest -v -m integration
 
-# End-to-end tests (Docker backend - recommended for local dev)
-make test-e2e-docker
-uv run pytest e2e --docker -v
-
-# End-to-end tests (Kubernetes backend: local minikube or custom cluster)
-make test-e2e-minikube
-uv run pytest e2e --kubernetes --cluster-url=https://my-cluster.example.com -v
-
-# E2E with custom registry and tag
-uv run pytest e2e --docker --registry=my-registry --tag=v1.0.0 -v
+# End-to-end tests (starts nemo services automatically)
+make test-e2e
+uv run --frozen pytest e2e -v --run-e2e
 
-# E2E with custom config
-uv run pytest e2e --docker --config=e2e/quickstart/custom.yaml -v
+# E2E against an already-running instance
+NMP_BASE_URL=http://localhost:8080 uv run --frozen pytest e2e -v --run-e2e
 
 # Regression tests
 make test-regression

conftest.py

@@ -285,6 +285,9 @@ def pytest_runtest_setup(item):
     if "slow" in [marker.name for marker in item.iter_markers()]:
         if not item.config.getoption("--run-slow"):
             skip_test("Skipping slow test (use --run-slow to run)")
+    if "e2e" in [marker.name for marker in item.iter_markers()]:
+        if not item.config.getoption("--run-e2e"):
+            skip_test("Skipping e2e test (use --run-e2e to run)")
 
 
 from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa: E402

e2e/conftest.py

@@ -0,0 +1,121 @@
+"""E2E test fixtures that run against a real ``nemo services`` process.
+
+Usage::
+
+    # Start services, run e2e tests, stop services
+    make test-e2e
+
+    # Or manually
+    uv run --frozen pytest e2e -v --run-e2e
+
+    # If you already have services running
+    NMP_BASE_URL=http://localhost:9090 uv run --frozen pytest e2e -v --run-e2e
+
+When ``NMP_BASE_URL`` is set the harness skips service startup/shutdown and
+connects to the given URL.  Otherwise it starts ``nemo services start`` on a
+free port, waits for ``/health/ready``, and tears the instance down after the
+session.
+"""
+
+import logging
+import os
+import socket
+import subprocess
+import uuid
+from collections.abc import Iterator
+
+import pytest
+from nemo_platform import NeMoPlatform
+
+logger = logging.getLogger(__name__)
+
+
+def _find_free_port() -> int:
+    """Bind to port 0 and let the OS assign a free port."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+@pytest.fixture(scope="session")
+def _services() -> Iterator[str]:
+    """Start ``nemo services`` and yield the base URL.
+
+    Skipped when ``NMP_BASE_URL`` is already set (external services).
+    """
+    external_url = os.environ.get("NMP_BASE_URL")
+    if external_url:
+        yield external_url
+        return
+
+    port = _find_free_port()
+    instance = f"e2e-test-{port}"
+    url = f"http://127.0.0.1:{port}"
+
+    logger.info("Starting nemo services on port %d (instance=%s)", port, instance)
+
+    result = subprocess.run(
+        [
+            "nemo",
+            "services",
+            "start",
+            "--service-group",
+            "all",
+            "--port",
+            str(port),
+            "--instance",
+            instance,
+        ],
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+
+    if result.returncode != 0:
+        pytest.fail(
+            f"nemo services start failed (exit {result.returncode}):\nstdout: {result.stdout}\nstderr: {result.stderr}"
+        )
+
+    logger.info("Platform services started: %s", result.stdout.strip())
+
+    # Set NMP_BASE_URL so the SDK picks it up automatically
+    os.environ["NMP_BASE_URL"] = url
+
+    yield url
+
+    logger.info("Stopping nemo services (instance=%s)", instance)
+    stop_result = subprocess.run(
+        [
+            "nemo",
+            "services",
+            "stop",
+            "--instance",
+            instance,
+            "--port",
+            str(port),
+        ],
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    if stop_result.returncode != 0:
+        logger.warning(
+            "nemo services stop failed (exit %d): %s",
+            stop_result.returncode,
+            stop_result.stderr,
+        )
+
+
+@pytest.fixture(scope="session")
+def sdk(_services: str) -> NeMoPlatform:
+    """Provide an SDK client connected to the running platform."""
+    return NeMoPlatform(base_url=_services, max_retries=2)
+
+
+@pytest.fixture(scope="function")
+def workspace(sdk: NeMoPlatform) -> Iterator[str]:
+    """Create a unique workspace for each test, deleted on teardown."""
+    name = f"e2e-{uuid.uuid4().hex[:8]}"
+    sdk.workspaces.create(name=name)
+    yield name
+    sdk.workspaces.delete(name)

e2e/test_smoke.py

@@ -0,0 +1,37 @@
+"""Smoke tests that verify the platform is reachable and core APIs respond.
+
+These are intentionally minimal — they validate the e2e harness works and
+that services are up. Add more substantive tests in separate files.
+"""
+
+from nemo_platform import NeMoPlatform
+
+
+def test_health_ready(sdk: NeMoPlatform):
+    """GET /health/ready returns 200 when all services are up."""
+    resp = sdk._client.get("/health/ready")
+    assert resp.status_code == 200
+
+
+def test_health_live(sdk: NeMoPlatform):
+    """GET /health/live returns 200 (liveness probe)."""
+    resp = sdk._client.get("/health/live")
+    assert resp.status_code == 200
+
+
+def test_create_and_delete_workspace(workspace: str, sdk: NeMoPlatform):
+    """Workspace CRUD round-trips through the platform.
+
+    Uses the ``workspace`` fixture which creates a unique workspace
+    and deletes it on teardown.
+    """
+    page = sdk.workspaces.list()
+    names = [w.name for w in page.data]
+    assert workspace in names
+
+
+def test_list_workspaces(sdk: NeMoPlatform, workspace: str):
+    """Listing workspaces returns at least the test workspace."""
+    page = sdk.workspaces.list()
+    names = [w.name for w in page.data]
+    assert workspace in names