diff --git a/.github/actions/changes/action.yaml b/.github/actions/changes/action.yaml index b089c8e85b..80893adba3 100644 --- a/.github/actions/changes/action.yaml +++ b/.github/actions/changes/action.yaml @@ -49,6 +49,9 @@ outputs: cpu-smoke: description: "'true' if CPU smoke image or Kubernetes smoke test inputs changed" value: ${{ steps.filter.outputs.deps == 'true' || steps.filter.outputs.docker == 'true' || steps.filter.outputs.docker-scripts == 'true' || steps.filter.outputs.helm == 'true' || steps.filter.outputs.openapi == 'true' || steps.filter.outputs.python-runtime == 'true' || steps.filter.outputs.web-studio == 'true' || steps.filter.outputs.k8s-smoke == 'true' }} + auth-idp: + description: "'true' if auth-idp tests or their containerized E2E harness inputs changed" + value: ${{ steps.filter.outputs.auth-idp == 'true' }} runs: using: "composite" @@ -110,3 +113,9 @@ runs: - '.github/scripts/ngc_metadata.py' - '.github/scripts/tests/test_ngc_metadata.py' - '.github/assets/ngc/**' + auth-idp: + - 'conftest.py' + - 'pytest.ini' + - 'tests/auth_idp/**' + - 'e2e/**' + - 'contrib/auth/**' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2cddf417d1..af23f7e033 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -46,6 +46,7 @@ jobs: cpu-smoke: ${{ steps.changes.outputs.cpu-smoke }} guardrails-benchmark: ${{ steps.changes.outputs.guardrails-benchmark }} ngc-metadata: ${{ steps.changes.outputs.ngc-metadata }} + auth-idp: ${{ steps.changes.outputs.auth-idp }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/changes @@ -124,7 +125,8 @@ jobs: if: > !cancelled() && ( github.event_name == 'workflow_dispatch' || - needs.changes.outputs.cpu-smoke == 'true' + needs.changes.outputs.cpu-smoke == 'true' || + needs.changes.outputs.auth-idp == 'true' ) runs-on: ubuntu-latest timeout-minutes: 90 @@ -891,6 +893,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Free disk space uses: ./.github/actions/free-disk-space - name: Download policy WASM @@ -935,6 +939,62 @@ jobs: coverage.xml coverage.json + python-auth-idp-test: + name: Python auth-idp tests + needs: [changes, policy-wasm, build-cpu-smoke-images] + if: > + !cancelled() && ( + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.cpu-smoke == 'true' || + needs.changes.outputs.auth-idp == 'true' + ) && + needs.policy-wasm.result == 'success' && + needs.build-cpu-smoke-images.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Free disk space + uses: ./.github/actions/free-disk-space + - name: Download policy WASM + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: policy-wasm + path: services/core/auth/src/nmp/core/auth/assets + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + python-version: "3.11" + enable-cache: true + cache-dependency-glob: uv.lock + - name: Log in to GHCR + if: needs.build-cpu-smoke-images.outputs.publish_images == 'true' + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Run auth-idp tests + run: make test-auth-idp + env: + _TYPER_FORCE_DISABLE_TERMINAL: "1" + E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs + IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }} + BAKE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }} + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-auth-idp-test-results + retention-days: 30 + path: | + ${{ runner.temp }}/e2e-services-logs/ + # Build wheels for all packages × python versions. Downstream jobs # (wheel-test, python-e2e-test) download these artifacts instead # of rebuilding. diff --git a/Makefile b/Makefile index 90c735ae52..8dccf7c7b6 100644 --- a/Makefile +++ b/Makefile @@ -122,6 +122,10 @@ docs-watch: ## Start Fern docs dev plus a repo-level watcher for docs/** changes docs-check: ## Validate the Fern docs (fern check + validate-mdx + gated-link check) cd docs/fern && npm run check +.PHONY: test-auth-idp +test-auth-idp: ## Run the auth-idp test suite + bash contrib/auth/authentik/run.sh test $(ARGS) + .PHONY: docs-check-python-snippets docs-check-python-snippets: ## Syntax-check and type-check Python snippets in one doc (DOCS_PATH=...) @if [ -z "$(strip $(DOCS_PATH))" ]; then echo "Usage: make docs-check-python-snippets DOCS_PATH=docs/customizer/tutorials/import-hf-model.mdx" >&2; exit 2; fi diff --git a/conftest.py b/conftest.py index 9c780c715f..5587105dce 100644 --- a/conftest.py +++ b/conftest.py @@ -18,6 +18,7 @@ import pytest from nmp.testing.pytest_outcomes import pytest_skip as skip_test +from tests.auth_idp.xdist import append_xdist_group_suffix from tests.discovery_exclusions import TEST_DISCOVERY_EXCLUSIONS # Set test environment variables BEFORE any imports @@ -211,6 +212,7 @@ def pytest_collection_modifyitems(config, items): category_markers = { "unit", "e2e", + "auth_idp", "smoke_gpu_tasks", "smoke_nmp_automodel_tasks", "smoke_nmp_automodel_training", @@ -247,6 +249,13 @@ def pytest_collection_modifyitems(config, items): if not marker_names.intersection(category_markers): item.add_marker(pytest.mark.unit) + if getattr(config.option, "numprocesses", None) or getattr(config, "workerinput", None) is not None: + group_names = set() + for mark in item.iter_markers("xdist_group"): + name = mark.args[0] if mark.args else mark.kwargs.get("name", "default") + group_names.add(str(name)) + item._nodeid = append_xdist_group_suffix(item.nodeid, group_names) + # ============================================================================ # Pytest command-line options @@ -295,6 +304,7 @@ def pytest_runtest_setup(item): skip_test("Skipping container-only test (requires NMP_BASE_URL)") +from xdist.scheduler.loadgroup import LoadGroupScheduling # noqa: E402 from xdist.scheduler.loadscope import LoadScopeScheduling # noqa: E402 # Temporary workaround for https://github.com/pytest-dev/pytest-xdist/issues/1189 @@ -310,3 +320,7 @@ def _patched_reschedule(self, node): LoadScopeScheduling._reschedule = _patched_reschedule # type: ignore[invalid-assignment] + + +def pytest_xdist_make_scheduler(config, log): + return LoadGroupScheduling(config, log) diff --git a/contrib/auth/README.md b/contrib/auth/README.md new file mode 100644 index 0000000000..7962cc9073 --- /dev/null +++ b/contrib/auth/README.md @@ -0,0 +1,17 @@ +# Identity Provider References + +This directory contains NeMo Platform identity-provider reference bundles. + +Each provider bundle defines one contract for local validation and production +adaptation: + +- expose OIDC discovery metadata +- include a gateway layer that strips inbound `X-NMP-Principal-*` headers +- define one human identity and one machine identity for shared auth testing +- treat external machine identities as ordinary OIDC principals authorized by + group binding, not as internal `service:*` principals +- document provider-specific setup in a local `README.md` + +Open-source providers with `mode: compose-ci` are intended for the shared auth +matrix. Reference-only providers stay documented and manifest-driven but are +excluded from the local Compose-backed matrix. diff --git a/contrib/auth/authentik/README.md b/contrib/auth/authentik/README.md new file mode 100644 index 0000000000..1a66ea4f2f --- /dev/null +++ b/contrib/auth/authentik/README.md @@ -0,0 +1,263 @@ +# Authentik Reference Example + +This directory contains a local Authentik-backed NeMo Platform example. Use it +to verify three user-visible flows: + +- log in to NeMo with Authentik +- call NeMo APIs through the Authentik gateway +- run a NeMo job whose workload uses a real Authentik workload token + +All credentials in this example are for local development only. + +## Prerequisites + +- Docker with `docker compose` +- a bootstrapped NeMo Platform checkout +- a shell from the repo root + +## Demo Identities + +The stack seeds these local-only identities: + +- Human user: `nemo-user` +- Human password: `nemo-user-password-dev` +- Human email: `nemo-user@example.com` +- CLI OIDC client: `nemo-platform-cli` +- Workload identity: `svc-nemo` +- Workload group: `nemo-editors` + +## Start The Stack + +From the repo root: + +```bash +contrib/auth/authentik/run.sh stack +``` + +This starts NeMo, Authentik, and the local gateway with the existing default +NeMo API image, `my-registry/nmp-api:local`. The `stack` action does not build +images. +Leave this process running. Stop it with `Ctrl-C` when you are done; the script +removes the Compose stack and volumes on exit. + +To use a different prebuilt image for the example, pass it explicitly: + +```bash +export IMAGE_REGISTRY=registry.example.com/nemo +export BAKE_TAG= + +contrib/auth/authentik/run.sh stack --image "$IMAGE_REGISTRY/nmp-api:$BAKE_TAG" +``` + +Use the same `IMAGE_REGISTRY` and `BAKE_TAG` values in the shell where you +submit the workload job so the job container image matches the running NeMo API +image. + +The auth-idp test suite uses the same helper script: + +```bash +contrib/auth/authentik/run.sh test +``` + +For iteration or prebuilt images, pass options to the script directly. See +`contrib/auth/authentik/run.sh --help` for the full option list. + +```bash +contrib/auth/authentik/run.sh test --lifecycle reuse +contrib/auth/authentik/run.sh test --image registry.example.com/nemo/nmp-api: +``` + +Wait until the platform is ready through the gateway: + +```bash +until curl -sf http://127.0.0.1:18080/health/ready >/dev/null; do + sleep 2 +done +echo "NeMo Platform Ready" +``` + +The local gateway URL is: + +```text +http://127.0.0.1:18080 +``` + +## Log In With Authentik + +Point the CLI at the Authentik gateway: + +```bash +nemo config set --context authentik-human --base-url http://127.0.0.1:18080 --activate +``` + +Start browser login: + +```bash +nemo auth login --context authentik-human --base-url http://127.0.0.1:18080 +``` + +Log in with: + +- username: `nemo-user` +- password: `nemo-user-password-dev` + +Verify the saved session: + +```bash +nemo --context authentik-human auth status +nemo --context authentik-human workspaces list +``` + +Expected result: `auth status` shows `Auth Type: oauth`, the email +`nemo-user@example.com`, and a refresh token. `workspaces list` should return +without an auth error. + +## Create A Demo Workspace + +```bash +export WORKSPACE=authentik-demo + +nemo --context authentik-human workspaces create "$WORKSPACE" \ + --description "Authentik reference example" \ + --wait-role-propagation +``` + +Grant the demo workload group access to the workspace: + +```bash +nemo --context authentik-human workspaces members create \ + --workspace "$WORKSPACE" \ + --principal nemo-editors \ + --roles Viewer \ + --roles JobRunner \ + --wait-role-propagation +``` + +Expected result: the human user can manage the workspace, and the workload +group can read the workspace from a job. + +## Run A Workload Job + +```bash +export JOB_NAME=authentik-workload-demo +``` + +Fetch a local demo token for the seeded workload identity: + +```bash +export WORKLOAD_ACCESS_TOKEN="$( + curl -fsS http://127.0.0.1:18080/application/o/token/ \ + -d grant_type=password \ + -d client_id=nemo-platform \ + -d client_secret=nemo-platform-secret-dev \ + -d username=svc-nemo \ + -d password=svc-nemo-token-secret-dev \ + -d scope="openid email groups" \ + | python -c 'import json, sys; print(json.load(sys.stdin)["access_token"])' +)" +``` + +Keep this token in a non-reserved shell variable. Do not export it as +`NEMO_WORKLOAD_TOKEN` in your shell. The NeMo CLI uses that variable as a +runtime credential override, which would make later CLI commands run as the +workload identity instead of `authentik-human`. + +Submit a job that runs the built-in hello-world workload auth task: + +```bash +export NMP_API_IMAGE="${NMP_API_IMAGE:-${IMAGE_REGISTRY:-my-registry}/nmp-api:${BAKE_TAG:-local}}" + +cat </nmp-api: + +Test options: + --lifecycle MODE Docker Compose lifecycle for tests: fresh or reuse. + Default: fresh. + --platform PLATFORM Platform for the default local test image build. + Default: current machine architecture. + +Other options: + --compose-dir DIR Compose directory for stack/down. Default: this script's directory. + --dry-run Print commands without running them. + -h, --help Show this help. + +Examples: + contrib/auth/authentik/run.sh stack + contrib/auth/authentik/run.sh stack --image my-registry/nmp-api:local + contrib/auth/authentik/run.sh test + contrib/auth/authentik/run.sh test --lifecycle reuse + contrib/auth/authentik/run.sh test --image my-registry/nmp-api:local + contrib/auth/authentik/run.sh down +EOF +} + +die() { + echo "error: $*" >&2 + echo >&2 + usage >&2 + exit 2 +} + +image_ref() { + printf "%s/nmp-api:%s" "${IMAGE_REGISTRY}" "${BAKE_TAG}" +} + +parse_image() { + local image="$1" + + if [[ "${image}" != */nmp-api:* ]]; then + die "--image must use the form /nmp-api:" + fi + + IMAGE_REGISTRY="${image%/nmp-api:*}" + BAKE_TAG="${image##*:}" + IMAGE_SELECTED="true" + + if [[ -z "${IMAGE_REGISTRY}" || -z "${BAKE_TAG}" ]]; then + die "--image must include both a registry path and a tag" + fi +} + +host_platform() { + case "$(uname -m)" in + x86_64 | amd64) + printf "linux/amd64" + ;; + arm64 | aarch64) + printf "linux/arm64" + ;; + *) + die "unsupported host architecture for test image build: $(uname -m). Pass --platform explicitly." + ;; + esac +} + +validate_test_lifecycle() { + case "${TEST_LIFECYCLE}" in + fresh | reuse) + ;; + *) + die "--lifecycle must be fresh or reuse" + ;; + esac +} + +quote_args() { + local arg + + for arg in "$@"; do + printf "%q " "${arg}" + done +} + +print_command_in_dir() { + local dir="$1" + shift + + printf "+ cd %q && " "${dir}" + quote_args "$@" + printf "\n" +} + +run_with_image_env_in_dir() { + local dir="$1" + shift + + if [[ "${DRY_RUN}" == "true" ]]; then + printf "+ cd %q && IMAGE_REGISTRY=%q BAKE_TAG=%q " "${dir}" "${IMAGE_REGISTRY}" "${BAKE_TAG}" + quote_args "$@" + printf "\n" + return + fi + + (cd "${dir}" && IMAGE_REGISTRY="${IMAGE_REGISTRY}" BAKE_TAG="${BAKE_TAG}" "$@") +} + +run_in_repo() { + if [[ "${DRY_RUN}" == "true" ]]; then + print_command_in_dir "${REPO_ROOT}" "$@" + return + fi + + (cd "${REPO_ROOT}" && "$@") +} + +stack_up() { + echo "Using existing NeMo API image: $(image_ref)" + + if [[ "${DRY_RUN}" == "true" ]]; then + run_with_image_env_in_dir "${COMPOSE_DIR}" docker compose up + return + fi + + trap 'run_with_image_env_in_dir "${COMPOSE_DIR}" docker compose down -v' EXIT INT TERM + run_with_image_env_in_dir "${COMPOSE_DIR}" docker compose up +} + +compose_down() { + run_with_image_env_in_dir "${COMPOSE_DIR}" docker compose down -v +} + +build_default_test_image() { + local platform="${TEST_PLATFORM:-$(host_platform)}" + echo "Building auth-idp test image for ${platform}: $(image_ref)" + run_in_repo make docker-load "DOCKER_TARGET=${TEST_DOCKER_TARGET}" "DOCKER_PLATFORMS=${platform}" +} + +run_tests() { + validate_test_lifecycle + + if [[ "${IMAGE_SELECTED}" == "true" ]]; then + echo "Using prebuilt auth-idp test image: $(image_ref)" + else + build_default_test_image + fi + + run_in_repo \ + env "IMAGE_REGISTRY=${IMAGE_REGISTRY}" "BAKE_TAG=${BAKE_TAG}" "NMP_E2E_COMPOSE_LIFECYCLE=${TEST_LIFECYCLE}" \ + uv run --frozen pytest tests/auth_idp -v --run-e2e +} + +if [[ $# -eq 0 ]]; then + usage + exit 0 +fi + +while [[ $# -gt 0 ]]; do + case "$1" in + stack | down | test) + if [[ -n "${ACTION}" ]]; then + die "only one action can be specified" + fi + ACTION="$1" + shift + ;; + --image) + [[ $# -ge 2 ]] || die "--image requires a value" + parse_image "$2" + shift 2 + ;; + --lifecycle) + [[ $# -ge 2 ]] || die "--lifecycle requires a value" + TEST_LIFECYCLE="$2" + TEST_LIFECYCLE_SET="true" + shift 2 + ;; + --platform) + [[ $# -ge 2 ]] || die "--platform requires a value" + TEST_PLATFORM="$2" + TEST_PLATFORM_SET="true" + shift 2 + ;; + --compose-dir) + [[ $# -ge 2 ]] || die "--compose-dir requires a value" + COMPOSE_DIR="$2" + COMPOSE_DIR_SET="true" + shift 2 + ;; + --dry-run) + DRY_RUN="true" + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac +done + +if [[ -z "${ACTION}" ]]; then + die "missing action" +fi + +if [[ -z "${IMAGE_REGISTRY}" || -z "${BAKE_TAG}" ]]; then + die "image registry and tag must be non-empty" +fi + +if [[ "${ACTION}" != "test" ]]; then + if [[ "${TEST_LIFECYCLE_SET}" == "true" ]]; then + die "--lifecycle is only valid with the test action" + fi + if [[ "${TEST_PLATFORM_SET}" == "true" ]]; then + die "--platform is only valid with the test action" + fi +fi + +if [[ "${ACTION}" == "test" && "${COMPOSE_DIR_SET}" == "true" ]]; then + die "--compose-dir is only valid with stack or down" +fi + +case "${ACTION}" in + stack) + stack_up + ;; + down) + compose_down + ;; + test) + run_tests + ;; +esac diff --git a/contrib/auth/authentik/seed/README.md b/contrib/auth/authentik/seed/README.md new file mode 100644 index 0000000000..4750577370 --- /dev/null +++ b/contrib/auth/authentik/seed/README.md @@ -0,0 +1,5 @@ +# Authentik Seed Notes + +Populate this directory with the provider export or bootstrap material needed to +create the demo user, demo machine identity, groups, and OAuth client described +in `manifest.yaml`. diff --git a/contrib/auth/manifest.schema.yaml b/contrib/auth/manifest.schema.yaml new file mode 100644 index 0000000000..e03d35f686 --- /dev/null +++ b/contrib/auth/manifest.schema.yaml @@ -0,0 +1,66 @@ +type: object +required: + - provider + - mode + - compose_file + - gateway_base_url + - issuer_url + - discovery_url + - nemo_config + - principal_contract + - human_identity + - workload_identity + - workload_contract + - healthchecks + - startup_timeouts +properties: + provider: + type: string + mode: + type: string + enum: + - compose-ci + - reference-only + compose_file: + type: + - string + - "null" + gateway_base_url: + type: string + issuer_url: + type: string + discovery_url: + type: string + nemo_config: + type: string + principal_contract: + type: object + required: + - subject_claim + - groups_claim + - external_machine_principals_use_service_prefix + - internal_service_prefix_reserved + human_identity: + type: object + workload_identity: + type: object + required: + - principal_id + - expected_groups + workload_contract: + type: object + required: + - audience + - principal_claim + - groups_claim + - groups_format + - token_env_vars + - forwarded_headers + healthchecks: + type: array + startup_timeouts: + type: object + required: + - healthchecks_seconds + - gateway_seconds + - token_endpoint_seconds diff --git a/docs/auth/authentication/idp-integration.mdx b/docs/auth/authentication/idp-integration.mdx new file mode 100644 index 0000000000..8e017a165e --- /dev/null +++ b/docs/auth/authentication/idp-integration.mdx @@ -0,0 +1,27 @@ +--- +title: "Integrate NeMo Platform With An Identity Provider" +description: "" +--- +NeMo Platform integrates with external OIDC identity providers. Some reference +providers live under `contrib/auth/*`. These are examples that can be adapted +to work in production. + +## Provider References + +- `contrib/auth/authentik` + +Each provider bundle includes a gateway layer that strips inbound +`X-NMP-Principal-*` headers before traffic reaches NeMo. + +For workload authentication, the gateway should validate the bearer token and +derive NeMo's trusted header contract from verified claims rather than passing +through caller-supplied identity headers. The Authentik reference demonstrates +the first-pass workload mapping: + +- `sub` -> `X-NMP-Principal-Id` +- `groups` -> `X-NMP-Principal-Groups` + +The demo workload token contract is: + +- `NEMO_WORKLOAD_TOKEN` +- `NEMO_WORKLOAD_TOKEN_FILE` diff --git a/docs/auth/authentication/index.mdx b/docs/auth/authentication/index.mdx index e6c8354f93..3d0c7c95e5 100644 --- a/docs/auth/authentication/index.mdx +++ b/docs/auth/authentication/index.mdx @@ -13,6 +13,7 @@ Start here — register an OAuth application in your IdP and configure NeMo Plat - [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Step-by-step: register an app, configure NeMo Platform, verify login. - [Azure AD (Entra ID)](/documentation/access-control/authentication/providers/azure-ad-entra-id) — Azure-specific walkthrough (app registration, scopes, claim mapping). - [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) — Checklist for any OIDC-compliant IdP. +- [IdP Integration References](/documentation/access-control/authentication/idp-integration) — Provider manifests and runnable reference bundles under `contrib/auth/*`. ## Log In and Make API Calls diff --git a/docs/auth/authorization/permissions-reference.mdx b/docs/auth/authorization/permissions-reference.mdx index 9e2d6abc6f..cfff2d86ab 100644 --- a/docs/auth/authorization/permissions-reference.mdx +++ b/docs/auth/authorization/permissions-reference.mdx @@ -11,110 +11,111 @@ For token-level access restrictions, see [API Scopes](/documentation/access-cont -PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level. +PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level. JobRunner is intended for workload identities, not interactive users. ## Entities API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| entities.(read | create | update | delete) | Read, create, update, delete entities | | | | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| entities.(read | create | update | delete) | Read, create, update, delete entities | | | | | ## Files API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| filesets.(read | list) | Read, list files | ✓ | ✓ | ✓ | -| filesets.(create | update | delete) | Create, update, delete files | | ✓ | ✓ | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| filesets.(read | list) | Read, list files | ✓ | ✓ | ✓ | | +| filesets.(create | update | delete) | Create, update, delete files | | ✓ | ✓ | | ## Guardrails API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| `guardrails.checks.exec` | Execute guardrail checks | | ✓ | ✓ | -| guardrails.configs.(read | list) | Read, list guardrails configs | ✓ | ✓ | ✓ | -| guardrails.configs.(create | update | delete) | Create, update, delete guardrails configs | | ✓ | ✓ | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| `guardrails.checks.exec` | Execute guardrail checks | | ✓ | ✓ | | +| guardrails.configs.(read | list) | Read, list guardrails configs | ✓ | ✓ | ✓ | | +| guardrails.configs.(create | update | delete) | Create, update, delete guardrails configs | | ✓ | ✓ | | ## IAM API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| iam.(read | list | create | delete) | Read, list, create, delete iam | | | ✓ | -| `iam.bundle.read` | Download OPA authorization bundle (external OPA / advanced ops) | | | | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| iam.(read | list | create | delete) | Read, list, create, delete iam | | | ✓ | | +| `iam.bundle.read` | Download OPA authorization bundle (external OPA / advanced ops) | | | | | ## Inference API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| inference.deployment-configs.(read | list) | Read, list inference deployment-configs | ✓ | ✓ | ✓ | -| inference.deployment-configs.(create | delete) | Create, delete inference deployment-configs | | ✓ | ✓ | -| inference.deployments.(read | list) | Read, list inference deployments | ✓ | ✓ | ✓ | -| inference.deployments.(create | update | delete) | Create, update, delete inference deployments | | ✓ | ✓ | -| `inference.gateway.model.exec` | Execute model gateway inference | ✓ | ✓ | ✓ | -| `inference.gateway.openai.exec` | Execute OpenAI-compatible gateway inference | ✓ | ✓ | ✓ | -| `inference.gateway.provider.exec` | Execute provider gateway inference | ✓ | ✓ | ✓ | -| inference.providers.(read | list) | Read, list inference providers | ✓ | ✓ | ✓ | -| inference.providers.(create | update | delete) | Create, update, delete inference providers | | ✓ | ✓ | -| inference.virtual-models.(read | list) | Read, list inference virtual-models | ✓ | ✓ | ✓ | -| inference.virtual-models.(create | update | delete) | Create, update, delete inference virtual-models | | ✓ | ✓ | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| inference.deployment-configs.(read | list) | Read, list inference deployment-configs | ✓ | ✓ | ✓ | | +| inference.deployment-configs.(create | delete) | Create, delete inference deployment-configs | | ✓ | ✓ | | +| inference.deployments.(read | list) | Read, list inference deployments | ✓ | ✓ | ✓ | | +| inference.deployments.(create | update | delete) | Create, update, delete inference deployments | | ✓ | ✓ | | +| `inference.gateway.model.exec` | Execute model gateway inference | ✓ | ✓ | ✓ | | +| `inference.gateway.openai.exec` | Execute OpenAI-compatible gateway inference | ✓ | ✓ | ✓ | | +| `inference.gateway.provider.exec` | Execute provider gateway inference | ✓ | ✓ | ✓ | | +| inference.providers.(read | list) | Read, list inference providers | ✓ | ✓ | ✓ | | +| inference.providers.(create | update | delete) | Create, update, delete inference providers | | ✓ | ✓ | | +| inference.virtual-models.(read | list) | Read, list inference virtual-models | ✓ | ✓ | ✓ | | +| inference.virtual-models.(create | update | delete) | Create, update, delete inference virtual-models | | ✓ | ✓ | | ## Jobs API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| jobs.(read | list) | Read, list jobs | ✓ | ✓ | ✓ | -| jobs.(create | update | delete | cancel) | Create, update, delete, cancel jobs | | ✓ | ✓ | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| jobs.(read | list) | Read, list jobs | ✓ | ✓ | ✓ | | +| jobs.(create | update | delete | cancel) | Create, update, delete, cancel jobs | | ✓ | ✓ | | +| `jobs.logs.create` | Upload job logs | | | | ✓ | ## Models API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| models.(read | list) | Read, list models | ✓ | ✓ | ✓ | -| models.(create | update | delete) | Create, update, delete models | | ✓ | ✓ | -| models.adapters.(read | list) | Read, list models adapters | ✓ | ✓ | ✓ | -| models.adapters.(create | update | delete) | Create, update, delete models adapters | | ✓ | ✓ | -| `models.prompts.read` | Read model prompts | ✓ | ✓ | ✓ | -| models.prompts.(create | update | delete) | Create, update, delete models prompts | | ✓ | ✓ | -| `models.prompts.list` | List model prompts | | | | -| `models.tool-call-plugin.set` | Whether this user can set tool_call_plugin on Models or Deployment Configs *(policy-enforced)* | | | ✓ | -| `models.trust-remote-code.set` | Whether this user can set trust_remote_code on Models *(policy-enforced)* | | | ✓ | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| models.(read | list) | Read, list models | ✓ | ✓ | ✓ | | +| models.(create | update | delete) | Create, update, delete models | | ✓ | ✓ | | +| models.adapters.(read | list) | Read, list models adapters | ✓ | ✓ | ✓ | | +| models.adapters.(create | update | delete) | Create, update, delete models adapters | | ✓ | ✓ | | +| `models.prompts.read` | Read model prompts | ✓ | ✓ | ✓ | | +| models.prompts.(create | update | delete) | Create, update, delete models prompts | | ✓ | ✓ | | +| `models.prompts.list` | List model prompts | | | | | +| `models.tool-call-plugin.set` | Whether this user can set tool_call_plugin on Models or Deployment Configs *(policy-enforced)* | | | ✓ | | +| `models.trust-remote-code.set` | Whether this user can set trust_remote_code on Models *(policy-enforced)* | | | ✓ | | ## Platform -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| `platform.admin` | Platform-wide administrative bypass *(policy-enforced)* | | | | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| `platform.admin` | Platform-wide administrative bypass *(policy-enforced)* | | | | | ## Projects API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| projects.(read | list) | Read, list projects | ✓ | ✓ | ✓ | -| projects.(create | update | delete) | Create, update, delete projects | | ✓ | ✓ | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| projects.(read | list) | Read, list projects | ✓ | ✓ | ✓ | | +| projects.(create | update | delete) | Create, update, delete projects | | ✓ | ✓ | | ## Safe Synthesizer API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| safe-synthesizer.jobs.(read | list | create | delete | cancel) | Read, list, create, delete, cancel safe synthesizer jobs | | | | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| safe-synthesizer.jobs.(read | list | create | delete | cancel) | Read, list, create, delete, cancel safe synthesizer jobs | | | | | ## Secrets API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| secrets.(read | list) | Read, list secrets | ✓ | ✓ | ✓ | -| secrets.(create | update | delete) | Create, update, delete secrets | | ✓ | ✓ | -| secrets.(access | rotate) | Access, rotate secrets | | | | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| secrets.(read | list) | Read, list secrets | ✓ | ✓ | ✓ | | +| secrets.(create | update | delete) | Create, update, delete secrets | | ✓ | ✓ | | +| secrets.(access | rotate) | Access, rotate secrets | | | | | ## Workspaces API -| Permission | Description | Viewer | Editor | Admin | -|------------|-------------|:------:|:------:|:-----:| -| workspaces.(read | list) | Read, list workspaces | ✓ | ✓ | ✓ | -| workspaces.(update | delete) | Update, delete workspaces | | ✓ | ✓ | -| `workspaces.create` | Create workspaces | | | | -| workspaces.members.(list | create | update | delete) | List, create, update, delete workspaces members | | | ✓ | -| `workspaces.members.read` | Read workspace member details | | | | +| Permission | Description | Viewer | Editor | Admin | JobRunner | +| ------------ | ------------- | :------: | :------: | :------: | :------: | +| workspaces.(read | list) | Read, list workspaces | ✓ | ✓ | ✓ | | +| workspaces.(update | delete) | Update, delete workspaces | | ✓ | ✓ | | +| `workspaces.create` | Create workspaces | | | | | +| workspaces.members.(list | create | update | delete) | List, create, update, delete workspaces members | | | ✓ | | +| `workspaces.members.read` | Read workspace member details | | | | | ## Related diff --git a/docs/auth/authorization/roles-and-permissions.mdx b/docs/auth/authorization/roles-and-permissions.mdx index 14ee5acc3c..9f6bc7d0c1 100644 --- a/docs/auth/authorization/roles-and-permissions.mdx +++ b/docs/auth/authorization/roles-and-permissions.mdx @@ -6,7 +6,7 @@ The authoritative reference for NeMo Platform roles and their permissions. For b ## Role Descriptions -NeMo Platform provides four predefined roles, each designed for a specific user persona: +NeMo Platform provides human-facing roles for interactive users and a separate workload role for job runtime identities: **Viewer** — For stakeholders who need visibility into resources but should not modify them. @@ -37,9 +37,15 @@ NeMo Platform provides four predefined roles, each designed for a specific user - Manage platform-level configuration - Create and delete any workspace +**JobRunner** — For workload identities used by running jobs, not interactive users. + +- Upload job logs produced by workload containers +- Does not include Viewer, Editor, or Admin permissions +- Should be granted alongside the minimum human-facing role the workload needs, such as Viewer when the workload only reads workspace resources + ## Role Hierarchy -Each role includes all permissions of the roles below it: +The human-facing roles include all permissions of the roles below them. JobRunner is intentionally outside this hierarchy. ```mermaid flowchart BT @@ -66,7 +72,7 @@ all operations"] ## Permission Matrix -Rows are operations; columns are roles. Read the hierarchy above first: each role inherits everything below it, and the tables call out the points where additional privileges appear. +Rows are operations; columns are human-facing roles. Read the hierarchy above first: each role inherits everything below it, and the tables call out the points where additional privileges appear. See [Permissions Reference](/documentation/access-control/authorization/permissions-reference) for workload-role permissions such as JobRunner. ### Workspace Operations @@ -103,6 +109,12 @@ Workspace creation is controlled by the `WorkspaceCreator` permission in the `sy | Cancel job | | ✓ | ✓ | ✓ | | Delete job | | ✓ | ✓ | ✓ | + + +JobRunner can upload logs from workload containers but does not grant job creation, job management, or workspace read access by itself. + + + ### Inference | Operation | Viewer | Editor | Admin | PlatformAdmin | diff --git a/docs/fern/gated-nav.yml b/docs/fern/gated-nav.yml index b9d6128dd9..3e73e8a93a 100644 --- a/docs/fern/gated-nav.yml +++ b/docs/fern/gated-nav.yml @@ -19,6 +19,8 @@ path: ../../auth/authentication/providers/azure-ad.mdx - page: Generic OIDC path: ../../auth/authentication/providers/generic.mdx + - page: IdP Integration References + path: ../../auth/authentication/idp-integration.mdx - page: Overview path: ../../auth/authentication/providers/index.mdx - page: Using Authentication diff --git a/e2e/backends/docker_compose.py b/e2e/backends/docker_compose.py new file mode 100644 index 0000000000..325382d732 --- /dev/null +++ b/e2e/backends/docker_compose.py @@ -0,0 +1,176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Literal + +import httpx + +ComposeLifecycle = Literal["fresh", "reuse"] + + +def _compose_env(env: dict[str, str] | None) -> dict[str, str]: + merged = dict(os.environ) + if env: + merged.update(env) + return merged + + +def _compose_base_args(compose_file: Path, project_name: str) -> list[str]: + return ["docker", "compose", "-f", str(compose_file), "-p", project_name] + + +def _parse_compose_ps_json(output: str) -> list[dict[str, object]]: + text = output.strip() + if not text: + return [] + try: + parsed = json.loads(text) + except json.JSONDecodeError: + parsed = [json.loads(line) for line in text.splitlines() if line.strip()] + if isinstance(parsed, dict): + return [parsed] + if isinstance(parsed, list) and all(isinstance(entry, dict) for entry in parsed): + return parsed + raise ValueError("docker compose ps did not return JSON objects") + + +def _compose_stack_readiness(entries: list[dict[str, object]], expected_services: set[str]) -> tuple[bool, list[str]]: + entries_by_service = { + str(entry.get("Service") or entry.get("Name")): entry + for entry in entries + if entry.get("Service") or entry.get("Name") + } + not_ready = [] + for service in sorted(expected_services): + entry = entries_by_service.get(service) + if entry is None: + not_ready.append(f"{service} (missing)") + continue + state = str(entry.get("State") or "").lower() + health = str(entry.get("Health") or "").lower() + if health: + if health != "healthy": + not_ready.append(f"{service} (state={state or 'unknown'}, health={health})") + elif state != "running": + not_ready.append(f"{service} (state={state or 'unknown'})") + return not not_ready, not_ready + + +class DockerComposeE2EBackend: + def __init__( + self, + *, + compose_file: Path, + config_path: Path, + project_name: str, + service_url: str, + wait_url: str | None = None, + env: dict[str, str] | None = None, + lifecycle: ComposeLifecycle = "fresh", + wait_timeout_seconds: int = 180, + ) -> None: + self.compose_file = compose_file + self.config_path = config_path + self.project_name = project_name + self.service_url = service_url + self.wait_url = wait_url or service_url + self.env = { + "NEMO_COMPOSE_CONFIG_PATH": str(config_path.resolve()), + **(env or {}), + } + self.lifecycle = lifecycle + self.wait_timeout_seconds = wait_timeout_seconds + + def _run(self, *extra_args: str, capture_output: bool = False) -> subprocess.CompletedProcess[str]: + args = _compose_base_args(self.compose_file, self.project_name) + args.extend(extra_args) + return subprocess.run( + args, + check=True, + text=True, + capture_output=capture_output, + env=_compose_env(self.env), + ) + + def _services(self, *extra_args: str) -> set[str]: + result = self._run(*extra_args, capture_output=True) + return {line for line in result.stdout.splitlines() if line} + + def _ps_entries(self) -> list[dict[str, object]]: + result = self._run("ps", "--all", "--format", "json", capture_output=True) + return _parse_compose_ps_json(result.stdout) + + def _stack_readiness(self, expected_services: set[str]) -> tuple[bool, list[str]]: + return _compose_stack_readiness(self._ps_entries(), expected_services) + + def start(self) -> None: + expected_services = self._services("config", "--services") + if not expected_services: + raise RuntimeError( + f"no services were discovered by docker compose config --services for {self.project_name}; " + "compose startup cannot proceed" + ) + if self.lifecycle == "reuse": + ready, _not_ready = self._stack_readiness(expected_services) + if ready: + self._wait_ready() + return + else: + try: + self.stop() + except subprocess.CalledProcessError: + pass + + self._run("up", "-d") + + deadline = time.monotonic() + self.wait_timeout_seconds + while time.monotonic() < deadline: + ready, _not_ready = self._stack_readiness(expected_services) + if ready: + break + time.sleep(2) + else: + _ready, not_ready = self._stack_readiness(expected_services) + raise TimeoutError(f"compose services did not become ready for {self.project_name}: {not_ready}") + + self._wait_ready() + + def _wait_ready(self) -> None: + deadline = time.monotonic() + self.wait_timeout_seconds + while time.monotonic() < deadline: + try: + response = httpx.get(self.wait_url, timeout=5) + if response.status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(2) + raise TimeoutError(f"compose backend did not become ready: {self.wait_url}") + + def stop(self) -> None: + if self.lifecycle == "reuse": + return + self._run("down", "-v") + + def write_logs(self, log_path: Path) -> None: + log_path.parent.mkdir(parents=True, exist_ok=True) + args = _compose_base_args(self.compose_file, self.project_name) + args.extend(["logs", "--no-color", "--timestamps"]) + with log_path.open("w", encoding="utf-8") as log_file: + result = subprocess.run( + args, + check=False, + text=True, + stdout=log_file, + stderr=subprocess.STDOUT, + env=_compose_env(self.env), + ) + if result.returncode != 0: + log_file.write(f"\n[docker compose logs exited with status {result.returncode}]\n") diff --git a/e2e/conftest.py b/e2e/conftest.py index 976d2c7e5d..7a54810afa 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -24,7 +24,7 @@ # Single repo-root-relative config file pytestmark = [pytest.mark.e2e_config("e2e/configs/local-subprocess.yaml")] - # Ordered config layers: files first, then inline overlays + # Ordered platform config layers: files first, then inline overlays pytestmark = [ pytest.mark.e2e_config( "e2e/configs/local-subprocess.yaml", @@ -32,6 +32,14 @@ ) ] + # Platform layers plus separate harness metadata + pytestmark = [ + pytest.mark.e2e_config( + "contrib/auth/authentik/config/platform-compose-authentik.yaml", + harness={"backend": "docker_compose", ...}, + ) + ] + Why this exists: - E2E modules should be able to declare the platform shape they need rather @@ -43,9 +51,10 @@ How pooling works: -- The harness resolves the ordered ``e2e_config(...)`` layers into one - effective config dict. -- That config is normalized into a canonical form and hashed. +- The harness resolves the ordered platform ``e2e_config(...)`` layers into one + effective config dict, and keeps any ``harness=...`` metadata separate. +- The platform config plus harness config are normalized into one canonical + pool identity and hashed. - Modules that resolve to the same hash share one running services instance for the session. - The pooled instance is shut down as soon as the last module using that hash diff --git a/e2e/services_pool.py b/e2e/services_pool.py index 79a5bb9a57..215cae3db0 100644 --- a/e2e/services_pool.py +++ b/e2e/services_pool.py @@ -16,14 +16,17 @@ from dataclasses import dataclass from importlib.metadata import entry_points from pathlib import Path -from typing import Any +from typing import Any, Callable, Literal, NotRequired, TypedDict import httpx import pytest import yaml from _pytest.nodes import Node +from nmp.testing.e2e import Docker as DockerE2EBackend from nmp.testing.e2e.config import deep_merge +from e2e.backends.docker_compose import DockerComposeE2EBackend + logger = logging.getLogger(__name__) _E2E_HARNESS_DEBUG = os.environ.get("E2E_HARNESS_DEBUG") == "1" @@ -33,6 +36,7 @@ _E2E_ADMIN_EMAIL = "admin@example.com" _E2E_REPO_ROOT = Path(__file__).resolve().parents[1] _DEFAULT_E2E_PLATFORM_CONFIG = _E2E_REPO_ROOT / "packages/nmp_platform/config/local.yaml" +_E2E_COMPOSE_LIFECYCLE_ENV = "NMP_E2E_COMPOSE_LIFECYCLE" def admin_headers() -> dict[str, str]: @@ -47,14 +51,30 @@ class ServicesPoolKey: config_hash: str +class E2EHarnessConfig(TypedDict, total=False): + backend: Literal["subprocess", "docker", "docker_compose"] + compose_file: str + compose_project_name: str + service_url: str + auth_ready_url: str + wait_url: str + lifecycle: Literal["fresh", "reuse"] + compose_project_prefix: str + env: dict[str, str] + + @dataclass class RunningServices: url: str log_path: Path | None proc: subprocess.Popen[Any] | None config_path: Path | None + close: Callable[[], None] | None = None auth_enabled: bool = False key: ServicesPoolKey | None = None + docker_network_name: str | None = None + docker_container_alias: str | None = None + docker_container_port: int | None = None @dataclass(frozen=True) @@ -63,6 +83,7 @@ class ModuleConfigState: key: ServicesPoolKey config_path: Path | None config_data: dict[str, Any] + harness_config: E2EHarnessConfig config_layers: tuple[str, ...] auth_enabled: bool @@ -116,7 +137,13 @@ def acquire_for_module(self, module: pytest.Module) -> RunningServices: services = self._running_by_key.get(state.key) if services is None: log_path = self._get_log_dir() / f"services-{state.key.config_hash}-{uuid.uuid4().hex[:8]}.log" - services = _start_services(state.config_path, state.config_data, state.key.config_hash, log_path) + services = _start_services( + state.config_path, + state.config_data, + state.harness_config, + state.key.config_hash, + log_path, + ) self._running_by_key[state.key] = services previous_key = self._active_service_key_by_module.get(module.nodeid) if previous_key is not None and previous_key != state.key: @@ -169,13 +196,15 @@ def _ensure_module_registered(self, module: pytest.Module) -> None: if module.nodeid in self._module_states: return resolved_paths, config_data = _load_effective_e2e_config_from_node(module) - key = _services_pool_key(_canonical_config_hash(config_data)) + harness_config = _resolve_e2e_harness_config_from_node(module) + key = _services_pool_key(_canonical_services_hash(config_data, harness_config)) auth_enabled = _e2e_auth_enabled(config_data) self._module_states[module.nodeid] = ModuleConfigState( module_id=module.nodeid, key=key, config_path=None, config_data=config_data, + harness_config=harness_config, config_layers=tuple(str(path) for path in resolved_paths), auth_enabled=auth_enabled, ) @@ -184,13 +213,14 @@ def _ensure_module_registered(self, module: pytest.Module) -> None: "Registered E2E module config", e2e_module=module.nodeid, config_hash=key.config_hash, + harness_backend=harness_config["backend"], config_layers=list(self._module_states[module.nodeid].config_layers), auth_enabled=auth_enabled, ) def _materialize_config_path(self, state: ModuleConfigState) -> ModuleConfigState: data_dir = e2e_services_data_dir(self._get_log_dir(), state.key.config_hash) - rendered_config_data = with_e2e_instance_paths(state.config_data, data_dir) + rendered_config_data = _render_e2e_config_for_backend(state.config_data, data_dir, state.harness_config) rendered_config = yaml.safe_dump(rendered_config_data, default_flow_style=False, sort_keys=True) config_path = self._get_generated_config_dir() / f"platform-{state.key.config_hash}.yaml" if not config_path.exists(): @@ -206,6 +236,7 @@ def _materialize_config_path(self, state: ModuleConfigState) -> ModuleConfigStat key=state.key, config_path=config_path, config_data=state.config_data, + harness_config=state.harness_config, config_layers=state.config_layers, auth_enabled=state.auth_enabled, ) @@ -226,6 +257,8 @@ def _get_log_dir(self) -> Path: @staticmethod def _terminate_services(services: RunningServices) -> None: if services.proc is None: + if services.close is not None: + services.close() return if services.proc.poll() is not None: E2EServicesPool._log_debug( @@ -257,6 +290,7 @@ def describe_module_binding( "auth_enabled": state.auth_enabled, "config_layers": list(state.config_layers), "config_path": str(state.config_path) if state.config_path is not None else None, + "harness_backend": state.harness_config["backend"], } if services is not None: details.update( @@ -264,6 +298,9 @@ def describe_module_binding( "service_url": services.url, "service_pid": services.proc.pid if services.proc is not None else None, "service_log_path": str(services.log_path) if services.log_path is not None else None, + "docker_network_name": services.docker_network_name, + "docker_container_alias": services.docker_container_alias, + "docker_container_port": services.docker_container_port, } ) return details @@ -291,6 +328,37 @@ def _resolve_e2e_config_layers_from_node(node: Node) -> list[str | dict[str, Any return layers +def _resolve_e2e_harness_config_from_node(node: Node) -> E2EHarnessConfig: + marker = node.get_closest_marker("e2e_config") + if marker is None: + return {"backend": "subprocess"} + unknown = set(marker.kwargs) - {"harness"} + if unknown: + raise pytest.UsageError(f"pytest.mark.e2e_config only supports the 'harness' keyword, got: {sorted(unknown)}") + harness = marker.kwargs.get("harness") + if harness is None: + return {"backend": "subprocess"} + if not isinstance(harness, dict): + raise pytest.UsageError("pytest.mark.e2e_config harness must be a mapping") + normalized = _normalize_config(harness) + backend = normalized.get("backend", "subprocess") + if backend not in {"subprocess", "docker", "docker_compose"}: + raise pytest.UsageError(f"unsupported e2e harness backend: {backend}") + normalized["backend"] = backend + if backend == "docker_compose": + required = {"compose_file", "service_url"} + missing = sorted(required - set(normalized)) + if missing: + raise pytest.UsageError(f"docker_compose harness config missing required keys: {missing}") + lifecycle = normalized.get("lifecycle", os.environ.get(_E2E_COMPOSE_LIFECYCLE_ENV, "fresh")) + if lifecycle not in {"fresh", "reuse"}: + raise pytest.UsageError( + f"unsupported docker_compose lifecycle from {_E2E_COMPOSE_LIFECYCLE_ENV}: {lifecycle}" + ) + normalized["lifecycle"] = lifecycle + return normalized + + def _resolve_config_path(config_ref: str) -> Path: candidate = Path(config_ref) if not candidate.is_absolute(): @@ -323,6 +391,19 @@ def _canonical_config_hash(config_data: dict[str, Any]) -> str: return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:12] +def _canonical_services_hash(config_data: dict[str, Any], harness_config: E2EHarnessConfig) -> str: + payload = json.dumps( + { + "platform": _normalize_config(config_data), + "harness": _normalize_config(harness_config), + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:12] + + def _load_effective_e2e_config_from_node(node: Node) -> tuple[list[Path], dict[str, Any]]: effective_config: dict[str, Any] = {} resolved_paths: list[Path] = [] @@ -405,6 +486,35 @@ def _real_service_plugin_allowlist() -> str | None: return ",".join(names) if names else None +def _e2e_backend(harness_config: E2EHarnessConfig) -> Literal["subprocess", "docker", "docker_compose"]: + return harness_config.get("backend", "subprocess") + + +def _render_e2e_config_for_backend( + config_data: dict[str, Any], data_dir: Path, harness_config: E2EHarnessConfig +) -> dict[str, Any]: + if _e2e_backend(harness_config) in {"docker", "docker_compose"}: + return deepcopy(config_data) + return with_e2e_instance_paths(config_data, data_dir) + + +class DockerBackendOverrides(TypedDict, total=False): + registry: str + tag: str + gpu_requested: NotRequired[bool] + + +def _docker_backend_overrides() -> DockerBackendOverrides: + registry = os.environ.get("NMP_E2E_IMAGE_REGISTRY") or os.environ.get("IMAGE_REGISTRY") + tag = os.environ.get("NMP_E2E_IMAGE_TAG") or os.environ.get("BAKE_TAG") + overrides: DockerBackendOverrides = {} + if registry: + overrides["registry"] = registry + if tag: + overrides["tag"] = tag + return overrides + + def e2e_services_env(config_path: Path, data_dir: Path) -> dict[str, str]: """Environment for the ``nemo services run`` child process.""" env = os.environ.copy() @@ -455,10 +565,10 @@ def _wait_for_healthy(url: str, proc: subprocess.Popen[Any], timeout: float = _H return False -def _wait_for_auth_ready(url: str, proc: subprocess.Popen[Any], timeout: float = _AUTH_READY_TIMEOUT) -> bool: +def _wait_for_auth_ready(url: str, proc: subprocess.Popen[Any] | None, timeout: float = _AUTH_READY_TIMEOUT) -> bool: deadline = time.monotonic() + timeout while time.monotonic() < deadline: - if _process_exited(proc): + if proc is not None and _process_exited(proc): return False probe_name = f"auth-probe-{uuid.uuid4().hex[:8]}" entity_name = f"auth-probe-entity-{uuid.uuid4().hex[:8]}" @@ -470,7 +580,7 @@ def _wait_for_auth_ready(url: str, proc: subprocess.Popen[Any], timeout: float = timeout=5.0, ) if create_resp.status_code != 201: - if _process_exited(proc): + if proc is not None and _process_exited(proc): return False time.sleep(_HEALTH_POLL_INTERVAL) continue @@ -503,13 +613,28 @@ def _wait_for_auth_ready(url: str, proc: subprocess.Popen[Any], timeout: float = return True except httpx.RequestError as exc: logger.debug("Auth readiness probe failed; will retry: %s", exc) - if _process_exited(proc): + if proc is not None and _process_exited(proc): return False time.sleep(_HEALTH_POLL_INTERVAL) return False def _start_services( + config_path: Path, + config_data: dict[str, Any], + harness_config: E2EHarnessConfig, + config_hash: str, + log_path: Path, +) -> RunningServices: + backend = _e2e_backend(harness_config) + if backend == "docker": + return _start_services_docker(config_path, config_data, config_hash) + if backend == "docker_compose": + return _start_services_docker_compose(config_path, config_data, harness_config, config_hash, log_path) + return _start_services_subprocess(config_path, config_data, config_hash, log_path) + + +def _start_services_subprocess( config_path: Path, config_data: dict[str, Any], config_hash: str, log_path: Path ) -> RunningServices: port = _find_free_port() @@ -570,3 +695,102 @@ def _start_services( auth_enabled=auth_enabled, key=_services_pool_key(config_hash), ) + + +def _start_services_docker(config_path: Path, config_data: dict[str, Any], config_hash: str) -> RunningServices: + backend = DockerE2EBackend(config_path=config_path, **_docker_backend_overrides()) + try: + backend.start() + except Exception: + backend.stop() + raise + + auth_enabled = _e2e_auth_enabled(config_data) + if auth_enabled and not _wait_for_auth_ready(backend.base_url, None): + backend.stop() + pytest.fail(f"Platform auth seed did not become ready within {_AUTH_READY_TIMEOUT}s.") + + services = RunningServices( + url=backend.base_url, + log_path=None, + proc=None, + config_path=config_path, + close=backend.stop, + auth_enabled=auth_enabled, + key=_services_pool_key(config_hash), + docker_network_name=backend.network_name, + docker_container_alias=backend.network_alias, + docker_container_port=backend.container_port, + ) + + return services + + +def _start_services_docker_compose( + config_path: Path, + config_data: dict[str, Any], + harness_config: E2EHarnessConfig, + config_hash: str, + log_path: Path, +) -> RunningServices: + compose_file = _resolve_config_path(harness_config["compose_file"]) + project_name = harness_config.get("compose_project_name") + if project_name is None: + project_prefix = harness_config.get("compose_project_prefix", "e2e-compose") + project_name = f"{project_prefix}-{config_hash}" + service_url = harness_config["service_url"] + wait_url = harness_config.get("wait_url") + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name=project_name, + service_url=service_url, + wait_url=wait_url, + env=harness_config.get("env"), + lifecycle=harness_config["lifecycle"], + ) + try: + backend.start() + except Exception: + _write_docker_compose_logs(backend, log_path) + backend.stop() + raise + + auth_enabled = _e2e_auth_enabled(config_data) + auth_ready_url = harness_config.get("auth_ready_url", backend.service_url) + if auth_enabled and not _wait_for_auth_ready(auth_ready_url, None): + _write_docker_compose_logs(backend, log_path) + backend.stop() + pytest.fail( + f"Platform auth seed did not become ready within {_AUTH_READY_TIMEOUT}s.\nlog:\n{_read_log_text(log_path)}" + ) + + def close() -> None: + try: + _write_docker_compose_logs(backend, log_path) + finally: + backend.stop() + + return RunningServices( + url=backend.service_url, + log_path=log_path, + proc=None, + config_path=config_path, + close=close, + auth_enabled=auth_enabled, + key=_services_pool_key(config_hash), + ) + + +def _write_docker_compose_logs(backend: DockerComposeE2EBackend, log_path: Path) -> None: + try: + backend.write_logs(log_path) + except Exception: + logger.exception("Could not write docker compose services log", extra={"log_path": str(log_path)}) + + +def _read_log_text(log_path: Path) -> str: + try: + return log_path.read_text() + except OSError as exc: + return f"" diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py index 7b22c0e838..888dd8ebc0 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/auth.py @@ -50,6 +50,16 @@ def is_auth_disabled(base_url: str, timeout: float = 3.0) -> bool | None: return None +def _runtime_token_source_label() -> str | None: + """Return the runtime token override source using the same precedence as config loading.""" + from nemo_platform_ext.config.config import Config + + try: + return Config.runtime_access_token_source_label() + except ValueError: + return "NEMO_WORKLOAD_TOKEN_FILE environment override could not be read" + + def ensure_valid_token(context: Context, refresh_buffer_seconds: int = 300) -> bool: """ Check if the current token is valid and refresh if needed. @@ -504,6 +514,11 @@ def login( console.print(" Refresh token: [yellow]not available[/] (add 'offline_access' scope to enable)") console.print("\n[bold green]Credentials saved to config file.[/]") + if runtime_token_source := _runtime_token_source_label(): + console.print( + f"[yellow]Warning:[/] {runtime_token_source} is active and will override these saved credentials. " + "Unset the runtime token override to use this login for future commands." + ) console.print("\n[dim]Run 'nemo workspaces list' to verify your access.[/]") @@ -514,6 +529,7 @@ def logout(ctx: typer.Context) -> None: from rich.console import Console from nemo_platform_ext.config.config import Config + from nemo_platform_ext.config.models import NoAuthUser cli_context: CLIContext = ctx.obj context = cli_context.get_sdk_context() @@ -526,8 +542,32 @@ def logout(ctx: typer.Context) -> None: return logout_params: ConfigParams = {"access_token": None, "refresh_token": None} - Config.write(logout_params, context_name=context.context_name) + updated_config = Config.write(logout_params, context_name=context.context_name) + config_path = Config.get_default_config_path() + if isinstance(updated_config, Config): + config_path = updated_config.get_config_path() or config_path + persisted_config = Config.load(config_path=config_path).get_config_file() + persisted_context = next((ctx for ctx in persisted_config.contexts if ctx.name == context.context_name), None) + persisted_user = None + if persisted_context is not None: + persisted_user = next( + (user for user in persisted_config.users if user.name == persisted_context.user), None + ) + + if persisted_context is None or not isinstance(persisted_user, NoAuthUser): + raise AuthError( + "Logout did not clear credentials for " + f"context '{context.context_name}' in config file '{config_path.name}' at {config_path}. " + "Run 'nemo auth status' and check the Config File and Credential Source rows." + ) + console.print("[green]Logged out successfully.[/]") + console.print(f" Context: [cyan]{context.context_name}[/]") + console.print(f" Config file: [cyan]{config_path}[/]") + if runtime_token_source := _runtime_token_source_label(): + console.print( + f" [yellow]Warning:[/] {runtime_token_source} is still active and will override saved credentials." + ) @app.command("refresh") @@ -713,6 +753,7 @@ def status(ctx: typer.Context) -> None: from rich.console import Console from rich.table import Table + from nemo_platform_ext.config.config import Config from nemo_platform_ext.config.models import OAuthUser cli_context: CLIContext = ctx.obj @@ -733,13 +774,22 @@ def status(ctx: typer.Context) -> None: table = Table(title="Authentication Status", show_header=False) table.add_column("Property", style="cyan") - table.add_column("Value") + table.add_column("Value", overflow="fold") table.add_row("Cluster", str(context.cluster.base_url)) table.add_row("Context", context.context_name) + table.add_row("Config File", str(Config.get_default_config_path())) + runtime_token_source = _runtime_token_source_label() if context.user: - table.add_row("Auth Type", context.user.type) + if runtime_token_source: + table.add_row( + "Credential Source", + f"[yellow]{runtime_token_source}[/]", + ) + else: + table.add_row("Auth Type", context.user.type) + table.add_row("Credential Source", "config file") if isinstance(context.user, OAuthUser): # OAuth token authentication @@ -751,27 +801,28 @@ def status(ctx: typer.Context) -> None: if claims: # Show decoded JWT info - email = claims.get("upn") or claims.get("email") or claims.get("preferred_username") - if email: - table.add_row("Email", email) - - subject = claims.get("oid") or claims.get("sub") - if subject: - table.add_row("User ID", subject) - - scopes = claims.get("scp") or claims.get("scope") or "" - if isinstance(scopes, str): - scopes = scopes.split() - if scopes: - table.add_row("Scopes", " ".join(scopes)) - else: - table.add_row("Scopes", "[dim]none[/]") + if not runtime_token_source: + email = claims.get("upn") or claims.get("email") or claims.get("preferred_username") + if email: + table.add_row("Email", email) + + subject = claims.get("oid") or claims.get("sub") + if subject: + table.add_row("User ID", subject) + + scopes = claims.get("scp") or claims.get("scope") or "" + if isinstance(scopes, str): + scopes = scopes.split() + if scopes: + table.add_row("Scopes", " ".join(scopes)) + else: + table.add_row("Scopes", "[dim]none[/]") - groups = claims.get("groups") or claims.get("cognito:groups") or [] - if isinstance(groups, str): - groups = [groups] - if groups: - table.add_row("Groups", ", ".join(groups)) + groups = claims.get("groups") or claims.get("cognito:groups") or [] + if isinstance(groups, str): + groups = [groups] + if groups: + table.add_row("Groups", ", ".join(groups)) exp = claims.get("exp") if exp: @@ -785,11 +836,12 @@ def status(ctx: typer.Context) -> None: else: table.add_row("Expires", f"[red]EXPIRED[/] ({exp_dt.isoformat()})") - # Show refresh token status - if context.user.refresh_token: - table.add_row("Refresh Token", "[green]available[/] (run 'nemo auth refresh' to renew)") - else: - table.add_row("Refresh Token", "[yellow]not available[/]") + if not runtime_token_source: + # Show refresh token status for saved credentials only. + if context.user.refresh_token: + table.add_row("Refresh Token", "[green]available[/] (run 'nemo auth refresh' to renew)") + else: + table.add_row("Refresh Token", "[yellow]not available[/]") # Show redacted token redacted = f"{token_value[:20]}...{token_value[-10:]}" if len(token_value) > 30 else "***" diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py b/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py index 78f694946b..4365b889c2 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/config/config.py @@ -8,6 +8,7 @@ import logging import os import stat +from dataclasses import dataclass from pathlib import Path import yaml @@ -28,6 +29,15 @@ logger = logging.getLogger(__name__) +_WORKLOAD_TOKEN_ENVVAR = "NEMO_WORKLOAD_TOKEN" +_WORKLOAD_TOKEN_FILE_ENVVAR = "NEMO_WORKLOAD_TOKEN_FILE" + + +@dataclass(frozen=True) +class _RuntimeAccessTokenSource: + token: str + label: str + class Config(BaseModel): """ @@ -127,6 +137,30 @@ def _migrate_legacy_api_key_users(cls, config_data: dict) -> None: if migrated_count: logger.warning("Migrated %s legacy api-key user(s) to oauth users", migrated_count) + @classmethod + def _runtime_access_token_source_from_env(cls) -> _RuntimeAccessTokenSource | None: + if token := os.environ.get("NMP_ACCESS_TOKEN"): + return _RuntimeAccessTokenSource(token, "NMP_ACCESS_TOKEN environment override") + if token := os.environ.get(_WORKLOAD_TOKEN_ENVVAR): + return _RuntimeAccessTokenSource(token, f"{_WORKLOAD_TOKEN_ENVVAR} environment override") + if token_path := os.environ.get(_WORKLOAD_TOKEN_FILE_ENVVAR): + try: + token = Path(token_path).read_text(encoding="utf-8").strip() + except OSError as exc: + raise ValueError(f"Unable to read {_WORKLOAD_TOKEN_FILE_ENVVAR} at {token_path}: {exc}") from exc + if token: + return _RuntimeAccessTokenSource( + token, + f"{_WORKLOAD_TOKEN_FILE_ENVVAR} environment override ({token_path})", + ) + return None + + @classmethod + def runtime_access_token_source_label(cls) -> str | None: + """Return the effective runtime access token override source label.""" + source = cls._runtime_access_token_source_from_env() + return source.label if source else None + @classmethod def _load_from_env(cls) -> dict[str, object]: """Load configuration from environment variables with NMP_ prefix.""" @@ -135,6 +169,9 @@ def _load_from_env(cls) -> dict[str, object]: env_key = f"NMP_{field_name.upper()}" if val := os.environ.get(env_key): env_values[field_name] = val + if "access_token" not in env_values: + if source := cls._runtime_access_token_source_from_env(): + env_values["access_token"] = source.token return env_values @classmethod diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_auth.py b/packages/nemo_platform_ext/tests/cli/commands/test_auth.py index c6053903f4..f4ef89cb56 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/test_auth.py +++ b/packages/nemo_platform_ext/tests/cli/commands/test_auth.py @@ -67,6 +67,9 @@ def _decode_jwt_noop(token: str) -> dict: @pytest.fixture def oauth_config_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + for env_key in ("NMP_ACCESS_TOKEN", "NEMO_WORKLOAD_TOKEN", "NEMO_WORKLOAD_TOKEN_FILE"): + monkeypatch.delenv(env_key, raising=False) + config_data = { "current_context": "default", "clusters": [ @@ -133,6 +136,8 @@ def test_auth_logout_clears_selected_context_credentials( assert_exit_code(result, 0) assert "Logged out successfully" in result.output + assert "Context: foo" in result.output + assert "Config file:" in result.output with open(oauth_config_file) as f: data = yaml.safe_load(f) @@ -148,6 +153,43 @@ def test_auth_logout_clears_selected_context_credentials( assert "refresh_token" not in foo_user +def test_auth_logout_warns_when_runtime_token_override_remains( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + monkeypatch.setenv( + "NEMO_WORKLOAD_TOKEN", + generate_unsigned_jwt( + principal_id="svc-nemo-ci", + email="svc-nemo-ci@example.com", + expires_in_seconds=900, + ), + ) + + result = runner.invoke(app, ["--context", "foo", "auth", "logout"]) + + assert_exit_code(result, 0) + assert "Logged out successfully" in result.output + assert "NEMO_WORKLOAD_TOKEN environment override is still active" in result.output + + +def test_auth_logout_fails_if_credentials_remain(oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from nemo_platform_ext.config.config import Config + + def fake_write(*args, **kwargs): + return Config.load(config_path=oauth_config_file) + + monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + monkeypatch.setattr("nemo_platform_ext.config.config.Config.write", fake_write) + + result = runner.invoke(app, ["--context", "foo", "auth", "logout"]) + + assert_exit_code(result, 1) + assert "Logout did not clear credentials" in result.output + assert "context 'foo'" in result.output + assert oauth_config_file.name in result.output + + # --------------------------------------------------------------------------- # refresh # --------------------------------------------------------------------------- @@ -266,6 +308,33 @@ def test_auth_status_shows_warning_for_unsigned_token(oauth_config_file: Path, m assert "local/testing" in result.output +def test_runtime_token_source_label_handles_unreadable_token_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from nemo_platform_ext.cli.commands.auth import _runtime_token_source_label + + token_file = tmp_path / "missing-workload-token.jwt" + monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False) + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN", raising=False) + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_file)) + + assert _runtime_token_source_label() == "NEMO_WORKLOAD_TOKEN_FILE environment override could not be read" + + +def test_auth_status_shows_config_file_credential_source( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + + result = runner.invoke(app, ["--context", "foo", "auth", "status"]) + + assert_exit_code(result, 0) + assert "Config File" in result.output + assert oauth_config_file.name in result.output + assert "Credential Source" in result.output + assert "config file" in result.output + + # --------------------------------------------------------------------------- # login # --------------------------------------------------------------------------- @@ -351,6 +420,52 @@ def password_grant(**kwargs) -> SimpleNamespace: assert foo_user["refresh_token"] == "foo-refresh-token" +def test_auth_login_warns_when_env_access_token_will_override_saved_credentials( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def password_grant(**kwargs) -> SimpleNamespace: + return SimpleNamespace(token_for_nmp="foo-access-token", refresh_token="foo-refresh-token") + + monkeypatch.setenv( + "NMP_ACCESS_TOKEN", + generate_unsigned_jwt( + principal_id="svc-nemo-ci", + email="svc-nemo-ci@example.com", + expires_in_seconds=900, + ), + ) + monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.discover_nmp_config", _discover_oidc_config) + monkeypatch.setattr("nemo_platform_ext.auth.device_flow.authenticate_with_password_grant", password_grant) + monkeypatch.setattr("nemo_platform_ext.cli.commands.auth.decode_jwt_claims", _decode_jwt_noop) + + result = runner.invoke( + app, + [ + "auth", + "login", + "--context", + "foo", + "--base-url", + "https://foo-updated.example.com", + "--username", + "user", + "--password", + "secret", + ], + ) + + assert_exit_code(result, 0) + assert "NMP_ACCESS_TOKEN environment override is active" in result.output + assert "Unset the runtime token override" in result.output + + with open(oauth_config_file) as f: + data = yaml.safe_load(f) + + foo_user = next(user for user in data["users"] if user["name"] == "foo") + assert foo_user["token"] == "foo-access-token" + assert foo_user["refresh_token"] == "foo-refresh-token" + + def test_auth_login_with_base_url_creates_selected_context(oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch): def password_grant(**kwargs) -> SimpleNamespace: return SimpleNamespace(token_for_nmp="dev-access-token", refresh_token="dev-refresh-token") diff --git a/packages/nemo_platform_ext/tests/config/test_config.py b/packages/nemo_platform_ext/tests/config/test_config.py index 4767cd9f12..87889f9955 100644 --- a/packages/nemo_platform_ext/tests/config/test_config.py +++ b/packages/nemo_platform_ext/tests/config/test_config.py @@ -337,6 +337,61 @@ def test_config_from_env_access_token_only(self, tmp_path: Path, monkeypatch: py assert config.user.refresh_token is None assert not hasattr(config.user, "token_endpoint") + def test_config_from_workload_token_env_only(self, monkeypatch: pytest.MonkeyPatch): + """NEMO_WORKLOAD_TOKEN should bootstrap OAuth auth without a config file.""" + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + + config = get_context() + + assert isinstance(config.user, OAuthUser) + assert config.user.token.get_secret_value() == "workload-token-123" + + def test_config_from_workload_token_file_env_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """NEMO_WORKLOAD_TOKEN_FILE should bootstrap OAuth auth without a config file.""" + token_path = tmp_path / "workload.token" + token_path.write_text("workload-token-from-file\n", encoding="utf-8") + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_path)) + + config = get_context() + + assert isinstance(config.user, OAuthUser) + assert config.user.token.get_secret_value() == "workload-token-from-file" + + def test_config_from_missing_workload_token_file_reports_configuration_error( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + """NEMO_WORKLOAD_TOKEN_FILE should fail clearly when the configured token file cannot be read.""" + token_path = tmp_path / "missing-workload.token" + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_path)) + + with pytest.raises(ValueError, match="NEMO_WORKLOAD_TOKEN_FILE"): + get_context() + + def test_nmp_access_token_precedes_workload_token_env(self, monkeypatch: pytest.MonkeyPatch): + """NMP_ACCESS_TOKEN remains the highest-precedence token env var.""" + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NMP_ACCESS_TOKEN", "preferred-token") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + + config = get_context() + + assert isinstance(config.user, OAuthUser) + assert config.user.token.get_secret_value() == "preferred-token" + + def test_runtime_access_token_source_label_uses_config_precedence( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + """Runtime token source labels should share Config's token env precedence.""" + missing_token_path = tmp_path / "missing-workload.token" + monkeypatch.setenv("NMP_ACCESS_TOKEN", "preferred-token") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(missing_token_path)) + + assert Config.runtime_access_token_source_label() == "NMP_ACCESS_TOKEN environment override" + def test_config_from_env_access_token_ignores_legacy_api_key_env(self, monkeypatch: pytest.MonkeyPatch): """NMP_ACCESS_TOKEN is used when legacy NMP_API_KEY is present without config.""" monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") diff --git a/packages/nmp_testing/src/nmp/testing/e2e/docker.py b/packages/nmp_testing/src/nmp/testing/e2e/docker.py index 66a3bd67a1..42bf0ebac9 100644 --- a/packages/nmp_testing/src/nmp/testing/e2e/docker.py +++ b/packages/nmp_testing/src/nmp/testing/e2e/docker.py @@ -15,6 +15,7 @@ import shutil import tempfile import time +import uuid from pathlib import Path from typing import TYPE_CHECKING @@ -35,6 +36,7 @@ HEALTH_ENDPOINT = "/health/ready" STARTUP_TIMEOUT_SECONDS = 60 NMP_API_NETWORK_ALIAS = "nmp-quickstart" +NMP_API_CONTAINER_NAME_PREFIX = "nmp-api-test" # Docker client timeout in seconds. This needs to be higher than the default 60s # to handle Docker-in-Docker (DinD) environments in CI where the Docker daemon @@ -42,6 +44,11 @@ DOCKER_CLIENT_TIMEOUT_SECONDS = 180 +def _api_container_name() -> str: + """Return a readable, collision-resistant Docker container name for NeMo E2E.""" + return f"{NMP_API_CONTAINER_NAME_PREFIX}-{uuid.uuid4().hex[:8]}" + + class Docker(E2EBackend): """Docker-based test backend using testcontainers. @@ -100,6 +107,7 @@ def start(self) -> None: docker_client_kw={"timeout": DOCKER_CLIENT_TIMEOUT_SECONDS}, ) self.container.with_kwargs(init=True) + self.container.with_name(_api_container_name()) self.container.with_network(self.network) self.container.with_network_aliases(NMP_API_NETWORK_ALIAS) self.container.with_exposed_ports(CONTAINER_PORT) @@ -441,6 +449,21 @@ def get_sdk(self, principal_id: str | None = None) -> NeMoPlatform: headers = {"X-NMP-Principal-Id": principal_id} if principal_id else None return NeMoPlatform(base_url=base_url, default_headers=headers) + @property + def network_name(self) -> str | None: + """Get the dedicated Docker network name for this backend.""" + return self.network.name if self.network is not None else None + + @property + def network_alias(self) -> str: + """Get the stable network alias used by sibling containers.""" + return NMP_API_NETWORK_ALIAS + + @property + def container_port(self) -> int: + """Get the internal API port exposed within the Docker network.""" + return CONTAINER_PORT + @property def base_url(self) -> str: """Get the base URL for the running container. diff --git a/pytest.ini b/pytest.ini index 76a755d178..cc60fa0df8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -62,7 +62,8 @@ markers = smoke_nmp_automodel_tasks: Import smoke tests for the nmp-automodel-tasks image smoke_nmp_automodel_training: Import smoke tests for the nmp-automodel-training image e2e: End-to-end tests - test complete customer workflows on deployed infrastructure (Helm/Docker Compose) - e2e_config(*layers): Ordered list of repo-root-relative config paths and/or inline dict overlays; empty means default local config + auth_idp: Auth IdP e2e tests - provider-backed auth compose coverage through the gateway + e2e_config(*layers, harness=...): Ordered list of repo-root-relative config paths and/or inline dict overlays; harness config stays separate from platform config subprocess_only: Test only works in subprocess mode (not on Kubernetes); skipped when NMP_BASE_URL is set container_only: Test requires a container backend (Docker or Kubernetes); skipped unless NMP_BASE_URL is set regression: Regression tests - test individual functional microservices for baseline functionality diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py index a3498e967f..a8a6b0aca5 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/auth.py @@ -50,6 +50,16 @@ def is_auth_disabled(base_url: str, timeout: float = 3.0) -> bool | None: return None +def _runtime_token_source_label() -> str | None: + """Return the runtime token override source using the same precedence as config loading.""" + from nemo_platform.config.config import Config + + try: + return Config.runtime_access_token_source_label() + except ValueError: + return "NEMO_WORKLOAD_TOKEN_FILE environment override could not be read" + + def ensure_valid_token(context: Context, refresh_buffer_seconds: int = 300) -> bool: """ Check if the current token is valid and refresh if needed. @@ -504,6 +514,11 @@ def login( console.print(" Refresh token: [yellow]not available[/] (add 'offline_access' scope to enable)") console.print("\n[bold green]Credentials saved to config file.[/]") + if runtime_token_source := _runtime_token_source_label(): + console.print( + f"[yellow]Warning:[/] {runtime_token_source} is active and will override these saved credentials. " + "Unset the runtime token override to use this login for future commands." + ) console.print("\n[dim]Run 'nemo workspaces list' to verify your access.[/]") @@ -514,6 +529,7 @@ def logout(ctx: typer.Context) -> None: from rich.console import Console from nemo_platform.config.config import Config + from nemo_platform.config.models import NoAuthUser cli_context: CLIContext = ctx.obj context = cli_context.get_sdk_context() @@ -526,8 +542,32 @@ def logout(ctx: typer.Context) -> None: return logout_params: ConfigParams = {"access_token": None, "refresh_token": None} - Config.write(logout_params, context_name=context.context_name) + updated_config = Config.write(logout_params, context_name=context.context_name) + config_path = Config.get_default_config_path() + if isinstance(updated_config, Config): + config_path = updated_config.get_config_path() or config_path + persisted_config = Config.load(config_path=config_path).get_config_file() + persisted_context = next((ctx for ctx in persisted_config.contexts if ctx.name == context.context_name), None) + persisted_user = None + if persisted_context is not None: + persisted_user = next( + (user for user in persisted_config.users if user.name == persisted_context.user), None + ) + + if persisted_context is None or not isinstance(persisted_user, NoAuthUser): + raise AuthError( + "Logout did not clear credentials for " + f"context '{context.context_name}' in config file '{config_path.name}' at {config_path}. " + "Run 'nemo auth status' and check the Config File and Credential Source rows." + ) + console.print("[green]Logged out successfully.[/]") + console.print(f" Context: [cyan]{context.context_name}[/]") + console.print(f" Config file: [cyan]{config_path}[/]") + if runtime_token_source := _runtime_token_source_label(): + console.print( + f" [yellow]Warning:[/] {runtime_token_source} is still active and will override saved credentials." + ) @app.command("refresh") @@ -713,6 +753,7 @@ def status(ctx: typer.Context) -> None: from rich.console import Console from rich.table import Table + from nemo_platform.config.config import Config from nemo_platform.config.models import OAuthUser cli_context: CLIContext = ctx.obj @@ -733,13 +774,22 @@ def status(ctx: typer.Context) -> None: table = Table(title="Authentication Status", show_header=False) table.add_column("Property", style="cyan") - table.add_column("Value") + table.add_column("Value", overflow="fold") table.add_row("Cluster", str(context.cluster.base_url)) table.add_row("Context", context.context_name) + table.add_row("Config File", str(Config.get_default_config_path())) + runtime_token_source = _runtime_token_source_label() if context.user: - table.add_row("Auth Type", context.user.type) + if runtime_token_source: + table.add_row( + "Credential Source", + f"[yellow]{runtime_token_source}[/]", + ) + else: + table.add_row("Auth Type", context.user.type) + table.add_row("Credential Source", "config file") if isinstance(context.user, OAuthUser): # OAuth token authentication @@ -751,27 +801,28 @@ def status(ctx: typer.Context) -> None: if claims: # Show decoded JWT info - email = claims.get("upn") or claims.get("email") or claims.get("preferred_username") - if email: - table.add_row("Email", email) - - subject = claims.get("oid") or claims.get("sub") - if subject: - table.add_row("User ID", subject) - - scopes = claims.get("scp") or claims.get("scope") or "" - if isinstance(scopes, str): - scopes = scopes.split() - if scopes: - table.add_row("Scopes", " ".join(scopes)) - else: - table.add_row("Scopes", "[dim]none[/]") + if not runtime_token_source: + email = claims.get("upn") or claims.get("email") or claims.get("preferred_username") + if email: + table.add_row("Email", email) + + subject = claims.get("oid") or claims.get("sub") + if subject: + table.add_row("User ID", subject) + + scopes = claims.get("scp") or claims.get("scope") or "" + if isinstance(scopes, str): + scopes = scopes.split() + if scopes: + table.add_row("Scopes", " ".join(scopes)) + else: + table.add_row("Scopes", "[dim]none[/]") - groups = claims.get("groups") or claims.get("cognito:groups") or [] - if isinstance(groups, str): - groups = [groups] - if groups: - table.add_row("Groups", ", ".join(groups)) + groups = claims.get("groups") or claims.get("cognito:groups") or [] + if isinstance(groups, str): + groups = [groups] + if groups: + table.add_row("Groups", ", ".join(groups)) exp = claims.get("exp") if exp: @@ -785,11 +836,12 @@ def status(ctx: typer.Context) -> None: else: table.add_row("Expires", f"[red]EXPIRED[/] ({exp_dt.isoformat()})") - # Show refresh token status - if context.user.refresh_token: - table.add_row("Refresh Token", "[green]available[/] (run 'nemo auth refresh' to renew)") - else: - table.add_row("Refresh Token", "[yellow]not available[/]") + if not runtime_token_source: + # Show refresh token status for saved credentials only. + if context.user.refresh_token: + table.add_row("Refresh Token", "[green]available[/] (run 'nemo auth refresh' to renew)") + else: + table.add_row("Refresh Token", "[yellow]not available[/]") # Show redacted token redacted = f"{token_value[:20]}...{token_value[-10:]}" if len(token_value) > 30 else "***" diff --git a/sdk/python/nemo-platform/src/nemo_platform/config/config.py b/sdk/python/nemo-platform/src/nemo_platform/config/config.py index 9621f4e2b1..ac8b2f70b9 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/config/config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/config/config.py @@ -8,6 +8,7 @@ import logging import os import stat +from dataclasses import dataclass from pathlib import Path import yaml @@ -28,6 +29,15 @@ logger = logging.getLogger(__name__) +_WORKLOAD_TOKEN_ENVVAR = "NEMO_WORKLOAD_TOKEN" +_WORKLOAD_TOKEN_FILE_ENVVAR = "NEMO_WORKLOAD_TOKEN_FILE" + + +@dataclass(frozen=True) +class _RuntimeAccessTokenSource: + token: str + label: str + class Config(BaseModel): """ @@ -127,6 +137,30 @@ def _migrate_legacy_api_key_users(cls, config_data: dict) -> None: if migrated_count: logger.warning("Migrated %s legacy api-key user(s) to oauth users", migrated_count) + @classmethod + def _runtime_access_token_source_from_env(cls) -> _RuntimeAccessTokenSource | None: + if token := os.environ.get("NMP_ACCESS_TOKEN"): + return _RuntimeAccessTokenSource(token, "NMP_ACCESS_TOKEN environment override") + if token := os.environ.get(_WORKLOAD_TOKEN_ENVVAR): + return _RuntimeAccessTokenSource(token, f"{_WORKLOAD_TOKEN_ENVVAR} environment override") + if token_path := os.environ.get(_WORKLOAD_TOKEN_FILE_ENVVAR): + try: + token = Path(token_path).read_text(encoding="utf-8").strip() + except OSError as exc: + raise ValueError(f"Unable to read {_WORKLOAD_TOKEN_FILE_ENVVAR} at {token_path}: {exc}") from exc + if token: + return _RuntimeAccessTokenSource( + token, + f"{_WORKLOAD_TOKEN_FILE_ENVVAR} environment override ({token_path})", + ) + return None + + @classmethod + def runtime_access_token_source_label(cls) -> str | None: + """Return the effective runtime access token override source label.""" + source = cls._runtime_access_token_source_from_env() + return source.label if source else None + @classmethod def _load_from_env(cls) -> dict[str, object]: """Load configuration from environment variables with NMP_ prefix.""" @@ -135,6 +169,9 @@ def _load_from_env(cls) -> dict[str, object]: env_key = f"NMP_{field_name.upper()}" if val := os.environ.get(env_key): env_values[field_name] = val + if "access_token" not in env_values: + if source := cls._runtime_access_token_source_from_env(): + env_values["access_token"] = source.token return env_values @classmethod diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py index bc8fd32ca2..fc148fb5e8 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_auth.py @@ -67,6 +67,9 @@ def _decode_jwt_noop(token: str) -> dict: @pytest.fixture def oauth_config_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + for env_key in ("NMP_ACCESS_TOKEN", "NEMO_WORKLOAD_TOKEN", "NEMO_WORKLOAD_TOKEN_FILE"): + monkeypatch.delenv(env_key, raising=False) + config_data = { "current_context": "default", "clusters": [ @@ -133,6 +136,8 @@ def test_auth_logout_clears_selected_context_credentials( assert_exit_code(result, 0) assert "Logged out successfully" in result.output + assert "Context: foo" in result.output + assert "Config file:" in result.output with open(oauth_config_file) as f: data = yaml.safe_load(f) @@ -148,6 +153,43 @@ def test_auth_logout_clears_selected_context_credentials( assert "refresh_token" not in foo_user +def test_auth_logout_warns_when_runtime_token_override_remains( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("nemo_platform.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + monkeypatch.setenv( + "NEMO_WORKLOAD_TOKEN", + generate_unsigned_jwt( + principal_id="svc-nemo-ci", + email="svc-nemo-ci@example.com", + expires_in_seconds=900, + ), + ) + + result = runner.invoke(app, ["--context", "foo", "auth", "logout"]) + + assert_exit_code(result, 0) + assert "Logged out successfully" in result.output + assert "NEMO_WORKLOAD_TOKEN environment override is still active" in result.output + + +def test_auth_logout_fails_if_credentials_remain(oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from nemo_platform.config.config import Config + + def fake_write(*args, **kwargs): + return Config.load(config_path=oauth_config_file) + + monkeypatch.setattr("nemo_platform.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + monkeypatch.setattr("nemo_platform.config.config.Config.write", fake_write) + + result = runner.invoke(app, ["--context", "foo", "auth", "logout"]) + + assert_exit_code(result, 1) + assert "Logout did not clear credentials" in result.output + assert "context 'foo'" in result.output + assert oauth_config_file.name in result.output + + # --------------------------------------------------------------------------- # refresh # --------------------------------------------------------------------------- @@ -266,6 +308,33 @@ def test_auth_status_shows_warning_for_unsigned_token(oauth_config_file: Path, m assert "local/testing" in result.output +def test_runtime_token_source_label_handles_unreadable_token_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from nemo_platform.cli.commands.auth import _runtime_token_source_label + + token_file = tmp_path / "missing-workload-token.jwt" + monkeypatch.delenv("NMP_ACCESS_TOKEN", raising=False) + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN", raising=False) + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_file)) + + assert _runtime_token_source_label() == "NEMO_WORKLOAD_TOKEN_FILE environment override could not be read" + + +def test_auth_status_shows_config_file_credential_source( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("nemo_platform.cli.commands.auth.discover_nmp_config", _discover_auth_enabled) + + result = runner.invoke(app, ["--context", "foo", "auth", "status"]) + + assert_exit_code(result, 0) + assert "Config File" in result.output + assert oauth_config_file.name in result.output + assert "Credential Source" in result.output + assert "config file" in result.output + + # --------------------------------------------------------------------------- # login # --------------------------------------------------------------------------- @@ -351,6 +420,52 @@ def password_grant(**kwargs) -> SimpleNamespace: assert foo_user["refresh_token"] == "foo-refresh-token" +def test_auth_login_warns_when_env_access_token_will_override_saved_credentials( + oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def password_grant(**kwargs) -> SimpleNamespace: + return SimpleNamespace(token_for_nmp="foo-access-token", refresh_token="foo-refresh-token") + + monkeypatch.setenv( + "NMP_ACCESS_TOKEN", + generate_unsigned_jwt( + principal_id="svc-nemo-ci", + email="svc-nemo-ci@example.com", + expires_in_seconds=900, + ), + ) + monkeypatch.setattr("nemo_platform.cli.commands.auth.discover_nmp_config", _discover_oidc_config) + monkeypatch.setattr("nemo_platform.auth.device_flow.authenticate_with_password_grant", password_grant) + monkeypatch.setattr("nemo_platform.cli.commands.auth.decode_jwt_claims", _decode_jwt_noop) + + result = runner.invoke( + app, + [ + "auth", + "login", + "--context", + "foo", + "--base-url", + "https://foo-updated.example.com", + "--username", + "user", + "--password", + "secret", + ], + ) + + assert_exit_code(result, 0) + assert "NMP_ACCESS_TOKEN environment override is active" in result.output + assert "Unset the runtime token override" in result.output + + with open(oauth_config_file) as f: + data = yaml.safe_load(f) + + foo_user = next(user for user in data["users"] if user["name"] == "foo") + assert foo_user["token"] == "foo-access-token" + assert foo_user["refresh_token"] == "foo-refresh-token" + + def test_auth_login_with_base_url_creates_selected_context(oauth_config_file: Path, monkeypatch: pytest.MonkeyPatch): def password_grant(**kwargs) -> SimpleNamespace: return SimpleNamespace(token_for_nmp="dev-access-token", refresh_token="dev-refresh-token") diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py index 3a7ebb4b4f..dbe34f96c8 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/config/test_config.py @@ -337,6 +337,61 @@ def test_config_from_env_access_token_only(self, tmp_path: Path, monkeypatch: py assert config.user.refresh_token is None assert not hasattr(config.user, "token_endpoint") + def test_config_from_workload_token_env_only(self, monkeypatch: pytest.MonkeyPatch): + """NEMO_WORKLOAD_TOKEN should bootstrap OAuth auth without a config file.""" + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + + config = get_context() + + assert isinstance(config.user, OAuthUser) + assert config.user.token.get_secret_value() == "workload-token-123" + + def test_config_from_workload_token_file_env_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """NEMO_WORKLOAD_TOKEN_FILE should bootstrap OAuth auth without a config file.""" + token_path = tmp_path / "workload.token" + token_path.write_text("workload-token-from-file\n", encoding="utf-8") + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_path)) + + config = get_context() + + assert isinstance(config.user, OAuthUser) + assert config.user.token.get_secret_value() == "workload-token-from-file" + + def test_config_from_missing_workload_token_file_reports_configuration_error( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + """NEMO_WORKLOAD_TOKEN_FILE should fail clearly when the configured token file cannot be read.""" + token_path = tmp_path / "missing-workload.token" + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_path)) + + with pytest.raises(ValueError, match="NEMO_WORKLOAD_TOKEN_FILE"): + get_context() + + def test_nmp_access_token_precedes_workload_token_env(self, monkeypatch: pytest.MonkeyPatch): + """NMP_ACCESS_TOKEN remains the highest-precedence token env var.""" + monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") + monkeypatch.setenv("NMP_ACCESS_TOKEN", "preferred-token") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + + config = get_context() + + assert isinstance(config.user, OAuthUser) + assert config.user.token.get_secret_value() == "preferred-token" + + def test_runtime_access_token_source_label_uses_config_precedence( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + """Runtime token source labels should share Config's token env precedence.""" + missing_token_path = tmp_path / "missing-workload.token" + monkeypatch.setenv("NMP_ACCESS_TOKEN", "preferred-token") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(missing_token_path)) + + assert Config.runtime_access_token_source_label() == "NMP_ACCESS_TOKEN environment override" + def test_config_from_env_access_token_ignores_legacy_api_key_env(self, monkeypatch: pytest.MonkeyPatch): """NMP_ACCESS_TOKEN is used when legacy NMP_API_KEY is present without config.""" monkeypatch.setenv("NMP_BASE_URL", "https://api.example.com") diff --git a/services/core/auth/scripts/auth-tools.py b/services/core/auth/scripts/auth-tools.py index 7d829c1825..4de724a60d 100755 --- a/services/core/auth/scripts/auth-tools.py +++ b/services/core/auth/scripts/auth-tools.py @@ -1210,7 +1210,7 @@ def _generate_permissions_reference(auth_config: Dict) -> str: role_perms_map[role_name] = extract_role_permissions_recursive(roles_data, role_name) area_groups = _build_docs_area_groups(registry) - ordered_roles = ["Viewer", "Editor", "Admin"] + ordered_roles = ["Viewer", "Editor", "Admin", "JobRunner"] lines: List[str] = [] lines.append("---") @@ -1238,7 +1238,10 @@ def _generate_permissions_reference(auth_config: Dict) -> str: lines.append("") lines.append("") lines.append("") - lines.append("PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level.") + lines.append( + "PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level. " + "JobRunner is intended for workload identities, not interactive users." + ) lines.append("") lines.append("") @@ -1247,8 +1250,10 @@ def _generate_permissions_reference(auth_config: Dict) -> str: lines.append(f"## {display_name}") lines.append("") - lines.append("| Permission | Description | Viewer | Editor | Admin |") - lines.append("|------------|-------------|:------:|:------:|:-----:|") + header_cells = ["Permission", "Description", *ordered_roles] + alignment_cells = ["------------", "-------------", *[":------:" for _ in ordered_roles]] + lines.append(f"| {' | '.join(header_cells)} |") + lines.append(f"| {' | '.join(alignment_cells)} |") rows = _build_grouped_rows(perm_names, registry, role_perms_map, ordered_roles) lines.extend(rows) diff --git a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml index ec6a9f7dfd..ea8130d16c 100644 --- a/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml +++ b/services/core/auth/src/nmp/core/auth/assets/static-authz.yaml @@ -166,6 +166,9 @@ authz: description: "Delete jobs" list: description: "List jobs" + logs: + create: + description: "Upload job logs" read: description: "Read jobs" update: @@ -402,6 +405,10 @@ authz: - secrets.rotate - models.trust-remote-code.set - models.tool-call-plugin.set + JobRunner: + description: "Runtime permissions for job workload containers" + permissions: + - jobs.logs.create ServiceSystem: description: >- Internal platform service principal (principal IDs matching service:*). Used when no @@ -648,7 +655,7 @@ authz: /apis/files/v2/workspaces/{workspace}/filesets/{name}/otlp/v1/logs: post: permissions: - - filesets.create + - jobs.logs.create scopes: - files:write - platform:write diff --git a/services/core/auth/tests/integration/test_gateway_header_spoofing.py b/services/core/auth/tests/integration/test_gateway_header_spoofing.py new file mode 100644 index 0000000000..f77992b5ca --- /dev/null +++ b/services/core/auth/tests/integration/test_gateway_header_spoofing.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import yaml + +from tests.auth_idp.providers import load_provider_names_by_mode + +REQUIRED_STRIPPED_HEADERS = { + "x-nmp-principal-id", + "x-nmp-principal-email", + "x-nmp-principal-groups", + "x-nmp-principal-on-behalf-of", + "x-nmp-principal-on-behalf-of-email", + "x-nmp-principal-on-behalf-of-groups", +} + + +def _lua_removed_headers(http_filters: list[dict]) -> set[str]: + removed_headers = set() + for http_filter in http_filters: + if http_filter.get("name") != "envoy.filters.http.lua": + continue + inline_code = http_filter.get("typed_config", {}).get("inline_code", "").lower() + for header in REQUIRED_STRIPPED_HEADERS: + if f'headers:remove("{header}")' in inline_code or f"headers:remove('{header}')" in inline_code: + removed_headers.add(header) + return removed_headers + + +def test_compose_backed_idp_providers_are_discovered_from_manifests(): + providers = load_provider_names_by_mode("compose-ci") + assert providers + for provider in providers: + assert Path(f"contrib/auth/{provider}/gateway/envoy.yaml").exists() + + +def test_all_compose_backed_gateways_strip_trusted_identity_headers(): + for provider in load_provider_names_by_mode("compose-ci"): + config = yaml.safe_load(Path(f"contrib/auth/{provider}/gateway/envoy.yaml").read_text()) + typed_config = config["static_resources"]["listeners"][0]["filter_chains"][0]["filters"][0]["typed_config"] + virtual_host = typed_config["route_config"]["virtual_hosts"][0] + stripped_headers = set(virtual_host.get("request_headers_to_remove", [])) + stripped_headers.update(_lua_removed_headers(typed_config["http_filters"])) + assert REQUIRED_STRIPPED_HEADERS.issubset(stripped_headers) + + +def test_all_compose_backed_gateways_define_minimal_envoy_runtime(): + for provider in load_provider_names_by_mode("compose-ci"): + config = yaml.safe_load(Path(f"contrib/auth/{provider}/gateway/envoy.yaml").read_text()) + listener = config["static_resources"]["listeners"][0] + cluster = config["static_resources"]["clusters"][0] + assert listener["name"] + assert cluster["name"] == "nemo" diff --git a/services/core/auth/tests/test_embedded_pdp.py b/services/core/auth/tests/test_embedded_pdp.py index fc82470f82..b6c1d0cc7a 100644 --- a/services/core/auth/tests/test_embedded_pdp.py +++ b/services/core/auth/tests/test_embedded_pdp.py @@ -499,6 +499,44 @@ def test_viewer_role_has_permissions(self, static_authz_data): ) assert result["allowed"] is True + def test_job_runner_can_upload_otlp_logs_without_editor(self, static_authz_data): + static_authz_data["authz"]["principals"] = { + "viewer@test.com": {"workspaces": {"my-ws": ["Viewer"]}}, + "editor@test.com": {"workspaces": {"my-ws": ["Editor"]}}, + "job-runner@test.com": {"workspaces": {"my-ws": ["Viewer", "JobRunner"]}}, + } + set_policy_data(static_authz_data) + + path = "/apis/files/v2/workspaces/my-ws/filesets/job-fileset-test/otlp/v1/logs" + viewer_result = evaluate( + "allow", + { + "principal_id": "viewer@test.com", + "method": "POST", + "path": path, + }, + ) + editor_result = evaluate( + "allow", + { + "principal_id": "editor@test.com", + "method": "POST", + "path": path, + }, + ) + job_runner_result = evaluate( + "allow", + { + "principal_id": "job-runner@test.com", + "method": "POST", + "path": path, + }, + ) + + assert viewer_result["allowed"] is False + assert editor_result["allowed"] is False + assert job_runner_result["allowed"] is True + class TestIntakeAuthorization: """Verify active Intake endpoints are workspace-scoped in static authz data.""" diff --git a/services/core/auth/tests/test_machine_group_authz.py b/services/core/auth/tests/test_machine_group_authz.py new file mode 100644 index 0000000000..f7f4fa8fb8 --- /dev/null +++ b/services/core/auth/tests/test_machine_group_authz.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import uuid + +from fastapi.testclient import TestClient +from nemo_platform_ext.auth.helpers import generate_unsigned_jwt +from nmp.common.config import AuthConfig +from nmp.testing.client import create_test_client + +SERVICE_PRINCIPAL = "service:integration-test" +WORKSPACES_PATH = "/apis/entities/v2/workspaces" +IAM_ROLE_BINDINGS_PATH = "/apis/auth/v2/iam/role-bindings" + + +def _machine_headers(*, principal_id: str, email: str, groups: list[str]) -> dict[str, str]: + token = generate_unsigned_jwt(principal_id=principal_id, email=email, groups=groups) + return {"Authorization": f"Bearer {token}"} + + +def test_external_machine_identity_group_binding_grants_workspace_access(): + workspace_id = f"machine-ws-{uuid.uuid4().hex[:8]}" + group_name = f"machine-group-{uuid.uuid4().hex[:8]}" + machine_principal_id = f"machine-{uuid.uuid4().hex[:8]}" + assert not machine_principal_id.startswith("service:") + machine_headers = _machine_headers( + principal_id=machine_principal_id, + email=f"{machine_principal_id}@example.com", + groups=[group_name], + ) + service_headers = {"X-NMP-Principal-Id": SERVICE_PRINCIPAL} + + with create_test_client( + client_type=TestClient, + auth_enabled=True, + service_configs={ + AuthConfig: AuthConfig( + enabled=True, + allow_unsigned_jwt=True, + policy_decision_point_provider="embedded", + policy_decision_point_base_url="http://testserver", + propagation_poll_interval_seconds=0.05, + ) + }, + ) as client: + response = client.post( + WORKSPACES_PATH, + json={"name": workspace_id, "description": "Workspace for external machine identity auth"}, + headers=service_headers, + ) + assert response.status_code in (200, 201), f"Failed to create workspace: {response.text}" + + try: + denied = client.get( + f"{WORKSPACES_PATH}/{workspace_id}", + headers=_machine_headers( + principal_id=machine_principal_id, + email=f"{machine_principal_id}@example.com", + groups=[], + ), + ) + assert denied.status_code == 403, ( + f"Machine identity without the bound group should be denied. Got {denied.status_code}: {denied.text}" + ) + + response = client.post( + f"{IAM_ROLE_BINDINGS_PATH}?wait_role_propagation=true", + json={"principal": group_name, "role": "Viewer", "workspace": workspace_id}, + headers=service_headers, + ) + assert response.status_code in (200, 201), f"Failed to create role binding: {response.text}" + + allowed = client.get(f"{WORKSPACES_PATH}/{workspace_id}", headers=machine_headers) + assert allowed.status_code == 200, ( + f"Machine identity with the bound group should be allowed. Got {allowed.status_code}: {allowed.text}" + ) + + finally: + client.delete(f"{WORKSPACES_PATH}/{workspace_id}", headers=service_headers) diff --git a/services/core/jobs/jobs-launcher/cmd/run.go b/services/core/jobs/jobs-launcher/cmd/run.go index 48a1b470eb..b16a732d69 100644 --- a/services/core/jobs/jobs-launcher/cmd/run.go +++ b/services/core/jobs/jobs-launcher/cmd/run.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log/slog" + "net/url" "os" "os/exec" "os/signal" @@ -136,6 +137,7 @@ func runExecWithStdin(args []string) (int, error) { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + configureOTELHeadersFromWorkloadToken() otelShutdown, _, err := setupOTELSDK(ctx) if err != nil { return 1, err @@ -148,6 +150,29 @@ func runExecWithStdin(args []string) (int, error) { return runExec(args, os.Stdin) } +func configureOTELHeadersFromWorkloadToken() { + token := os.Getenv("NEMO_WORKLOAD_TOKEN") + if token == "" { + return + } + + const headersEnv = "OTEL_EXPORTER_OTLP_LOGS_HEADERS" + headers := os.Getenv(headersEnv) + for _, item := range strings.Split(headers, ",") { + key, _, _ := strings.Cut(strings.TrimSpace(item), "=") + if strings.EqualFold(key, "authorization") { + return + } + } + + authHeader := "Authorization=" + url.PathEscape("Bearer "+token) + if headers == "" { + os.Setenv(headersEnv, authHeader) + return + } + os.Setenv(headersEnv, headers+","+authHeader) +} + // runExec runs the specified command with arguments, injecting secrets as environment variables if specified func runExec(args []string, stdinReader io.Reader) (int, error) { // Command and arguments diff --git a/services/core/jobs/jobs-launcher/cmd/run_test.go b/services/core/jobs/jobs-launcher/cmd/run_test.go index f1e696a8aa..ffc9586259 100644 --- a/services/core/jobs/jobs-launcher/cmd/run_test.go +++ b/services/core/jobs/jobs-launcher/cmd/run_test.go @@ -306,6 +306,66 @@ func TestRunExecWithoutSecrets(t *testing.T) { } } +func TestConfigureOTELHeadersFromWorkloadToken(t *testing.T) { + testCases := []struct { + name string + token string + existingHeaders string + expectedHeaders string + }{ + { + name: "adds_authorization_header", + token: "token.with-symbols_123", + expectedHeaders: "Authorization=Bearer%20token.with-symbols_123", + }, + { + name: "preserves_existing_headers", + token: "abc.def", + existingHeaders: "X-NMP-Principal-Id=nemo-user", + expectedHeaders: "X-NMP-Principal-Id=nemo-user,Authorization=Bearer%20abc.def", + }, + { + name: "keeps_existing_authorization_header", + token: "abc.def", + existingHeaders: "authorization=Bearer+explicit", + expectedHeaders: "authorization=Bearer+explicit", + }, + { + name: "does_nothing_without_token", + existingHeaders: "X-Test=value", + expectedHeaders: "X-Test=value", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + origEnvVars := map[string]envVarState{ + "NEMO_WORKLOAD_TOKEN": getEnvState("NEMO_WORKLOAD_TOKEN"), + "OTEL_EXPORTER_OTLP_LOGS_HEADERS": getEnvState("OTEL_EXPORTER_OTLP_LOGS_HEADERS"), + } + defer restoreEnvVars(origEnvVars) + + if tc.token != "" { + os.Setenv("NEMO_WORKLOAD_TOKEN", tc.token) + } else { + os.Unsetenv("NEMO_WORKLOAD_TOKEN") + } + if tc.existingHeaders != "" { + os.Setenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", tc.existingHeaders) + } else { + os.Unsetenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS") + } + + configureOTELHeadersFromWorkloadToken() + + got := os.Getenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS") + if got != tc.expectedHeaders { + t.Errorf("Expected OTEL headers %q, got %q", tc.expectedHeaders, got) + } + }) + } +} + func TestParseSecretReferences(t *testing.T) { testCases := []struct { name string diff --git a/services/hello-world/src/nmp/hello_world/tasks/workload_workspace_get/__main__.py b/services/hello-world/src/nmp/hello_world/tasks/workload_workspace_get/__main__.py new file mode 100644 index 0000000000..0fa1c0a863 --- /dev/null +++ b/services/hello-world/src/nmp/hello_world/tasks/workload_workspace_get/__main__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Entry point for running the workload workspace read task as a module.""" + +from nmp.hello_world.tasks.workload_workspace_get.run import run + +if __name__ == "__main__": + raise SystemExit(run()) diff --git a/services/hello-world/src/nmp/hello_world/tasks/workload_workspace_get/run.py b/services/hello-world/src/nmp/hello_world/tasks/workload_workspace_get/run.py new file mode 100644 index 0000000000..49977c403d --- /dev/null +++ b/services/hello-world/src/nmp/hello_world/tasks/workload_workspace_get/run.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task that exercises workload-auth by reading a workspace through the public SDK.""" + +import os + +from nemo_platform import NeMoPlatform +from nmp.common.jobs.config import get_task_config +from pydantic import BaseModel + +_WORKLOAD_TOKEN_ENV_VARS = ("NEMO_WORKLOAD_TOKEN", "NEMO_WORKLOAD_TOKEN_FILE") + + +class WorkloadWorkspaceGetConfig(BaseModel): + """Configuration for the workload workspace read task.""" + + workspace: str + + +def _load_workload_token() -> str: + if token := os.environ.get("NEMO_WORKLOAD_TOKEN"): + return token + if token_file := os.environ.get("NEMO_WORKLOAD_TOKEN_FILE"): + with open(token_file, encoding="utf-8") as token_handle: + token = token_handle.read().strip() + if token: + return token + token_vars = " or ".join(_WORKLOAD_TOKEN_ENV_VARS) + raise RuntimeError(f"workload token not configured; set {token_vars}") + + +def run(*, sdk: NeMoPlatform | None = None) -> int: + """Read the configured workspace using the public bearer-token SDK path.""" + try: + config = get_task_config(WorkloadWorkspaceGetConfig) + if sdk is None: + token = _load_workload_token() + sdk = NeMoPlatform(default_headers={"Authorization": f"Bearer {token}"}) + workspace = sdk.workspaces.retrieve(config.workspace) + print(f"Successfully retrieved workspace: {workspace.name}") + return 0 + except Exception as exc: + print(f"Workload workspace retrieval failed: {exc}") + return 1 diff --git a/services/hello-world/tests/integration/tasks/test_workload_workspace_get_task.py b/services/hello-world/tests/integration/tasks/test_workload_workspace_get_task.py new file mode 100644 index 0000000000..dc195dc6aa --- /dev/null +++ b/services/hello-world/tests/integration/tasks/test_workload_workspace_get_task.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +from nmp.common.jobs.constants import TASK_CONFIG_ENVVAR +from nmp.hello_world.tasks.workload_workspace_get.run import run as task_run + + +class _StubWorkspaces: + def __init__(self) -> None: + self.requested: list[str] = [] + + def retrieve(self, workspace: str) -> SimpleNamespace: + self.requested.append(workspace) + return SimpleNamespace(name=workspace) + + +class _StubSDK: + def __init__(self) -> None: + self.workspaces = _StubWorkspaces() + + +def test_workload_workspace_get_reads_workspace_via_public_sdk(monkeypatch): + sdk = _StubSDK() + sdk_kwargs = {} + + def create_sdk(**kwargs): + sdk_kwargs.update(kwargs) + return sdk + + monkeypatch.setenv(TASK_CONFIG_ENVVAR, '{"workspace":"workload-read-target"}') + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN", "workload-token-123") + monkeypatch.setattr("nmp.hello_world.tasks.workload_workspace_get.run.NeMoPlatform", create_sdk) + + exit_code = task_run() + + assert exit_code == 0 + assert sdk.workspaces.requested == ["workload-read-target"] + assert sdk_kwargs == {"default_headers": {"Authorization": "Bearer workload-token-123"}} + + +def test_workload_workspace_get_requires_workload_token_env(monkeypatch): + monkeypatch.setenv(TASK_CONFIG_ENVVAR, '{"workspace":"workload-read-target"}') + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN", raising=False) + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN_FILE", raising=False) + + exit_code = task_run() + + assert exit_code == 1 + + +def test_workload_workspace_get_uses_injected_sdk_without_workload_token(monkeypatch): + sdk = _StubSDK() + monkeypatch.setenv(TASK_CONFIG_ENVVAR, '{"workspace":"workload-read-target"}') + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN", raising=False) + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN_FILE", raising=False) + + exit_code = task_run(sdk=sdk) + + assert exit_code == 0 + assert sdk.workspaces.requested == ["workload-read-target"] + + +def test_workload_workspace_get_accepts_workload_token_file_env(monkeypatch, tmp_path): + sdk = _StubSDK() + sdk_kwargs = {} + + def create_sdk(**kwargs): + sdk_kwargs.update(kwargs) + return sdk + + token_path = tmp_path / "workload.token" + token_path.write_text("workload-token-from-file\n", encoding="utf-8") + monkeypatch.setenv(TASK_CONFIG_ENVVAR, '{"workspace":"workload-read-target"}') + monkeypatch.delenv("NEMO_WORKLOAD_TOKEN", raising=False) + monkeypatch.setenv("NEMO_WORKLOAD_TOKEN_FILE", str(token_path)) + monkeypatch.setattr("nmp.hello_world.tasks.workload_workspace_get.run.NeMoPlatform", create_sdk) + + exit_code = task_run() + + assert exit_code == 0 + assert sdk.workspaces.requested == ["workload-read-target"] + assert sdk_kwargs == {"default_headers": {"Authorization": "Bearer workload-token-from-file"}} diff --git a/tests/auth_idp/authentik_live.py b/tests/auth_idp/authentik_live.py new file mode 100644 index 0000000000..77ffa470dc --- /dev/null +++ b/tests/auth_idp/authentik_live.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +AUTHENTIK_COMPOSE_PROJECT_NAME = "authentik-e2e" +AUTHENTIK_WORKLOAD_NETWORK_NAME = f"{AUTHENTIK_COMPOSE_PROJECT_NAME}_workload" +AUTHENTIK_NEMO_DIRECT_PORT = "38081" + +AUTHENTIK_DOCKER_E2E_CONFIG = pytest.mark.e2e_config( + "contrib/auth/authentik/config/platform-compose-authentik.yaml", + { + "auth": { + "oidc": { + "additional_issuers": [ + "http://authentik-server:9000/application/o/nemo/", + "http://127.0.0.1:38080/application/o/nemo-cli/", + "http://127.0.0.1:38080/application/o/nemo/", + ], + "token_endpoint": "http://127.0.0.1:38080/application/o/token/", + "device_authorization_endpoint": "http://127.0.0.1:38080/application/o/device/", + } + }, + }, + harness={ + "backend": "docker_compose", + "compose_file": "contrib/auth/authentik/docker-compose.yml", + "compose_project_name": AUTHENTIK_COMPOSE_PROJECT_NAME, + "service_url": "http://127.0.0.1:38080", + "auth_ready_url": f"http://127.0.0.1:{AUTHENTIK_NEMO_DIRECT_PORT}", + "wait_url": "http://127.0.0.1:38080/application/o/nemo/.well-known/openid-configuration", + "env": { + "AUTHENTIK_GATEWAY_PORT": "38080", + "NEMO_DIRECT_PORT": AUTHENTIK_NEMO_DIRECT_PORT, + "AUTHENTIK_WORKLOAD_NETWORK_NAME": AUTHENTIK_WORKLOAD_NETWORK_NAME, + }, + }, +) + +AUTHENTIK_DOCKER_PYTESTMARK = [ + pytest.mark.auth_idp, + AUTHENTIK_DOCKER_E2E_CONFIG, + pytest.mark.xdist_group("idp-live"), +] diff --git a/tests/auth_idp/conftest.py b/tests/auth_idp/conftest.py new file mode 100644 index 0000000000..401992a93e --- /dev/null +++ b/tests/auth_idp/conftest.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import time +import uuid +from collections.abc import Iterator +from dataclasses import replace + +import httpx +import pytest +from nemo_platform import NeMoPlatform + +from tests.auth_idp.providers import ProviderConfig +from tests.auth_idp.runtime import get_authentik_docker_test_runtime + +pytest_plugins = ("e2e.conftest",) + + +def _token_request_body(grant: dict[str, str]) -> dict[str, str]: + grant_type = grant["grant_type"] + body = { + "grant_type": grant_type, + "client_id": grant["client_id"], + } + if "client_secret" in grant: + body["client_secret"] = grant["client_secret"] + if grant_type == "password": + body["username"] = grant["username"] + body["password"] = grant["password"] + if "scope" in grant: + body["scope"] = grant["scope"] + return body + if grant_type == "client_credentials": + if "scope" in grant: + body["scope"] = grant["scope"] + return body + raise ValueError(f"unsupported grant_type for auth_idp token exchange: {grant_type}") + + +def _exchange_token_with_retries(token_endpoint: str, grant: dict[str, str], timeout: float = 60.0) -> str: + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + response = httpx.post( + token_endpoint, + data=_token_request_body(grant), + timeout=30.0, + ) + if response.status_code >= 500: + last_error = httpx.HTTPStatusError( + f"token endpoint not ready: {response.status_code}", + request=response.request, + response=response, + ) + time.sleep(2) + continue + response.raise_for_status() + return response.json()["access_token"] + except httpx.RequestError as exc: + last_error = exc + time.sleep(2) + if last_error is not None: + raise last_error + raise TimeoutError(f"token endpoint did not become ready: {token_endpoint}") + + +@pytest.fixture(scope="session") +def idp_e2e_enabled(pytestconfig: pytest.Config) -> bool: + return bool(pytestconfig.getoption("--run-e2e")) + + +@pytest.fixture(scope="session") +def require_idp_e2e(idp_e2e_enabled: bool) -> Iterator[None]: + if not idp_e2e_enabled: + pytest.skip("set --run-e2e to execute provider stack validation") + yield + + +@pytest.fixture(scope="module") +def authentik_provider(authentik_docker_runtime: ProviderConfig) -> ProviderConfig: + provider = authentik_docker_runtime + assert provider.token_endpoint is not None + assert provider.machine_grant is not None + return provider + + +@pytest.fixture(scope="session") +def authentik_docker_runtime() -> ProviderConfig: + provider = get_authentik_docker_test_runtime() + assert provider.token_endpoint is not None + assert provider.machine_grant is not None + return provider + + +@pytest.fixture(scope="module") +def authentik_stack( + require_idp_e2e: None, + authentik_provider: ProviderConfig, + _services: str, +) -> ProviderConfig: + return replace( + authentik_provider, + gateway_base_url=_services, + discovery_url=f"{_services}/application/o/nemo/.well-known/openid-configuration", + token_endpoint=f"{_services}/application/o/token/", + ) + + +@pytest.fixture(scope="module") +def machine_token(authentik_stack: ProviderConfig) -> str: + grant = authentik_stack.machine_grant + assert grant is not None + assert authentik_stack.token_endpoint is not None + return _exchange_token_with_retries(authentik_stack.token_endpoint, grant) + + +@pytest.fixture(scope="module") +def human_token(authentik_stack: ProviderConfig) -> str: + grant = authentik_stack.human_grant + assert grant is not None + assert authentik_stack.token_endpoint is not None + return _exchange_token_with_retries(authentik_stack.token_endpoint, grant) + + +@pytest.fixture(scope="module") +def authentik_human_sdk(authentik_stack: ProviderConfig, human_token: str) -> NeMoPlatform: + return NeMoPlatform( + base_url=authentik_stack.gateway_base_url, + default_headers={"Authorization": f"Bearer {human_token}"}, + max_retries=0, + ) + + +@pytest.fixture(scope="module") +def machine_sdk(authentik_stack: ProviderConfig, machine_token: str) -> NeMoPlatform: + return NeMoPlatform( + base_url=authentik_stack.gateway_base_url, + default_headers={"Authorization": f"Bearer {machine_token}"}, + max_retries=0, + ) + + +@pytest.fixture +def authentik_workspace(authentik_human_sdk: NeMoPlatform) -> Iterator[str]: + workspace_name = f"authentik-ws-{uuid.uuid4().hex[:8]}" + authentik_human_sdk.workspaces.create( + name=workspace_name, + description="Workspace for Authentik live auth tests", + wait_role_propagation=True, + ) + try: + yield workspace_name + finally: + authentik_human_sdk.workspaces.delete(workspace_name) diff --git a/tests/auth_idp/providers.py b/tests/auth_idp/providers.py new file mode 100644 index 0000000000..9b810015e6 --- /dev/null +++ b/tests/auth_idp/providers.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass +from pathlib import Path + +import yaml + + +@dataclass(frozen=True) +class ProviderConfig: + name: str + mode: str + compose_file: Path | None + gateway_base_url: str + issuer_url: str + discovery_url: str + nemo_config: Path + workload_principal_id: str + workload_expected_groups: list[str] + workload_audience: str + workload_principal_claim: str + workload_groups_claim: str + workload_groups_format: str + workload_token_env_vars: list[str] + workload_forwarded_headers: dict[str, str] + token_endpoint: str | None + human_grant: dict[str, str] | None + machine_grant: dict[str, str] | None + healthchecks: list[dict[str, str]] + startup_timeouts: dict[str, int] + compose_project_name: str | None = None + + +def load_provider_config(manifest_path: Path) -> ProviderConfig: + data = yaml.safe_load(manifest_path.read_text()) + return ProviderConfig( + name=data["provider"], + mode=data["mode"], + compose_file=(None if not data.get("compose_file") else manifest_path.parent / data["compose_file"]), + gateway_base_url=data["gateway_base_url"], + issuer_url=data["issuer_url"], + discovery_url=data["discovery_url"], + nemo_config=manifest_path.parent / data["nemo_config"], + workload_principal_id=data["workload_identity"]["principal_id"], + workload_expected_groups=list(data["workload_identity"]["expected_groups"]), + workload_audience=data["workload_contract"]["audience"], + workload_principal_claim=data["workload_contract"]["principal_claim"], + workload_groups_claim=data["workload_contract"]["groups_claim"], + workload_groups_format=data["workload_contract"]["groups_format"], + workload_token_env_vars=list(data["workload_contract"]["token_env_vars"]), + workload_forwarded_headers=dict(data["workload_contract"]["forwarded_headers"]), + token_endpoint=data.get("token_acquisition", {}).get("token_endpoint"), + human_grant=data.get("token_acquisition", {}).get("human_grant"), + machine_grant=data.get("token_acquisition", {}).get("machine_grant"), + healthchecks=list(data.get("healthchecks", [])), + startup_timeouts=dict(data.get("startup_timeouts", {})), + ) + + +def load_provider_configs() -> list[ProviderConfig]: + configs: list[ProviderConfig] = [] + for manifest_path in sorted(Path("contrib/auth").glob("*/manifest.yaml")): + configs.append(load_provider_config(manifest_path)) + return configs + + +def load_provider_configs_by_mode(mode: str) -> list[ProviderConfig]: + return [provider for provider in load_provider_configs() if provider.mode == mode] + + +def load_provider_names() -> list[str]: + return [provider.name for provider in load_provider_configs()] + + +def load_provider_names_by_mode(mode: str) -> list[str]: + return [provider.name for provider in load_provider_configs_by_mode(mode)] diff --git a/tests/auth_idp/runtime.py b/tests/auth_idp/runtime.py new file mode 100644 index 0000000000..7d0385d7d6 --- /dev/null +++ b/tests/auth_idp/runtime.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import uuid +from dataclasses import replace +from functools import lru_cache +from pathlib import Path + +from tests.auth_idp.providers import ProviderConfig, load_provider_config + + +@lru_cache(maxsize=1) +def get_authentik_docker_test_runtime() -> ProviderConfig: + provider = load_provider_config(Path("contrib/auth/authentik/manifest.yaml")) + return replace( + provider, + nemo_config=Path("contrib/auth/authentik/config/platform-compose-authentik.yaml"), + compose_project_name=f"authentik-e2e-{uuid.uuid4().hex[:8]}", + ) diff --git a/tests/auth_idp/test_authentik_cli_login.py b/tests/auth_idp/test_authentik_cli_login.py new file mode 100644 index 0000000000..cb1e6552b3 --- /dev/null +++ b/tests/auth_idp/test_authentik_cli_login.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import json +import uuid + +import httpx +from nemo_platform_ext.auth.helpers import discover_nmp_config + +from tests.auth_idp.authentik_live import AUTHENTIK_DOCKER_PYTESTMARK + +pytestmark = AUTHENTIK_DOCKER_PYTESTMARK + + +def _jwt_claims(token: str) -> dict[str, object]: + parts = token.split(".") + if len(parts) != 3: + return {} + payload = parts[1] + "=" * (-len(parts[1]) % 4) + return json.loads(base64.urlsafe_b64decode(payload)) + + +def _delete_workspace_for_cleanup(base_url: str, workspace_name: str, headers: dict[str, str]) -> None: + response = httpx.delete( + f"{base_url}/apis/entities/v2/workspaces/{workspace_name}", + headers=headers, + timeout=10.0, + ) + response.raise_for_status() + + +def test_authentik_discovery_exposes_gateway_reachable_device_flow(authentik_stack): + oidc = discover_nmp_config(authentik_stack.gateway_base_url) + + assert oidc.auth_enabled is True + assert oidc.client_id == "nemo-platform-cli" + assert oidc.token_endpoint == "http://127.0.0.1:38080/application/o/token/" + assert oidc.device_authorization_endpoint == "http://127.0.0.1:38080/application/o/device/" + assert oidc.default_scopes == "openid email offline_access groups" + + response = httpx.post( + oidc.device_authorization_endpoint, + data={ + "client_id": oidc.client_id, + "scope": oidc.default_scopes, + }, + timeout=30.0, + ) + response.raise_for_status() + body = response.json() + + assert body["verification_uri"] == "http://127.0.0.1:38080/device" + assert body["verification_uri_complete"].startswith("http://127.0.0.1:38080/device?code=") + assert body["device_code"] + assert body["user_code"] + + +def test_authentik_cli_provider_token_is_accepted_by_gateway(authentik_stack): + token_response = httpx.post( + authentik_stack.token_endpoint, + data={ + "grant_type": "password", + "client_id": "nemo-platform-cli", + "username": "nemo-user", + "password": "nemo-user-token-secret-dev", + "scope": "openid email offline_access groups", + }, + timeout=30.0, + ) + token_response.raise_for_status() + access_token = token_response.json()["access_token"] + claims = _jwt_claims(access_token) + workspace_name = f"cli-audience-check-{uuid.uuid4().hex[:8]}" + headers = {"Authorization": f"Bearer {access_token}"} + + assert claims["aud"] == "nemo-platform-cli" + + try: + create_response = httpx.post( + f"{authentik_stack.gateway_base_url}/apis/entities/v2/workspaces", + json={"name": workspace_name, "description": "CLI audience check"}, + headers=headers, + timeout=10.0, + ) + create_response.raise_for_status() + assert create_response.json()["created_by"] == "nemo-user" + finally: + _delete_workspace_for_cleanup(authentik_stack.gateway_base_url, workspace_name, headers) diff --git a/tests/auth_idp/test_authentik_gateway_live.py b/tests/auth_idp/test_authentik_gateway_live.py new file mode 100644 index 0000000000..7961c3f6ac --- /dev/null +++ b/tests/auth_idp/test_authentik_gateway_live.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import json +import uuid + +import httpx +from nmp.testing import grant_workspace_role + +from tests.auth_idp.authentik_live import AUTHENTIK_DOCKER_PYTESTMARK + +pytestmark = AUTHENTIK_DOCKER_PYTESTMARK + + +def _jwt_claims(token: str) -> dict[str, object]: + parts = token.split(".") + if len(parts) != 3: + return {} + payload = parts[1] + "=" * (-len(parts[1]) % 4) + return json.loads(base64.urlsafe_b64decode(payload)) + + +def test_authentik_gateway_rejects_unauthenticated_requests(authentik_stack): + response = httpx.get(f"{authentik_stack.gateway_base_url}/apis/entities/v2/workspaces", timeout=10.0) + assert response.status_code in {401, 403} + + +def test_authentik_gateway_rejects_spoofed_principal_headers(authentik_stack, machine_token: str): + workspace_name = f"spoof-check-{uuid.uuid4().hex[:8]}" + claims = _jwt_claims(machine_token) + authenticated_principal_id = str(claims["sub"]) + expected_binding_principal = authenticated_principal_id + headers = { + "Authorization": f"Bearer {machine_token}", + "X-NMP-Principal-Id": "service:bootstrap", + "X-NMP-Principal-Email": "attacker@example.com", + } + + try: + create_response = httpx.post( + f"{authentik_stack.gateway_base_url}/apis/entities/v2/workspaces", + json={"name": workspace_name, "description": "Spoofed header check"}, + headers=headers, + timeout=10.0, + ) + create_response.raise_for_status() + assert create_response.json()["created_by"] == authenticated_principal_id + + members_response = httpx.get( + f"{authentik_stack.gateway_base_url}/apis/entities/v2/workspaces/{workspace_name}/members", + headers=headers, + timeout=10.0, + ) + members_response.raise_for_status() + admin_member = next(member for member in members_response.json()["data"] if "Admin" in member["roles"]) + + assert admin_member["granted_by"] == authenticated_principal_id + assert admin_member["principal"] == expected_binding_principal + assert admin_member["principal"] not in {"service:bootstrap", "attacker@example.com"} + finally: + httpx.delete( + f"{authentik_stack.gateway_base_url}/apis/entities/v2/workspaces/{workspace_name}", + headers=headers, + timeout=10.0, + ) + + +def test_authentik_gateway_forwards_workload_groups( + authentik_stack, + authentik_human_sdk, + authentik_workspace, + authentik_provider, + machine_token: str, +): + claims = _jwt_claims(machine_token) + claim_groups = claims.get("groups") + assert isinstance(claim_groups, str) + token_groups = {group.strip() for group in claim_groups.split(",") if group.strip()} + bound_group = authentik_provider.workload_expected_groups[0] + assert bound_group in token_groups + + grant_workspace_role(authentik_human_sdk, workspace=authentik_workspace, principal=bound_group, roles=["Viewer"]) + + headers = {"Authorization": f"Bearer {machine_token}"} + + response = httpx.get( + f"{authentik_stack.gateway_base_url}/apis/entities/v2/workspaces/{authentik_workspace}", + headers=headers, + timeout=10.0, + ) + + assert response.status_code == 200 + assert response.json()["name"] == authentik_workspace diff --git a/tests/auth_idp/test_authentik_real_oidc.py b/tests/auth_idp/test_authentik_real_oidc.py new file mode 100644 index 0000000000..6ce4c5c2a2 --- /dev/null +++ b/tests/auth_idp/test_authentik_real_oidc.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + +import pytest +from nemo_platform import APIStatusError +from nmp.testing import grant_workspace_role +from nmp.testing.e2e import wait_for_job_logs, wait_for_platform_job + +from tests.auth_idp.authentik_live import AUTHENTIK_DOCKER_PYTESTMARK + +pytestmark = AUTHENTIK_DOCKER_PYTESTMARK + + +def _nmp_api_image() -> str: + registry = os.environ.get("IMAGE_REGISTRY", "my-registry") + tag = os.environ.get("BAKE_TAG", "local") + return f"{registry}/nmp-api:{tag}" + + +def test_authentik_workload_token_is_real(machine_token: str, authentik_provider): + assert machine_token + assert authentik_provider.token_endpoint + + +def test_authentik_workload_identity_is_denied_before_binding(machine_sdk, authentik_workspace): + with pytest.raises(APIStatusError) as exc_info: + machine_sdk.workspaces.retrieve(authentik_workspace) + assert exc_info.value.status_code == 403 + + +def test_authentik_workload_identity_is_allowed_after_binding( + authentik_human_sdk, + machine_sdk, + authentik_workspace, + authentik_provider, +): + for group in authentik_provider.workload_expected_groups: + grant_workspace_role(authentik_human_sdk, workspace=authentik_workspace, principal=group, roles=["Viewer"]) + + retrieved = machine_sdk.workspaces.retrieve(authentik_workspace) + assert retrieved.name == authentik_workspace + + +def test_authentik_workload_identity_returns_to_denied_after_revoke( + authentik_human_sdk, + machine_sdk, + authentik_workspace, + authentik_provider, +): + for group in authentik_provider.workload_expected_groups: + grant_workspace_role( + authentik_human_sdk, + workspace=authentik_workspace, + principal=group, + roles=["Viewer"], + ) + authentik_human_sdk.workspaces.members.delete( + group, + workspace=authentik_workspace, + wait_role_propagation=True, + ) + + with pytest.raises(APIStatusError) as exc_info: + machine_sdk.workspaces.retrieve(authentik_workspace) + assert exc_info.value.status_code == 403 + + +def test_authentik_workload_job_runs_via_docker_profile( + authentik_human_sdk, + authentik_workspace, + authentik_provider, + machine_token: str, +): + for group in authentik_provider.workload_expected_groups: + grant_workspace_role( + authentik_human_sdk, + workspace=authentik_workspace, + principal=group, + roles=["Viewer", "JobRunner"], + ) + + job = authentik_human_sdk.jobs.create( + workspace=authentik_workspace, + source="authentik-live-workload-job", + spec={"test": "workload-job"}, + platform_spec={ + "steps": [ + { + "name": "workload-workspace-get", + "executor": { + "provider": "cpu", + "profile": "workload", + "container": { + "image": _nmp_api_image(), + "entrypoint": ["sh", "-c"], + "command": ["nemo-platform run task --task nmp.hello_world.tasks.workload_workspace_get"], + }, + }, + "environment": [ + { + "name": "NEMO_WORKLOAD_TOKEN", + "value": machine_token, + } + ], + "config": { + "workspace": authentik_workspace, + }, + } + ] + }, + ) + + completed_job = wait_for_platform_job(authentik_human_sdk, job.name, authentik_workspace, timeout=240) + assert completed_job.status == "completed" + + step_logs = wait_for_job_logs(authentik_human_sdk, job.name, authentik_workspace, min_log_count=1, timeout=240) + assert any("Successfully retrieved workspace" in log.message for log in step_logs.data) diff --git a/tests/auth_idp/test_authentik_startup_smoke.py b/tests/auth_idp/test_authentik_startup_smoke.py new file mode 100644 index 0000000000..fd2aae0fc1 --- /dev/null +++ b/tests/auth_idp/test_authentik_startup_smoke.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import httpx + +from tests.auth_idp.authentik_live import AUTHENTIK_DOCKER_PYTESTMARK + +pytestmark = AUTHENTIK_DOCKER_PYTESTMARK + + +def test_authentik_discovery_is_reachable(authentik_stack): + response = httpx.get(authentik_stack.discovery_url, timeout=10.0) + assert response.status_code == 200 + assert response.json()["issuer"].endswith("/application/o/nemo/") diff --git a/tests/auth_idp/test_docs_links.py b/tests/auth_idp/test_docs_links.py new file mode 100644 index 0000000000..e9804a9f73 --- /dev/null +++ b/tests/auth_idp/test_docs_links.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.auth_idp] + + +def test_auth_docs_link_to_contrib_references(): + content = Path("docs/auth/authentication/idp-integration.mdx").read_text() + assert "contrib/auth/authentik" in content + assert "NEMO_WORKLOAD_TOKEN" in content diff --git a/tests/auth_idp/test_fixture_helpers.py b/tests/auth_idp/test_fixture_helpers.py new file mode 100644 index 0000000000..5bf76ed834 --- /dev/null +++ b/tests/auth_idp/test_fixture_helpers.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for auth-idp pytest fixtures and fixture-only helper functions.""" + +from pathlib import Path + +import pytest + +from tests.auth_idp import conftest +from tests.auth_idp.conftest import _token_request_body +from tests.auth_idp.providers import ProviderConfig +from tests.auth_idp.xdist import append_xdist_group_suffix + +pytestmark = [pytest.mark.auth_idp] + + +def test_authentik_stack_fixture_uses_pooled_gateway_metadata(): + provider = ProviderConfig( + name="authentik", + mode="compose-ci", + compose_file=Path("docker-compose.yml"), + gateway_base_url="http://127.0.0.1:18080", + issuer_url="http://authentik-server:9000/application/o/nemo/", + discovery_url="http://127.0.0.1:18080/application/o/nemo/.well-known/openid-configuration", + token_endpoint="http://127.0.0.1:18080/application/o/token/", + nemo_config=Path("config/platform-compose-authentik.yaml"), + workload_principal_id="svc-nemo", + workload_expected_groups=["nemo-editors"], + workload_audience="nemo-platform", + workload_principal_claim="sub", + workload_groups_claim="groups", + workload_groups_format="comma_string", + workload_token_env_vars=["NEMO_WORKLOAD_TOKEN", "NEMO_WORKLOAD_TOKEN_FILE"], + workload_forwarded_headers={ + "principal_id": "X-NMP-Principal-Id", + "principal_groups": "X-NMP-Principal-Groups", + }, + human_grant={"grant_type": "password"}, + machine_grant={"grant_type": "password", "username": "svc-nemo", "password": "svc-nemo-token-secret-dev"}, + healthchecks=[], + startup_timeouts={}, + ) + fixture_fn = conftest.authentik_stack.__wrapped__ + stack = fixture_fn(None, provider, "http://127.0.0.1:28080") + + assert stack.gateway_base_url == "http://127.0.0.1:28080" + assert stack.discovery_url == "http://127.0.0.1:28080/application/o/nemo/.well-known/openid-configuration" + assert stack.token_endpoint == "http://127.0.0.1:28080/application/o/token/" + assert stack.nemo_config == provider.nemo_config + + +def test_token_request_body_for_password_grant_includes_username_and_password(): + assert _token_request_body( + { + "grant_type": "password", + "client_id": "nemo-platform", + "client_secret": "secret", + "username": "akadmin", + "password": "akadmin-dev", + "scope": "openid profile email groups", + } + ) == { + "grant_type": "password", + "client_id": "nemo-platform", + "client_secret": "secret", + "username": "akadmin", + "password": "akadmin-dev", + "scope": "openid profile email groups", + } + + +def test_token_request_body_for_workload_password_grant_includes_username_and_password(): + assert _token_request_body( + { + "grant_type": "password", + "client_id": "nemo-platform", + "client_secret": "secret", + "username": "svc-nemo", + "password": "svc-nemo-token-secret-dev", + "scope": "openid email groups", + } + ) == { + "grant_type": "password", + "client_id": "nemo-platform", + "client_secret": "secret", + "username": "svc-nemo", + "password": "svc-nemo-token-secret-dev", + "scope": "openid email groups", + } + + +def test_append_xdist_group_suffix_only_appends_once_and_sorts_groups(): + nodeid = "tests/auth_idp/test_authentik_real_oidc.py::test_authentik_machine_token_is_real" + assert append_xdist_group_suffix(nodeid, {"idp-live"}) == f"{nodeid}@idp-live" + assert append_xdist_group_suffix(nodeid, {"b", "a"}) == f"{nodeid}@a_b" + assert append_xdist_group_suffix(f"{nodeid}@idp-live", {"idp-live"}) == f"{nodeid}@idp-live" diff --git a/tests/auth_idp/test_provider_layout.py b/tests/auth_idp/test_provider_layout.py new file mode 100644 index 0000000000..573998d81d --- /dev/null +++ b/tests/auth_idp/test_provider_layout.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +import yaml + +from tests.auth_idp.providers import load_provider_names_by_mode + +pytestmark = [pytest.mark.auth_idp] + + +def test_compose_backed_providers_ship_required_assets(): + for provider in load_provider_names_by_mode("compose-ci"): + root = Path(f"contrib/auth/{provider}") + assert (root / "docker-compose.yml").exists() + assert (root / "gateway").exists() + assert (root / "README.md").exists() + assert (root / "manifest.yaml").exists() + + +def test_reference_only_providers_do_not_require_compose(): + for provider in load_provider_names_by_mode("reference-only"): + root = Path(f"contrib/auth/{provider}") + assert (root / "README.md").exists() + assert not (root / "docker-compose.yml").exists() + + +def test_authentik_compose_disables_model_provider_seed_without_ngc_key(): + compose = yaml.safe_load(Path("contrib/auth/authentik/docker-compose.yml").read_text()) + nemo_service = compose["services"]["nemo"] + nemo_env = nemo_service["environment"] + + assert nemo_env["NMP_SEED_ON_STARTUP"] == "true" + assert nemo_env["NMP_PLATFORM_SEED_MODEL_PROVIDER_ENABLED"] == "false" + assert "${NEMO_DIRECT_PORT:-18081}:8080" in nemo_service["ports"] diff --git a/tests/auth_idp/test_provider_manifest.py b/tests/auth_idp/test_provider_manifest.py new file mode 100644 index 0000000000..f3fc1d50b2 --- /dev/null +++ b/tests/auth_idp/test_provider_manifest.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +import yaml +from jsonschema.validators import validator_for + +from tests.auth_idp.providers import load_provider_configs + +pytestmark = [pytest.mark.auth_idp] + + +def _load_provider_manifest_schema() -> dict: + return yaml.safe_load(Path("contrib/auth/manifest.schema.yaml").read_text()) + + +def test_all_provider_manifests_share_the_same_contract(): + schema = _load_provider_manifest_schema() + validator = validator_for(schema)(schema) + for provider in load_provider_configs(): + manifest = yaml.safe_load(Path(f"contrib/auth/{provider.name}/manifest.yaml").read_text()) + validator.validate(manifest) + assert manifest["provider"] == provider.name + + +def test_authentik_manifest_declares_real_token_acquisition_contract(): + manifest = yaml.safe_load(Path("contrib/auth/authentik/manifest.yaml").read_text()) + token_acquisition = manifest["token_acquisition"] + principal_contract = manifest["principal_contract"] + workload_identity = manifest["workload_identity"] + workload_contract = manifest["workload_contract"] + + assert token_acquisition["token_endpoint"] + assert token_acquisition["human_grant"]["grant_type"] == "password" + assert token_acquisition["machine_grant"]["grant_type"] == "password" + assert token_acquisition["human_grant"]["client_id"] + assert token_acquisition["machine_grant"]["client_id"] + assert token_acquisition["human_grant"]["password"] == "nemo-user-token-secret-dev" + assert "offline_access" in token_acquisition["human_grant"]["scope"].split() + assert workload_identity["principal_id"] + assert not workload_identity["principal_id"].startswith(principal_contract["internal_service_prefix_reserved"]) + assert workload_identity["expected_groups"] + assert workload_contract["audience"] == "nemo-platform" + assert workload_contract["groups_format"] == "comma_string" + assert workload_contract["forwarded_headers"]["principal_id"] == "X-NMP-Principal-Id" + assert workload_contract["forwarded_headers"]["principal_groups"] == "X-NMP-Principal-Groups" + + +def test_authentik_manifest_declares_extended_startup_timeouts_for_real_oidc(): + manifest = yaml.safe_load(Path("contrib/auth/authentik/manifest.yaml").read_text()) + startup_timeouts = manifest["startup_timeouts"] + + assert startup_timeouts["healthchecks_seconds"] >= 240 + assert startup_timeouts["gateway_seconds"] >= 30 + assert startup_timeouts["token_endpoint_seconds"] >= 60 + + +def test_authentik_provider_config_loads_token_acquisition_fields(): + provider = next(config for config in load_provider_configs() if config.name == "authentik") + + assert provider.nemo_config == Path("contrib/auth/authentik/config/platform-compose-authentik.yaml") + assert provider.token_endpoint == "http://127.0.0.1:18080/application/o/token/" + assert provider.human_grant["grant_type"] == "password" + assert provider.machine_grant["grant_type"] == "password" + assert provider.workload_audience == "nemo-platform" + assert provider.workload_principal_claim == "sub" + assert provider.workload_groups_claim == "groups" + assert provider.workload_groups_format == "comma_string" + assert provider.workload_token_env_vars == ["NEMO_WORKLOAD_TOKEN", "NEMO_WORKLOAD_TOKEN_FILE"] + assert provider.startup_timeouts == { + "healthchecks_seconds": 600, + "gateway_seconds": 30, + "token_endpoint_seconds": 180, + } diff --git a/tests/auth_idp/xdist.py b/tests/auth_idp/xdist.py new file mode 100644 index 0000000000..2333be5873 --- /dev/null +++ b/tests/auth_idp/xdist.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + + +def append_xdist_group_suffix(nodeid: str, groups: set[str]) -> str: + if not groups: + return nodeid + if nodeid.rfind("@") > nodeid.rfind("]"): + return nodeid + return f"{nodeid}@{'_'.join(sorted(groups))}" diff --git a/tests/test_e2e_docker_compose_backend.py b/tests/test_e2e_docker_compose_backend.py new file mode 100644 index 0000000000..55237fda9b --- /dev/null +++ b/tests/test_e2e_docker_compose_backend.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import subprocess +from pathlib import Path + +import pytest + +from e2e.backends.docker_compose import DockerComposeE2EBackend + + +def _compose_ps_json(*entries: dict[str, str]) -> str: + return json.dumps(list(entries)) + + +def test_compose_backend_injects_generated_nemo_config_path(monkeypatch, tmp_path: Path) -> None: + commands: list[tuple[list[str], dict[str, str] | None]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + + def fake_run(args, *, check, text=False, capture_output=False, env=None): + commands.append((list(args), env)) + stdout = "" + if capture_output and "config" in args: + stdout = "nemo\ngateway\nauthentik-server\nauthentik-worker\nauthentik-postgres\nauthentik-redis\n" + if capture_output and "ps" in args: + stdout = _compose_ps_json( + {"Service": "nemo", "State": "running"}, + {"Service": "gateway", "State": "running"}, + {"Service": "authentik-server", "State": "running", "Health": "healthy"}, + {"Service": "authentik-worker", "State": "running", "Health": "healthy"}, + {"Service": "authentik-postgres", "State": "running", "Health": "healthy"}, + {"Service": "authentik-redis", "State": "running", "Health": "healthy"}, + ) + return subprocess.CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + + class Response: + status_code = 200 + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + monkeypatch.setattr("e2e.backends.docker_compose.httpx.get", lambda *args, **kwargs: Response()) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + wait_url="http://127.0.0.1:38080/apis/auth/discovery", + env={"AUTHENTIK_GATEWAY_PORT": "38080"}, + ) + + backend.start() + + assert commands + first_env = commands[0][1] + assert first_env is not None + assert first_env["AUTHENTIK_GATEWAY_PORT"] == "38080" + assert first_env["NEMO_COMPOSE_CONFIG_PATH"] == str(config_path.resolve()) + + +def test_compose_backend_stop_uses_same_project_and_env(monkeypatch, tmp_path: Path) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + + def fake_run(args, *, check, text=False, capture_output=False, env=None): + calls.append((list(args), env)) + return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + env={"AUTHENTIK_GATEWAY_PORT": "38080"}, + ) + + backend.stop() + + args, env = calls[0] + assert args[:6] == ["docker", "compose", "-f", str(compose_file), "-p", "authentik-e2e-test"] + assert args[6:] == ["down", "-v"] + assert env is not None + assert env["AUTHENTIK_GATEWAY_PORT"] == "38080" + assert env["NEMO_COMPOSE_CONFIG_PATH"] == str(config_path.resolve()) + + +def test_compose_backend_write_logs_uses_same_project_and_env(monkeypatch, tmp_path: Path) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + log_path = tmp_path / "compose.log" + + def fake_run(args, *, check, text=False, stdout=None, stderr=None, env=None, **_kwargs): + calls.append((list(args), env)) + assert stdout is not None + stdout.write("nemo log line\n") + return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + env={"AUTHENTIK_GATEWAY_PORT": "38080"}, + ) + + backend.write_logs(log_path) + + args, env = calls[0] + assert args[:6] == ["docker", "compose", "-f", str(compose_file), "-p", "authentik-e2e-test"] + assert args[6:] == ["logs", "--no-color", "--timestamps"] + assert env is not None + assert env["AUTHENTIK_GATEWAY_PORT"] == "38080" + assert env["NEMO_COMPOSE_CONFIG_PATH"] == str(config_path.resolve()) + assert log_path.read_text(encoding="utf-8") == "nemo log line\n" + + +def test_compose_backend_reuse_mode_reuses_healthy_stack_without_restart(monkeypatch, tmp_path: Path) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + + def fake_run(args, *, check, text=False, capture_output=False, env=None): + calls.append((list(args), env)) + stdout = "" + if capture_output and "config" in args: + stdout = "nemo\ngateway\n" + if args[6:] == ["ps", "--services", "--status", "running"]: + raise AssertionError("compose readiness should use health-aware ps json, not running-only services") + if capture_output and "ps" in args: + stdout = _compose_ps_json( + {"Service": "nemo", "State": "running"}, + {"Service": "gateway", "State": "running"}, + ) + return subprocess.CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + + class Response: + status_code = 200 + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + monkeypatch.setattr("e2e.backends.docker_compose.httpx.get", lambda *args, **kwargs: Response()) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + wait_url="http://127.0.0.1:38080/apis/auth/discovery", + lifecycle="reuse", + ) + + backend.start() + + assert [args[6:] for args, _env in calls] == [ + ["config", "--services"], + ["ps", "--all", "--format", "json"], + ] + + +def test_compose_backend_fails_when_compose_config_discovers_no_services(monkeypatch, tmp_path: Path) -> None: + calls: list[list[str]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + + def fake_run(args, *, check, text=False, capture_output=False, env=None): + calls.append(list(args)) + if capture_output and args[6:] == ["config", "--services"]: + return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + raise AssertionError(f"unexpected compose command after empty service discovery: {args[6:]}") + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + ) + + with pytest.raises(RuntimeError, match="no services were discovered"): + backend.start() + + assert [args[6:] for args in calls] == [["config", "--services"]] + + +def test_compose_backend_waits_for_healthy_services_before_ready_probe(monkeypatch, tmp_path: Path) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + readiness_checks = 0 + ready_probe_calls = 0 + + def fake_run(args, *, check, text=False, capture_output=False, env=None): + nonlocal readiness_checks + calls.append((list(args), env)) + stdout = "" + if capture_output and "config" in args: + stdout = "nemo\ngateway\n" + if args[6:] == ["ps", "--services", "--status", "running"]: + raise AssertionError("compose readiness should use health-aware ps json, not running-only services") + if capture_output and "ps" in args: + readiness_checks += 1 + stdout = ( + _compose_ps_json( + {"Service": "nemo", "State": "running"}, + {"Service": "gateway", "State": "running", "Health": "starting"}, + ) + if readiness_checks == 1 + else _compose_ps_json( + {"Service": "nemo", "State": "running"}, + {"Service": "gateway", "State": "running", "Health": "healthy"}, + ) + ) + return subprocess.CompletedProcess(args=args, returncode=0, stdout=stdout, stderr="") + + class Response: + status_code = 200 + + def fake_get(*args, **kwargs): + nonlocal ready_probe_calls + ready_probe_calls += 1 + assert readiness_checks == 2 + return Response() + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + monkeypatch.setattr("e2e.backends.docker_compose.httpx.get", fake_get) + monkeypatch.setattr("e2e.backends.docker_compose.time.sleep", lambda _seconds: None) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + ) + + backend.start() + + assert ready_probe_calls == 1 + assert [args[6:] for args, _env in calls] == [ + ["config", "--services"], + ["down", "-v"], + ["up", "-d"], + ["ps", "--all", "--format", "json"], + ["ps", "--all", "--format", "json"], + ] + + +def test_compose_backend_stop_is_noop_in_reuse_mode(monkeypatch, tmp_path: Path) -> None: + calls: list[tuple[list[str], dict[str, str] | None]] = [] + compose_file = tmp_path / "docker-compose.yml" + compose_file.write_text("services: {}\n") + config_path = tmp_path / "platform.yaml" + config_path.write_text("platform: {}\n") + + def fake_run(args, *, check, text=False, capture_output=False, env=None): + calls.append((list(args), env)) + return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="") + + monkeypatch.setattr("e2e.backends.docker_compose.subprocess.run", fake_run) + + backend = DockerComposeE2EBackend( + compose_file=compose_file, + config_path=config_path, + project_name="authentik-e2e-test", + service_url="http://127.0.0.1:38080", + lifecycle="reuse", + ) + + backend.stop() + + assert calls == [] diff --git a/tests/test_e2e_services_pool.py b/tests/test_e2e_services_pool.py index 744fbffabc..1b2b8f019a 100644 --- a/tests/test_e2e_services_pool.py +++ b/tests/test_e2e_services_pool.py @@ -1,60 +1,229 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from pathlib import Path - -import httpx import e2e.services_pool as services_pool -from e2e.services_pool import E2EServicesPool, ModuleConfigState, ServicesPoolKey -class _StubModule: - def __init__(self, nodeid: str) -> None: - self.nodeid = nodeid +def test_render_e2e_config_for_docker_preserves_container_paths(tmp_path) -> None: + config = { + "jobs": { + "executors": [ + { + "provider": "subprocess", + "config": {"working_directory": "/data/subprocess-jobs"}, + } + ] + }, + "files": {"default_storage_config": {"type": "local", "path": "/data/files"}}, + } + rendered = services_pool._render_e2e_config_for_backend(config, tmp_path, {"backend": "docker"}) -class _ExitedProc: - def poll(self) -> int: - return 1 + assert rendered["jobs"]["executors"][0]["config"]["working_directory"] == "/data/subprocess-jobs" + assert rendered["files"]["default_storage_config"]["path"] == "/data/files" -def test_acquire_for_module_preserves_auth_for_external_url(monkeypatch) -> None: - pool = E2EServicesPool() - module = _StubModule("e2e/test_example.py") - pool._module_states[module.nodeid] = ModuleConfigState( - module_id=module.nodeid, - key=ServicesPoolKey(config_hash="abc123"), - config_path=Path("/tmp/platform.yaml"), - config_data={}, - config_layers=(), - auth_enabled=True, - ) - monkeypatch.setenv("NMP_BASE_URL", "http://external.example") +def test_render_e2e_config_for_subprocess_rewrites_instance_paths(tmp_path) -> None: + config = { + "jobs": { + "executors": [ + { + "provider": "subprocess", + "config": {"working_directory": ".tmp/e2e/subprocess-jobs"}, + } + ] + }, + "files": {"default_storage_config": {"type": "local", "path": ".tmp/e2e/files"}}, + } + + rendered = services_pool._render_e2e_config_for_backend(config, tmp_path, {"backend": "subprocess"}) + + assert rendered["jobs"]["executors"][0]["config"]["working_directory"] == str(tmp_path / "subprocess-jobs") + assert rendered["files"]["default_storage_config"]["path"] == str(tmp_path / "files") + + +def test_docker_backend_overrides_prefer_e2e_specific_env(monkeypatch) -> None: + monkeypatch.setenv("IMAGE_REGISTRY", "ghcr.io/example/default") + monkeypatch.setenv("BAKE_TAG", "default-tag") + monkeypatch.setenv("NMP_E2E_IMAGE_REGISTRY", "ghcr.io/example/e2e") + monkeypatch.setenv("NMP_E2E_IMAGE_TAG", "e2e-tag") + + overrides = services_pool._docker_backend_overrides() + + assert overrides == { + "registry": "ghcr.io/example/e2e", + "tag": "e2e-tag", + } + + +def test_docker_backend_overrides_fall_back_to_ci_bake_env(monkeypatch) -> None: + monkeypatch.delenv("NMP_E2E_IMAGE_REGISTRY", raising=False) + monkeypatch.delenv("NMP_E2E_IMAGE_TAG", raising=False) + monkeypatch.setenv("IMAGE_REGISTRY", "ghcr.io/example/default") + monkeypatch.setenv("BAKE_TAG", "default-tag") + + overrides = services_pool._docker_backend_overrides() + + assert overrides == { + "registry": "ghcr.io/example/default", + "tag": "default-tag", + } + + +def test_render_e2e_config_for_docker_compose_preserves_container_paths(tmp_path) -> None: + config = { + "jobs": { + "executors": [ + { + "provider": "subprocess", + "config": {"working_directory": "/data/subprocess-jobs"}, + } + ] + }, + "files": {"default_storage_config": {"type": "local", "path": "/data/files"}}, + } + + rendered = services_pool._render_e2e_config_for_backend(config, tmp_path, {"backend": "docker_compose"}) + + assert rendered["jobs"]["executors"][0]["config"]["working_directory"] == "/data/subprocess-jobs" + assert rendered["files"]["default_storage_config"]["path"] == "/data/files" + + +def test_start_services_docker_compose_waits_for_auth_ready_when_enabled(tmp_path, monkeypatch) -> None: + compose_file = tmp_path / "compose.yaml" + compose_file.write_text("services: {}\n", encoding="utf-8") + config_path = tmp_path / "config.yaml" + config_path.write_text("{}\n", encoding="utf-8") + + class FakeDockerComposeBackend: + def __init__(self, **kwargs) -> None: + self.service_url = kwargs["service_url"] + + def start(self) -> None: + return None + + def stop(self) -> None: + return None + + def write_logs(self, log_path) -> None: + log_path.write_text("compose logs\n", encoding="utf-8") + + wait_calls = [] + + def fake_wait_for_auth_ready(url, proc) -> bool: + wait_calls.append((url, proc)) + return True - services = pool.acquire_for_module(module) + monkeypatch.setattr(services_pool, "DockerComposeE2EBackend", FakeDockerComposeBackend) + monkeypatch.setattr(services_pool, "_wait_for_auth_ready", fake_wait_for_auth_ready) + + services = services_pool._start_services_docker_compose( + config_path, + {"auth": {"enabled": True}}, + { + "backend": "docker_compose", + "compose_file": str(compose_file), + "service_url": "http://127.0.0.1:8080", + "lifecycle": "fresh", + }, + "abc123", + tmp_path / "services.log", + ) - assert services.url == "http://external.example" + assert wait_calls == [("http://127.0.0.1:8080", None)] assert services.auth_enabled is True + assert services.url == "http://127.0.0.1:8080" -def test_wait_for_healthy_returns_false_immediately_when_process_has_exited(monkeypatch) -> None: - monkeypatch.setattr( - services_pool.httpx, - "get", - lambda *args, **kwargs: (_ for _ in ()).throw(httpx.RequestError("down")), - ) - monkeypatch.setattr(services_pool.time, "sleep", lambda _: (_ for _ in ()).throw(AssertionError("slept"))) +def test_start_services_docker_compose_uses_auth_ready_url_when_configured(tmp_path, monkeypatch) -> None: + compose_file = tmp_path / "compose.yaml" + compose_file.write_text("services: {}\n", encoding="utf-8") + config_path = tmp_path / "config.yaml" + config_path.write_text("{}\n", encoding="utf-8") + + class FakeDockerComposeBackend: + def __init__(self, **kwargs) -> None: + self.service_url = kwargs["service_url"] + + def start(self) -> None: + return None + + def stop(self) -> None: + return None - assert services_pool._wait_for_healthy("http://example.com", _ExitedProc(), timeout=0.1) is False + def write_logs(self, log_path) -> None: + log_path.write_text("compose logs\n", encoding="utf-8") + wait_calls = [] -def test_wait_for_auth_ready_returns_false_immediately_when_process_has_exited(monkeypatch) -> None: - monkeypatch.setattr( - services_pool.httpx, - "post", - lambda *args, **kwargs: (_ for _ in ()).throw(httpx.RequestError("down")), + def fake_wait_for_auth_ready(url, proc) -> bool: + wait_calls.append((url, proc)) + return True + + monkeypatch.setattr(services_pool, "DockerComposeE2EBackend", FakeDockerComposeBackend) + monkeypatch.setattr(services_pool, "_wait_for_auth_ready", fake_wait_for_auth_ready) + + services = services_pool._start_services_docker_compose( + config_path, + {"auth": {"enabled": True}}, + { + "backend": "docker_compose", + "compose_file": str(compose_file), + "service_url": "http://127.0.0.1:38080", + "auth_ready_url": "http://127.0.0.1:38081", + "lifecycle": "fresh", + }, + "abc123", + tmp_path / "services.log", ) - monkeypatch.setattr(services_pool.time, "sleep", lambda _: (_ for _ in ()).throw(AssertionError("slept"))) - assert services_pool._wait_for_auth_ready("http://example.com", _ExitedProc(), timeout=0.1) is False + assert wait_calls == [("http://127.0.0.1:38081", None)] + assert services.auth_enabled is True + assert services.url == "http://127.0.0.1:38080" + + +def test_start_services_docker_compose_exposes_log_path_and_captures_logs_on_close(tmp_path, monkeypatch) -> None: + compose_file = tmp_path / "compose.yaml" + compose_file.write_text("services: {}\n", encoding="utf-8") + config_path = tmp_path / "config.yaml" + config_path.write_text("{}\n", encoding="utf-8") + log_path = tmp_path / "services.log" + events = [] + + class FakeDockerComposeBackend: + def __init__(self, **kwargs) -> None: + self.service_url = kwargs["service_url"] + + def start(self) -> None: + events.append("start") + + def stop(self) -> None: + events.append("stop") + + def write_logs(self, path) -> None: + events.append(("logs", path)) + path.write_text("compose logs\n", encoding="utf-8") + + monkeypatch.setattr(services_pool, "DockerComposeE2EBackend", FakeDockerComposeBackend) + + services = services_pool._start_services_docker_compose( + config_path, + {}, + { + "backend": "docker_compose", + "compose_file": str(compose_file), + "service_url": "http://127.0.0.1:38080", + "lifecycle": "fresh", + }, + "abc123", + log_path, + ) + + assert services.log_path == log_path + assert services.close is not None + + services.close() + + assert events == ["start", ("logs", log_path), "stop"] + assert log_path.read_text(encoding="utf-8") == "compose logs\n" diff --git a/uv.lock b/uv.lock index 19990f1ce0..3826fc8753 100644 --- a/uv.lock +++ b/uv.lock @@ -2207,11 +2207,11 @@ wheels = [ [[package]] name = "idna" -version = "3.15" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]]