diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index 68f6d952cd1e..952b472cde8a 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -174,7 +174,7 @@ class OverridesSpec(BaseModel): profilingJob: Optional[Dict[str, Any]] = Field( default=None, - description="ProfilingJob allows overriding the profiling Job specification. Fields set here are merged into the controller-generated Job spec.", + description="ProfilingJob allows overriding the profiling Job specification. Fields set here are merged into the controller-generated Job spec. Security: creating a DGDR is workload-creation authority in its namespace — these overrides carry the same blast radius as creating a Job or Pod directly there, by design. Pod security is enforced centrally by Kubernetes Pod Security Admission on the resulting Pods once the namespace is labeled (see pod-security.kubernetes.io/enforce): it applies the full Pod Security Standards — covering privileged, host namespaces, and hostPath, not only securityContext — not this API. ServiceAccount identity is a separate layer: these overrides can set serviceAccountName and automountServiceAccountToken, which are bounded by RBAC and namespace membership rather than PSA — the same authority any Pod author in the namespace already holds. Grant create/update on DGDRs only to principals trusted to create Pods in the namespace. The profiling Job always runs in the DGDR's own namespace and overrides cannot change that — but namespace containment is not node or cross-tenant isolation. Dynamo's workloads, including this Job, satisfy the baseline standard, so enforce baseline (non-exempt) on every resulting Pod to close the privileged, host-namespace, host-device, and hostPath paths.", ) trustRemoteCode: bool = Field( default=False, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index 2ed3141deb05..f874bff74678 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -296,6 +296,22 @@ type ModelCacheSpec struct { type OverridesSpec struct { // ProfilingJob allows overriding the profiling Job specification. // Fields set here are merged into the controller-generated Job spec. + // + // Security: creating a DGDR is workload-creation authority in its namespace — + // these overrides carry the same blast radius as creating a Job or Pod directly + // there, by design. Pod security is enforced centrally by Kubernetes Pod Security + // Admission on the resulting Pods once the namespace is labeled (see + // pod-security.kubernetes.io/enforce): it applies the full Pod Security Standards — + // covering privileged, host namespaces, and hostPath, not only securityContext — + // not this API. ServiceAccount identity is a separate layer: these overrides can + // set serviceAccountName and automountServiceAccountToken, which are bounded by + // RBAC and namespace membership rather than PSA — the same authority any Pod author + // in the namespace already holds. Grant create/update on DGDRs only to principals + // trusted to create Pods in the namespace. The profiling Job always runs in the + // DGDR's own namespace and overrides cannot change that — but namespace containment + // is not node or cross-tenant isolation. Dynamo's workloads, including this Job, + // satisfy the baseline standard, so enforce baseline (non-exempt) on every resulting + // Pod to close the privileged, host-namespace, host-device, and hostPath paths. // +optional ProfilingJob *batchv1.JobSpec `json:"profilingJob,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index e92bf18a3214..39467c831d18 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -776,6 +776,22 @@ spec: description: |- ProfilingJob allows overriding the profiling Job specification. Fields set here are merged into the controller-generated Job spec. + + Security: creating a DGDR is workload-creation authority in its namespace — + these overrides carry the same blast radius as creating a Job or Pod directly + there, by design. Pod security is enforced centrally by Kubernetes Pod Security + Admission on the resulting Pods once the namespace is labeled (see + pod-security.kubernetes.io/enforce): it applies the full Pod Security Standards — + covering privileged, host namespaces, and hostPath, not only securityContext — + not this API. ServiceAccount identity is a separate layer: these overrides can + set serviceAccountName and automountServiceAccountToken, which are bounded by + RBAC and namespace membership rather than PSA — the same authority any Pod author + in the namespace already holds. Grant create/update on DGDRs only to principals + trusted to create Pods in the namespace. The profiling Job always runs in the + DGDR's own namespace and overrides cannot change that — but namespace containment + is not node or cross-tenant isolation. Dynamo's workloads, including this Job, + satisfy the baseline standard, so enforce baseline (non-exempt) on every resulting + Pod to close the privileged, host-namespace, host-device, and hostPath paths. properties: activeDeadlineSeconds: description: |- diff --git a/docs/fern/pages/kubernetes/auto-deployment/auto-deploy-with-dgdr.md b/docs/fern/pages/kubernetes/auto-deployment/auto-deploy-with-dgdr.md index bb8c4a2d38cb..43849cc30cb5 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/auto-deploy-with-dgdr.md +++ b/docs/fern/pages/kubernetes/auto-deployment/auto-deploy-with-dgdr.md @@ -452,6 +452,49 @@ replacement behavior and requires the complete desired argument list. For the complete merge, metadata, and validation rules, see [DGDR Reference — Generated DGD overrides](../../reference/kubernetes-api/dynamo-graph-deployment-request.mdx#generated-dgd-overrides). +### Profiling job overrides and the trust boundary + +`spec.overrides.profilingJob` accepts a partial Kubernetes `JobSpec` that the operator merges +into the profiling Job it launches (for example, to add tolerations or adjust resources). Because +the operator creates that Job on your behalf, treat this the way Kubernetes treats any +workload-creation API: + +> [!IMPORTANT] +> Any principal that can create workloads in a namespace where Dynamo runs — a `Pod` directly, or +> any resource that creates Pods (`Job`, `Deployment`, `DynamoGraphDeploymentRequest`, …) — is +> inside that namespace's trust boundary: it can run code with the Secrets and ServiceAccount +> tokens mounted in that namespace. Creating a DGDR is one such path and is no more privileged than +> creating a `Job` or `Pod` there — including through `overrides.profilingJob`. This is by design +> and matches how Kubernetes treats every Pod-spawning resource. +> +> Enforce Pod security **centrally on the resulting Pods** with +> [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) +> (and any admission webhooks), which applies the full +> [Pod Security Standards](https://kubernetes.io/docs/concepts/security/pod-security-standards/) — +> not only `securityContext` fields but also `privileged`, host namespaces, `hostPath` volumes, and +> host ports. PSA applies no policy until you label the namespace — set +> `pod-security.kubernetes.io/enforce: ` (plus the matching `audit`/`warn` labels) on every +> namespace where Dynamo runs, exactly as you would for any workload. The operator does not +> re-implement those checks. +> +> Dynamo's operator-generated workloads — including the DGDR profiling Job — satisfy the **`baseline`** +> standard. Enforce **`baseline`** to block the privileged-container, host-namespace, host-device, and +> `hostPath` escalation paths while keeping Dynamo running. +> +> PSA governs a Pod's security *posture*, not its *identity*: `overrides.profilingJob` can set the +> Job's `serviceAccountName` and `automountServiceAccountToken`, and those are bounded by RBAC and +> namespace membership, not by PSA — the same authority any Pod author in the namespace already has. +> So grant `create`/`update` on DGDRs — and on workload resources generally — only to principals you +> would trust to create Pods in that namespace, and use namespaces as the tenancy boundary — see +> [Kubernetes RBAC good practices](https://kubernetes.io/docs/concepts/security/rbac-good-practices/#workload-creation). + +The profiling Job object always remains in the DGDR's own namespace and overrides cannot relocate +it — but namespace containment is not node or cross-tenant isolation. If admission permits +privileged containers, host namespaces, host devices, or `hostPath` mounts, a DGDR creator can +obtain those capabilities through the operator, exactly as any Pod author in the namespace could. +Enforcing `baseline` non-exempt on every resulting Pod closes those paths; use namespaces as the +tenancy boundary for anything stronger. + ## Next steps | Goal | Guide | diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index fcdc3c856cd6..c255419e3a20 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2578,7 +2578,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `profilingJob` _[JobSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#jobspec-v1-batch)_ | ProfilingJob allows overriding the profiling Job specification.
Fields set here are merged into the controller-generated Job spec. | | Optional: \{\}
| +| `profilingJob` _[JobSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#jobspec-v1-batch)_ | ProfilingJob allows overriding the profiling Job specification.
Fields set here are merged into the controller-generated Job spec.
Security: creating a DGDR is workload-creation authority in its namespace —
these overrides carry the same blast radius as creating a Job or Pod directly
there, by design. Pod security is enforced centrally by Kubernetes Pod Security
Admission on the resulting Pods once the namespace is labeled (see
pod-security.kubernetes.io/enforce): it applies the full Pod Security Standards —
covering privileged, host namespaces, and hostPath, not only securityContext —
not this API. ServiceAccount identity is a separate layer: these overrides can
set serviceAccountName and automountServiceAccountToken, which are bounded by
RBAC and namespace membership rather than PSA — the same authority any Pod author
in the namespace already holds. Grant create/update on DGDRs only to principals
trusted to create Pods in the namespace. The profiling Job always runs in the
DGDR's own namespace and overrides cannot change that — but namespace containment
is not node or cross-tenant isolation. Dynamo's workloads, including this Job,
satisfy the baseline standard, so enforce baseline (non-exempt) on every resulting
Pod to close the privileged, host-namespace, host-device, and hostPath paths. | | Optional: \{\}
| | `trustRemoteCode` _boolean_ | TrustRemoteCode explicitly permits generated vLLM and SGLang workers to
execute custom code from the configured model repository. When enabled,
the profiler adds --trust-remote-code to every generated worker component
after the deployment topology has been generated. Enable this setting only
for model repositories you trust. | false | Optional: \{\}
| | `dgd` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | DGD provides a partial, versioned DynamoGraphDeployment override for the
profiler-generated deployment. Set apiVersion to nvidia.com/v1alpha1 or
nvidia.com/v1beta1 and kind to DynamoGraphDeployment.
The profiler merges the override using the schema for its declared version.
If the generated DGD uses another supported version, the complete DGD is
converted before the merge and converted back afterward. The final DGD
selected or created by a DGDR is nvidia.com/v1beta1.
The override can update DGD fields, but topology entries are limited to
services or components already present in the generated DGD. Metadata labels
and annotations are merged, metadata.name selects the final DGD name, and
other identity or runtime metadata is ignored.
V1alpha1 worker argument lists retain legacy append behavior. V1beta1 follows
structural schema merge behavior, including map-list merging and atomic-list
replacement.
The raw embedded resource preserves either supported schema. The API server
validates that it has apiVersion and kind; override processing validates the
DGD kind, supported version, and field schema. | | EmbeddedResource: \{\}
Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 7e840fd5b403..a6a643fb03dd 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3689,7 +3689,7 @@ OverridesSpec allows customizing the profiling job and the generated DynamoGraph **Appears in:** [DynamoGraphDeploymentRequestSpec](#v1beta1-dynamographdeploymentrequestspec) -ProfilingJob allows overriding the profiling Job specification. Fields set here are merged into the controller-generated Job spec. +ProfilingJob allows overriding the profiling Job specification. Fields set here are merged into the controller-generated Job spec. Security: creating a DGDR is workload-creation authority in its namespace — these overrides carry the same blast radius as creating a Job or Pod directly there, by design. Pod security is enforced centrally by Kubernetes Pod Security Admission on the resulting Pods once the namespace is labeled (see pod-security.kubernetes.io/enforce): it applies the full Pod Security Standards — covering privileged, host namespaces, and hostPath, not only securityContext — not this API. ServiceAccount identity is a separate layer: these overrides can set serviceAccountName and automountServiceAccountToken, which are bounded by RBAC and namespace membership rather than PSA — the same authority any Pod author in the namespace already holds. Grant create/update on DGDRs only to principals trusted to create Pods in the namespace. The profiling Job always runs in the DGDR's own namespace and overrides cannot change that — but namespace containment is not node or cross-tenant isolation. Dynamo's workloads, including this Job, satisfy the baseline standard, so enforce baseline (non-exempt) on every resulting Pod to close the privileged, host-namespace, host-device, and hostPath paths. See [JobSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#jobspec-v1-batch). **Validation:** Optional: \{\}