diff --git a/deploy/helm/charts/platform/components/operator/files/manager-role.yaml b/deploy/helm/charts/platform/components/operator/files/manager-role.yaml index 1903be0a30f8..60936ba8de17 100644 --- a/deploy/helm/charts/platform/components/operator/files/manager-role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/manager-role.yaml @@ -146,6 +146,18 @@ rules: - patch - update - watch +- apiGroups: + - disaggregatedset.x-k8s.io + resources: + - disaggregatedsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - discovery.k8s.io resources: diff --git a/deploy/operator/config/rbac/role.yaml b/deploy/operator/config/rbac/role.yaml index 1903be0a30f8..60936ba8de17 100644 --- a/deploy/operator/config/rbac/role.yaml +++ b/deploy/operator/config/rbac/role.yaml @@ -146,6 +146,18 @@ rules: - patch - update - watch +- apiGroups: + - disaggregatedset.x-k8s.io + resources: + - disaggregatedsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - discovery.k8s.io resources: diff --git a/deploy/operator/go.mod b/deploy/operator/go.mod index 5fa2acd6b13f..e182fb535de4 100644 --- a/deploy/operator/go.mod +++ b/deploy/operator/go.mod @@ -13,8 +13,8 @@ require ( github.com/go-logr/zapr v1.3.0 github.com/google/go-cmp v0.7.0 github.com/imdario/mergo v0.3.16 - github.com/onsi/ginkgo/v2 v2.30.0 - github.com/onsi/gomega v1.41.0 + github.com/onsi/ginkgo/v2 v2.32.0 + github.com/onsi/gomega v1.42.1 github.com/open-policy-agent/cert-controller v0.16.0 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.71.2 @@ -30,15 +30,15 @@ require ( helm.sh/helm/v3 v3.20.2 istio.io/api v1.23.1 istio.io/client-go v1.23.1 - k8s.io/api v0.36.2 - k8s.io/apiextensions-apiserver v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apiextensions-apiserver v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 k8s.io/klog/v2 v2.140.0 k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 sigs.k8s.io/controller-runtime v0.24.1 sigs.k8s.io/gateway-api-inference-extension v1.2.0 - sigs.k8s.io/lws v0.9.0 + sigs.k8s.io/lws v0.10.0 sigs.k8s.io/randfill v1.0.0 sigs.k8s.io/yaml v1.6.0 volcano.sh/apis v1.12.2 @@ -50,7 +50,7 @@ require ( github.com/BurntSushi/toml v1.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -68,7 +68,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.26.0 // indirect + github.com/google/cel-go v0.29.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/google/uuid v1.6.0 // indirect @@ -92,7 +92,6 @@ require ( github.com/spf13/cast v1.7.0 // indirect github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect @@ -122,12 +121,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.36.2 // indirect - k8s.io/component-base v0.36.2 // indirect - k8s.io/component-helpers v0.36.2 // indirect + k8s.io/apiserver v0.36.3 // indirect + k8s.io/component-base v0.36.3 // indirect + k8s.io/component-helpers v0.36.3 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect - k8s.io/streaming v0.36.2 // indirect + k8s.io/streaming v0.36.3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect ) diff --git a/deploy/operator/go.sum b/deploy/operator/go.sum index 12356b135d35..603dd46f5b99 100644 --- a/deploy/operator/go.sum +++ b/deploy/operator/go.sum @@ -20,8 +20,8 @@ github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.12-rc1 h1:rhi1HzLU68RO573k0 github.com/ai-dynamo/grove/operator/api v0.1.0-alpha.12-rc1/go.mod h1:aebLeDCrknulT10/uDj2G1i9K2MEIfVAa25bzrvWxsQ= github.com/ai-dynamo/snapshot/api v0.1.0-alpha.1 h1:WUEyrMmjy46L3bQqomimkLK5xzqaRYCPdJUChc8pBAE= github.com/ai-dynamo/snapshot/api v0.1.0-alpha.1/go.mod h1:O//3+6819boTBRdyWh1aSbpSpIA/Eaz6T5DzOCnTXZg= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -90,8 +90,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= -github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= +github.com/google/cel-go v0.29.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -151,10 +151,10 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.30.0 h1:zxM/9XneXFIy64j6/wAmBIX4zRC7Hu6U8XFNZvDnCQc= -github.com/onsi/ginkgo/v2 v2.30.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= -github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= +github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= +github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/open-policy-agent/cert-controller v0.16.0 h1:/w+wAM9dylWGwhPnvfCgepyBLfyz19PIa8YeaThsQDg= github.com/open-policy-agent/cert-controller v0.16.0/go.mod h1:w5qBWYbc8HwyHI9VYAZ6YjWOcZtQ39A30I9W4X7pVVk= github.com/open-policy-agent/frameworks/constraint v0.0.0-20241101234656-e78c8abd754a h1:gQtOJ50XFyL2Xh3lDD9zP4KQ2PY4mZKQ9hDcWc81Sp8= @@ -188,17 +188,10 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -294,7 +287,6 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnf gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= helm.sh/helm/v3 v3.20.2 h1:binM4rvPx5DcNsa1sIt7UZi55lRbu3pZUFmQkSoRh48= @@ -303,28 +295,28 @@ istio.io/api v1.23.1 h1:bm2XF0j058FfzWVHUfpmMj4sFDkcD1X609qs5AU97Pc= istio.io/api v1.23.1/go.mod h1:QPSTGXuIQdnZFEm3myf9NZ5uBMwCdJWUvfj9ZZ+2oBM= istio.io/client-go v1.23.1 h1:IX2cgUUXnVYo+9H6bFGSp/vuKVLPUkmiN8qk1/mvsYs= istio.io/client-go v1.23.1/go.mod h1:+fxu+O2GkITM3HEREUWdobvRXqI/UhAAI7hfxqqpRh0= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= -k8s.io/apiextensions-apiserver v0.36.2 h1:3O5gqOj/dt2XWWbpMe+TXWpE9yU6pjM/tXxtHHJT/K4= -k8s.io/apiextensions-apiserver v0.36.2/go.mod h1:cL1tBWe8XSaP1H30iWKGo7hf6iAUUUJPEU70dskmAnA= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= -k8s.io/apiserver v0.36.2 h1:6vMnkmHZPeBloNkHUhmZYq7Ylv8WIB8xjyEl+eSt26E= -k8s.io/apiserver v0.36.2/go.mod h1:9PoQ2ikCytrZyZg11mGhLEF5m8Rgsb5FJmYJ4Wvnl1k= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= -k8s.io/component-base v0.36.2 h1:Z0VH80O7Ng0HDZnZj3WRR3urEGa0kTwmO8CwEwjVK1w= -k8s.io/component-base v0.36.2/go.mod h1:mGfFOA7Gwpdm1VW2cwSQYbiDIlz8GD2WGwH88QSeCyA= -k8s.io/component-helpers v0.36.2 h1:YsqocS183ThSUw90OXsxkKxIgdQF4qWInwrn6pZdDH8= -k8s.io/component-helpers v0.36.2/go.mod h1:YrHgzezjsyXAFq9+gKw6IbgJg7IHEUVwK41eEAiTRR4= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= +k8s.io/apiextensions-apiserver v0.36.3 h1:dPmOAPhwTtqb1bTxbFPsy18KHPhktQeO3WUPXunZIB0= +k8s.io/apiextensions-apiserver v0.36.3/go.mod h1:KTXFqgXiuw2pRoL+Wpmttqc+up9Xt/GohadPWeLLOa4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= +k8s.io/apiserver v0.36.3 h1:MGSg2SkdfuytiDEcRylT5mQFmmSsbx90XFUO67Y4bsQ= +k8s.io/apiserver v0.36.3/go.mod h1:fVH7zv9EUNUA7Fl7LtDKh8aB9W7u1VQPSGtWV5SjUxg= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= +k8s.io/component-base v0.36.3 h1:vc/UFvPCkW0irPz84LAodAL1j3f4xktPM6dDJIEheAY= +k8s.io/component-base v0.36.3/go.mod h1:hZbNFG+gCMl9EbykDGEu73feKP9/Cq6JsV4pTo9GTO8= +k8s.io/component-helpers v0.36.3 h1:hya22S0Mto0SlHaiD4kMIi817f/tK7uTMsShxrDKQaY= +k8s.io/component-helpers v0.36.3/go.mod h1:QjREK1lOFXR+jxTqzrtHgOtzUc2s9sm8zuFSiK+TW+c= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-aggregator v0.35.2 h1:bnF7E238wUOVaPpTyKrqGCAEXOAJ6HRTARvJTZ0UIC0= k8s.io/kube-aggregator v0.35.2/go.mod h1:7Xl9zFJFsFIrPnwBfu7hve+G5QgLsDZRIedc8gA1mq4= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= -k8s.io/streaming v0.36.2 h1:NSKthPPg9UFSKsRauVJUVGH2Dvn8fhKmY4qrMkw/p98= -k8s.io/streaming v0.36.2/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= +k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= @@ -335,12 +327,12 @@ sigs.k8s.io/gateway-api-inference-extension v1.2.0 h1:7H+ijrUImnW2ubcTakNgV723xD sigs.k8s.io/gateway-api-inference-extension v1.2.0/go.mod h1:/HWeqxuOMjFM56YwJ2Spt3qceK7Spz4hk6ZfXYgE9a8= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/lws v0.9.0 h1:DcJCgxjjoGOx//QHZkeKpyiblxYq70aCRZNYSfykWyI= -sigs.k8s.io/lws v0.9.0/go.mod h1:oYkORu4fUxGwA2Q/V4IOXfn5yGpQVDAHAZx2TVbi9LY= +sigs.k8s.io/lws v0.10.0 h1:KqNtGMIJAGj4QpZ9P+F3NixyNyRsnWvCIxDT05xjGTA= +sigs.k8s.io/lws v0.10.0/go.mod h1:bYGhxHiSF1NrOFRMqIBAt3q1pri47w3cx3elU0o5wNQ= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.4.0 h1:qmp2e3ZfFi1/jJbDGpD4mt3wyp6PE1NfKHCYLqgNQJo= -sigs.k8s.io/structured-merge-diff/v6 v6.4.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= volcano.sh/apis v1.12.2 h1:KvNyM/kMizFVlALiH/uFHPwYFHRtxuVnBL0upbFbDss= diff --git a/deploy/operator/internal/consts/consts.go b/deploy/operator/internal/consts/consts.go index 7293c5181c4c..d213aa39df33 100644 --- a/deploy/operator/internal/consts/consts.go +++ b/deploy/operator/internal/consts/consts.go @@ -40,12 +40,14 @@ const ( KubeLabelDynamoSelector = "nvidia.com/selector" - KubeAnnotationEnableGrove = "nvidia.com/enable-grove" + KubeAnnotationEnableGrove = "nvidia.com/enable-grove" + KubeAnnotationEnableDisaggregatedSet = "nvidia.com/enable-disaggregatedset" // KubeAnnotationWorkloadProvider records the controller-owned immutable graph-level workload provider. - KubeAnnotationWorkloadProvider = "nvidia.com/workload-provider" - WorkloadProviderComponent = "component" - WorkloadProviderGrove = "grove" + KubeAnnotationWorkloadProvider = "nvidia.com/workload-provider" + WorkloadProviderComponent = "component" + WorkloadProviderGrove = "grove" + WorkloadProviderDisaggregatedSet = "disaggregatedset" // KubeAnnotationGroveUpdateStrategy temporarily exposes the Grove // PodCliqueSet update strategy while the long-term DGD API is settled. diff --git a/deploy/operator/internal/controller/dgd_component_program.go b/deploy/operator/internal/controller/dgd_component_program.go index d6ac1d0ea45d..7a8d7944b1b1 100644 --- a/deploy/operator/internal/controller/dgd_component_program.go +++ b/deploy/operator/internal/controller/dgd_component_program.go @@ -133,7 +133,6 @@ func (p *componentProgram) Reconcile( return programResult, fmt.Errorf("failed to reconcile scaling adapters: %w", err) } } - programResult.applyReconcileResult(req.DGD.Generation, result) return programResult, nil } diff --git a/deploy/operator/internal/controller/dgd_component_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_component_workloads_reconciler.go index 76390dba9397..3d11daf31956 100644 --- a/deploy/operator/internal/controller/dgd_component_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_component_workloads_reconciler.go @@ -137,6 +137,14 @@ func (r *componentWorkloadsReconciler) Reconcile( func (r *componentWorkloadsReconciler) getExistingRestartAnnotationsDCD( ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) (map[string]string, error) { + return getExistingRestartAnnotationsDCD(ctx, r.syncer, dgd) +} + +func getExistingRestartAnnotationsDCD( + ctx context.Context, + reader client.Reader, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, ) (map[string]string, error) { logger := log.FromContext(ctx) hashes, err := desiredWorkerHashes(dgd) @@ -151,7 +159,7 @@ func (r *componentWorkloadsReconciler) getExistingRestartAnnotationsDCD( existingDCD := &nvidiacomv1beta1.DynamoComponentDeployment{} for _, workerHash := range workerHashes { dcdName := dynamo.GetDCDResourceName(dgd, componentName, workerHash) - err := r.syncer.Get( + err := reader.Get( ctx, types.NamespacedName{Name: dcdName, Namespace: dgd.Namespace}, existingDCD, @@ -181,31 +189,41 @@ func (r *componentWorkloadsReconciler) applyCheckpointStartupPolicy( dcd *nvidiacomv1beta1.DynamoComponentDeployment, checkpointInfo *checkpoint.CheckpointInfo, ) error { - if dcd == nil || checkpointInfo == nil || !checkpointInfo.Enabled { + if dcd == nil { + return nil + } + return applyCheckpointStartupPolicy(&dcd.Spec.DynamoComponentDeploymentSharedSpec, checkpointInfo) +} + +func applyCheckpointStartupPolicy( + component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + checkpointInfo *checkpoint.CheckpointInfo, +) error { + if component == nil || checkpointInfo == nil || !checkpointInfo.Enabled { return nil } if checkpointInfo.Exists && checkpointInfo.CheckpointName != "" { - if dcd.Spec.Experimental == nil { - dcd.Spec.Experimental = &nvidiacomv1beta1.ExperimentalSpec{} + if component.Experimental == nil { + component.Experimental = &nvidiacomv1beta1.ExperimentalSpec{} } - if dcd.Spec.Experimental.Checkpoint == nil { - dcd.Spec.Experimental.Checkpoint = &nvidiacomv1beta1.ComponentCheckpointConfig{} + if component.Experimental.Checkpoint == nil { + component.Experimental.Checkpoint = &nvidiacomv1beta1.ComponentCheckpointConfig{} } checkpointName := checkpointInfo.CheckpointName - dcd.Spec.Experimental.Checkpoint.Enabled = true - dcd.Spec.Experimental.Checkpoint.CheckpointRef = &checkpointName - dcd.Spec.Experimental.Checkpoint.Identity = nil - dcd.Spec.Experimental.Checkpoint.Job = nil + component.Experimental.Checkpoint.Enabled = true + component.Experimental.Checkpoint.CheckpointRef = &checkpointName + component.Experimental.Checkpoint.Identity = nil + component.Experimental.Checkpoint.Job = nil startupPolicy := checkpointInfo.StartupPolicy if startupPolicy == "" { startupPolicy = nvidiacomv1alpha1.CheckpointStartupPolicyImmediate } - dcd.Spec.Experimental.Checkpoint.StartupPolicy = nvidiacomv1beta1.CheckpointStartupPolicy(startupPolicy) + component.Experimental.Checkpoint.StartupPolicy = nvidiacomv1beta1.CheckpointStartupPolicy(startupPolicy) } if checkpointInfo.StartupPolicy == nvidiacomv1alpha1.CheckpointStartupPolicyWaitForCheckpoint && !checkpointInfo.Ready { - dcd.Spec.Replicas = ptr.To(int32(0)) + component.Replicas = ptr.To(int32(0)) return nil } if checkpointInfo.StartupPolicy != "" && @@ -213,25 +231,25 @@ func (r *componentWorkloadsReconciler) applyCheckpointStartupPolicy( return nil } - labels := dynamo.GetPodTemplateLabels(&dcd.Spec.DynamoComponentDeploymentSharedSpec) + labels := dynamo.GetPodTemplateLabels(component) if labels == nil { - if dcd.Spec.PodTemplate == nil { - dcd.Spec.PodTemplate = &corev1.PodTemplateSpec{} + if component.PodTemplate == nil { + component.PodTemplate = &corev1.PodTemplateSpec{} } - if dcd.Spec.PodTemplate.Labels == nil { - dcd.Spec.PodTemplate.Labels = map[string]string{} + if component.PodTemplate.Labels == nil { + component.PodTemplate.Labels = map[string]string{} } - labels = dcd.Spec.PodTemplate.Labels + labels = component.PodTemplate.Labels } - annotations := dynamo.GetPodTemplateAnnotations(&dcd.Spec.DynamoComponentDeploymentSharedSpec) + annotations := dynamo.GetPodTemplateAnnotations(component) if annotations == nil { - if dcd.Spec.PodTemplate == nil { - dcd.Spec.PodTemplate = &corev1.PodTemplateSpec{} + if component.PodTemplate == nil { + component.PodTemplate = &corev1.PodTemplateSpec{} } - if dcd.Spec.PodTemplate.Annotations == nil { - dcd.Spec.PodTemplate.Annotations = map[string]string{} + if component.PodTemplate.Annotations == nil { + component.PodTemplate.Annotations = map[string]string{} } - annotations = dcd.Spec.PodTemplate.Annotations + annotations = component.PodTemplate.Annotations } return checkpoint.ApplyRestoreCandidateMetadata(labels, annotations, checkpointInfo) } @@ -239,9 +257,17 @@ func (r *componentWorkloadsReconciler) applyCheckpointStartupPolicy( func (r *componentWorkloadsReconciler) preserveExistingBackendFramework( ctx context.Context, desired *nvidiacomv1beta1.DynamoComponentDeployment, +) error { + return preserveExistingBackendFramework(ctx, r.syncer, desired) +} + +func preserveExistingBackendFramework( + ctx context.Context, + reader client.Reader, + desired *nvidiacomv1beta1.DynamoComponentDeployment, ) error { existing := &nvidiacomv1beta1.DynamoComponentDeployment{} - err := r.syncer.Get( + err := reader.Get( ctx, types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace}, existing, diff --git a/deploy/operator/internal/controller/dgd_disaggregatedset_program.go b/deploy/operator/internal/controller/dgd_disaggregatedset_program.go new file mode 100644 index 000000000000..d5cb585173cc --- /dev/null +++ b/deploy/operator/internal/controller/dgd_disaggregatedset_program.go @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "fmt" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + corev1 "k8s.io/api/core/v1" + apiMeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// disaggregatedSetEligibleConditionType reports whether the DisaggregatedSet +// pathway was selected for a DGD and why a requested selection is unsupported. +const disaggregatedSetEligibleConditionType = "DisaggregatedSetEligible" + +type disaggregatedSetProgram struct { + sharedResources *dgdSharedResourcesReconciler + rollout *dgdWorkerRolloutReconciler + workloads *disaggregatedSetWorkloadsReconciler + scalingAdapters *dgdScalingAdaptersReconciler + gate features.Gate +} + +func (r *DynamoGraphDeploymentReconciler) newDisaggregatedSetProgram() *disaggregatedSetProgram { + rollout := newDGDWorkerRolloutReconciler(r.Client, r.Recorder) + return &disaggregatedSetProgram{ + sharedResources: newDGDSharedResourcesReconciler( + r.Client, + r.Recorder, + r.Config, + r.RuntimeConfig, + r.RestConfig, + r.DockerSecretRetriever, + r.SSHKeyManager, + r.RBACManager, + ), + rollout: rollout, + workloads: r.newDisaggregatedSetWorkloadsReconciler(rollout), + scalingAdapters: newDGDScalingAdaptersReconciler(r.Client, r.Recorder), + gate: r.RuntimeConfig.Gate, + } +} + +func (p *disaggregatedSetProgram) Reconcile( + ctx context.Context, + req workloadProgramRequest, +) (programResult workloadProgramResult, retErr error) { + programResult = newWorkloadProgramResult(req.DGD) + if p.gate == nil || !p.gate.Enabled(features.DisaggregatedSet) { + err := failWorkloadProgram( + reasonSelectedWorkloadProviderUnavailable, + fmt.Errorf("selected workload provider %q is unavailable because DisaggregatedSet prerequisites are unavailable", workloadProviderDisaggregatedSet), + ) + programResult.Fail(req.DGD.Generation, reasonSelectedWorkloadProviderUnavailable, err) + return programResult, reconcile.TerminalError(err) + } + defer func() { + if retErr == nil { + return + } + reason := reasonFailedToReconcileResources + if classified, ok := workloadProgramFailureReason(retErr); ok { + reason = classified + } + programResult.Fail(req.DGD.Generation, reason, retErr) + }() + if reason := disaggregatedSetEligibilityReason(req.DGD, p.gate); reason != "" { + changed := setDisaggregatedSetEligibilityCondition( + &programResult, + req.DGD.Generation, + metav1.ConditionFalse, + "UnsupportedIntent", + reason, + ) + if changed { + programResult.Eventf( + corev1.EventTypeWarning, + "DisaggregatedSetUnsupported", + "DisaggregatedSet request cannot be reconciled: %s", + reason, + ) + } + programResult.applyReconcileResult(req.DGD.Generation, ReconcileResult{ + State: nvidiacomv1beta1.DGDStateFailed, + Reason: "disaggregated_set_intent_unsupported", + Message: Message(reason), + }) + return programResult, nil + } + + setDisaggregatedSetEligibilityCondition(&programResult, req.DGD.Generation, metav1.ConditionTrue, "Selected", "DisaggregatedSet pathway selected") + + if err := p.reconcileWorkerRollout(ctx, req.DGD); err != nil { + return programResult, err + } + + checkpoints, err := p.sharedResources.Reconcile(ctx, req.DGD) + if checkpoints.Statuses != nil { + programResult.Status.Checkpoints = checkpoints.Statuses + } + if err != nil { + return programResult, err + } + + previousRestart := programResult.Status.Restart + restart := p.workloads.ResolveRestart(ctx, req.DGD, &programResult.Status) + recordRestartTransition(previousRestart, restart.Status, &programResult) + programResult.Status.Restart = restart.Status + + result, err := p.workloads.Reconcile(ctx, req.DGD, restart.State, checkpoints.Infos) + if err != nil { + return programResult, fmt.Errorf("failed to reconcile DisaggregatedSet pathway: %w", err) + } + result = applyCheckpointStartupReadiness(result, checkpoints.Infos) + if result.State != nvidiacomv1beta1.DGDStatePending || result.Reason != reasonWaitingForCheckpoint { + if err := p.scalingAdapters.Reconcile(ctx, req.DGD); err != nil { + log.FromContext(ctx).Error(err, "Failed to reconcile scaling adapters") + return programResult, fmt.Errorf("failed to reconcile scaling adapters: %w", err) + } + } + + programResult.applyReconcileResult(req.DGD.Generation, result) + return programResult, nil +} + +func setDisaggregatedSetEligibilityCondition( + result *workloadProgramResult, + generation int64, + status metav1.ConditionStatus, + reason string, + message string, +) bool { + previous := apiMeta.FindStatusCondition(result.Status.Conditions, disaggregatedSetEligibleConditionType) + changed := previous == nil || + previous.Status != status || + previous.ObservedGeneration != generation || + previous.Reason != reason || + previous.Message != message + apiMeta.SetStatusCondition(&result.Status.Conditions, metav1.Condition{ + Type: disaggregatedSetEligibleConditionType, + Status: status, + ObservedGeneration: generation, + Reason: reason, + Message: message, + }) + return changed +} + +func (p *disaggregatedSetProgram) reconcileWorkerRollout( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) error { + if err := p.rollout.migrateCurrentWorkerHashIfNeeded(ctx, dgd); err != nil { + log.FromContext(ctx).Error(err, "Failed to migrate worker hash") + return failWorkloadProgram(reasonFailedToMigrateWorkerHash, err) + } + return p.rollout.ReconcileUnsupported(ctx, dgd, false) +} diff --git a/deploy/operator/internal/controller/dgd_disaggregatedset_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_disaggregatedset_workloads_reconciler.go new file mode 100644 index 000000000000..091c2afbab03 --- /dev/null +++ b/deploy/operator/internal/controller/dgd_disaggregatedset_workloads_reconciler.go @@ -0,0 +1,267 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "sort" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/checkpoint" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type disaggregatedSetWorkloadsReconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commoncontroller.RuntimeConfig + Recorder events.EventRecorder + DockerSecretRetriever DockerSecretRetriever + rollout *dgdWorkerRolloutReconciler + renderer *dcdWorkloadRenderer + componentRestartProgress *componentRestartProgressResolver +} + +func (r *DynamoGraphDeploymentReconciler) newDisaggregatedSetWorkloadsReconciler( + rollout *dgdWorkerRolloutReconciler, +) *disaggregatedSetWorkloadsReconciler { + workloads := newDisaggregatedSetWorkloadsReconciler( + r.Client, + r.Recorder, + r.Config, + r.RuntimeConfig, + r.DockerSecretRetriever, + rollout, + ) + return workloads +} +func newDisaggregatedSetWorkloadsReconciler( + k8sClient client.Client, + recorder events.EventRecorder, + config *configv1alpha1.OperatorConfiguration, + runtimeConfig *commoncontroller.RuntimeConfig, + dockerSecretRetriever DockerSecretRetriever, + rollout *dgdWorkerRolloutReconciler, +) *disaggregatedSetWorkloadsReconciler { + return &disaggregatedSetWorkloadsReconciler{ + Client: k8sClient, + Config: config, + RuntimeConfig: runtimeConfig, + Recorder: recorder, + DockerSecretRetriever: dockerSecretRetriever, + rollout: rollout, + renderer: newDCDWorkloadRenderer(k8sClient, config, runtimeConfig, dockerSecretRetriever), + componentRestartProgress: newComponentRestartProgressResolver(k8sClient), + } +} + +func (r *disaggregatedSetWorkloadsReconciler) GetRecorder() events.EventRecorder { + return r.Recorder +} + +func (r *disaggregatedSetWorkloadsReconciler) ResolveRestart( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + status *nvidiacomv1beta1.DynamoGraphDeploymentStatus, +) programRestart { + statusView := dgd.DeepCopy() + statusView.Status = *status + restartStatus := r.computeRestartStatus(ctx, statusView) + return programRestart{ + State: coalesceDisaggregatedSetRestartState(statusView, dynamo.DetermineRestartState(statusView, restartStatus)), + Status: restartStatus, + } +} + +func (r *disaggregatedSetWorkloadsReconciler) Reconcile( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + restartState *dynamo.RestartState, + checkpointInfos map[string]*checkpoint.CheckpointInfo, +) (ReconcileResult, error) { + return r.reconcileDisaggregatedSetResources(ctx, dgd, restartState, checkpointInfos) +} + +func (r *disaggregatedSetWorkloadsReconciler) computeRestartStatus( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) *nvidiacomv1beta1.RestartStatus { + if dgd.Spec.Restart == nil || dgd.Spec.Restart.ID == "" { + if dgd.Status.Restart != nil && + (dgd.Status.Restart.Phase == nvidiacomv1beta1.RestartPhaseCompleted || + dgd.Status.Restart.Phase == nvidiacomv1beta1.RestartPhaseFailed || + dgd.Status.Restart.Phase == nvidiacomv1beta1.RestartPhaseSuperseded) { + return dgd.Status.Restart + } + return nil + } + if isRestartAlreadyProcessed(dgd) { + return dgd.Status.Restart + } + if rollingUpdateInProgress(dgd.Status.RollingUpdate) { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: dgd.Spec.Restart.ID, + Phase: nvidiacomv1beta1.RestartPhaseSuperseded, + } + } + if dynamo.IsParallelRestart(dgd) { + return r.computeParallelRestartStatus(ctx, dgd) + } + return r.computeSequentialRestartStatus(ctx, dgd, dynamo.GetRestartOrder(dgd)) +} + +func (r *disaggregatedSetWorkloadsReconciler) computeParallelRestartStatus( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) *nvidiacomv1beta1.RestartStatus { + specID := dgd.Spec.Restart.ID + var componentsToCheck []string + if isNewRestartRequest(dgd) { + componentsToCheck = make([]string, 0, len(dgd.Spec.Components)) + for i := range dgd.Spec.Components { + componentsToCheck = append(componentsToCheck, dgd.Spec.Components[i].ComponentName) + } + sort.Strings(componentsToCheck) + if len(componentsToCheck) > 0 { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: componentsToCheck, + } + } + } else if dgd.Status.Restart != nil && len(dgd.Status.Restart.InProgress) > 0 { + componentsToCheck = dgd.Status.Restart.InProgress + } else { + componentsToCheck = make([]string, 0, len(dgd.Spec.Components)) + for i := range dgd.Spec.Components { + componentsToCheck = append(componentsToCheck, dgd.Spec.Components[i].ComponentName) + } + sort.Strings(componentsToCheck) + } + + if len(componentsToCheck) == 0 { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseCompleted, + } + } + + updatedInProgress := r.getUpdatedInProgressForDisaggregatedSet(ctx, dgd, componentsToCheck) + if len(updatedInProgress) == 0 { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseCompleted, + } + } + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: updatedInProgress, + } +} + +func (r *disaggregatedSetWorkloadsReconciler) computeSequentialRestartStatus( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + order []string, +) *nvidiacomv1beta1.RestartStatus { + specID := dgd.Spec.Restart.ID + if len(order) == 0 { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseCompleted, + } + } + if isNewRestartRequest(dgd) { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: []string{order[0]}, + } + } + + currentComponent := "" + if dgd.Status.Restart != nil && len(dgd.Status.Restart.InProgress) > 0 { + currentComponent = dgd.Status.Restart.InProgress[0] + } + if currentComponent == "" { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: []string{order[0]}, + } + } + + updatedInProgress := r.getUpdatedInProgressForDisaggregatedSet(ctx, dgd, []string{currentComponent}) + if len(updatedInProgress) > 0 { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: []string{currentComponent}, + } + } + + nextComponent, currentFound := r.getNextSequentialRestartComponent(dgd, order, currentComponent) + if !currentFound { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: []string{order[0]}, + } + } + if nextComponent == "" { + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseCompleted, + } + } + return &nvidiacomv1beta1.RestartStatus{ + ObservedID: specID, + Phase: nvidiacomv1beta1.RestartPhaseRestarting, + InProgress: []string{nextComponent}, + } +} + +func (r *disaggregatedSetWorkloadsReconciler) getNextSequentialRestartComponent( + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + order []string, + currentComponent string, +) (string, bool) { + nextComponent, currentFound := getNextComponentInOrder(order, currentComponent) + if !currentFound || nextComponent == "" { + return nextComponent, currentFound + } + selection, reason := selectDisaggregatedSetComponents(dgd) + if reason != "" { + return nextComponent, currentFound + } + if _, selected := selection.componentToRole[currentComponent]; !selected { + return nextComponent, currentFound + } + for nextComponent != "" { + if _, selected := selection.componentToRole[nextComponent]; !selected { + return nextComponent, true + } + nextComponent, _ = getNextComponentInOrder(order, nextComponent) + } + return "", true +} diff --git a/deploy/operator/internal/controller/dgd_grove_program.go b/deploy/operator/internal/controller/dgd_grove_program.go index 05e454570ebf..1698c7a6513e 100644 --- a/deploy/operator/internal/controller/dgd_grove_program.go +++ b/deploy/operator/internal/controller/dgd_grove_program.go @@ -141,7 +141,6 @@ func (p *groveProgram) Reconcile( return programResult, fmt.Errorf("failed to reconcile scaling adapters: %w", err) } } - programResult.applyReconcileResult(req.DGD.Generation, result) return programResult, nil } diff --git a/deploy/operator/internal/controller/dgd_grove_stable_resources_reconciler.go b/deploy/operator/internal/controller/dgd_grove_stable_resources_reconciler.go index 146a610d4bec..df60cbb63132 100644 --- a/deploy/operator/internal/controller/dgd_grove_stable_resources_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_stable_resources_reconciler.go @@ -151,7 +151,6 @@ func (r *groveStableResourcesReconciler) reconcileComponentService( logger.Error(err, "failed to generate the main component service") return nil, fmt.Errorf("failed to generate the main component service: %w", err) } - _, syncedService, err := commoncontroller.SyncResource( ctx, r, diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go index 028f1c393179..f8ea82ca8ad4 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler.go @@ -111,11 +111,6 @@ func (r *groveWorkloadsReconciler) Reconcile( return ReconcileResult{}, fmt.Errorf("failed to reconcile Grove scaling: %w", err) } - stableResources, err := r.stableResources.Reconcile(ctx, dgd, renderedPodCliqueSet.renderDeployment) - if err != nil { - return ReconcileResult{}, err - } - podCliqueSetResource, readiness, err := r.observePodCliqueSetReadiness( ctx, dgd, @@ -124,6 +119,10 @@ func (r *groveWorkloadsReconciler) Reconcile( if err != nil { return ReconcileResult{}, err } + stableResources, err := r.stableResources.Reconcile(ctx, dgd, renderedPodCliqueSet.renderDeployment) + if err != nil { + return ReconcileResult{}, err + } resources := append(stableResources, podCliqueSetResource) return checkGroveResourcesReadiness(resources, readiness.Classification), nil diff --git a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go index 6240acb799d6..8aeae2eb66df 100644 --- a/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go +++ b/deploy/operator/internal/controller/dgd_grove_workloads_reconciler_test.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" ) const updatedWorkerVersion = "new" @@ -551,3 +552,83 @@ func newUnstructuredGrovePodCliqueSet() *unstructured.Unstructured { object.SetKind(provideroverride.TargetPodCliqueSet) return object } + +func TestGroveServiceDoesNotPreserveDisaggregatedSetSelector(t *testing.T) { + dgd := betaDGD(t, &nvidiacomv1alpha1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "graph", Namespace: "default"}, + Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{ + BackendFramework: "vllm", + Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ + "frontend": {ComponentType: consts.ComponentTypeFrontend}, + }, + }, + }) + component := &dgd.Spec.Components[0] + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: dynamo.GetDCDResourceName(dgd, component.ComponentName, ""), + Namespace: dgd.Namespace, + }} + setDisaggregatedSetServiceSelector(service, "graph", "frontend", "old12345") + kubeClient := fake.NewClientBuilder(). + WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). + WithObjects(dgd, service). + Build() + reconciler := newGroveStableResourcesReconciler( + kubeClient, + events.NewFakeRecorder(10), + &configv1alpha1.OperatorConfiguration{}, + ) + + _, err := reconciler.reconcileComponentService(t.Context(), dgd, dgd, component, false) + require.NoError(t, err) + require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKeyFromObject(service), service)) + require.NotContains(t, service.Spec.Selector, disaggregatedsetv1.SetNameLabelKey) +} + +func TestGroveServiceRenderingCharacterization(t *testing.T) { + t.Log("Arrange a frontend component for the Grove stable-resource reconciler") + dgd := betaDGD(t, &nvidiacomv1alpha1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "graph", Namespace: "default", UID: "graph-uid"}, + Spec: nvidiacomv1alpha1.DynamoGraphDeploymentSpec{ + BackendFramework: "vllm", + Services: map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ + "frontend": { + ComponentType: consts.ComponentTypeFrontend, + DynamoNamespace: ptr.To("dynamo-graph"), + }, + }, + }, + }) + component := &dgd.Spec.Components[0] + kubeClient := fake.NewClientBuilder(). + WithScheme(newDynamoGraphDeploymentControllerTestScheme(t)). + WithObjects(dgd). + Build() + reconciler := newGroveStableResourcesReconciler( + kubeClient, events.NewFakeRecorder(10), &configv1alpha1.OperatorConfiguration{}, + ) + + t.Log("Act through the existing Grove component-service path") + _, err := reconciler.reconcileComponentService(t.Context(), dgd, dgd, component, false) + require.NoError(t, err) + + t.Log("Assert the stable Service name, selector, port, metadata, and DGD ownership") + service := &corev1.Service{} + require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKey{ + Namespace: dgd.Namespace, + Name: dynamo.GetDCDResourceName(dgd, component.ComponentName, ""), + }, service)) + dynamoNamespace := dgd.GetDynamoNamespaceForComponent(component) + require.Equal(t, map[string]string{ + consts.KubeLabelDynamoComponentType: consts.ComponentTypeFrontend, + consts.KubeLabelDynamoNamespace: dynamoNamespace, + consts.KubeLabelDynamoComponent: "frontend", + }, service.Spec.Selector) + require.Equal(t, consts.DynamoServicePortName, service.Spec.Ports[0].Name) + require.Equal(t, int32(consts.DynamoServicePort), service.Spec.Ports[0].Port) + require.Empty(t, service.Labels) + require.Equal(t, "1", service.Annotations[commoncontroller.NvidiaAnnotationGenerationKey]) + require.NotEmpty(t, service.Annotations[commoncontroller.NvidiaAnnotationHashKey]) + require.Len(t, service.OwnerReferences, 1) + require.Equal(t, dgd.UID, service.OwnerReferences[0].UID) +} diff --git a/deploy/operator/internal/controller/dgd_workload_program.go b/deploy/operator/internal/controller/dgd_workload_program.go index 0387b923301e..3812f1624bf4 100644 --- a/deploy/operator/internal/controller/dgd_workload_program.go +++ b/deploy/operator/internal/controller/dgd_workload_program.go @@ -79,6 +79,8 @@ func (r *DynamoGraphDeploymentReconciler) selectWorkloadProgram( switch provider { case workloadProviderGrove: return r.newGroveProgram(), nil + case workloadProviderDisaggregatedSet: + return r.newDisaggregatedSetProgram(), nil case workloadProviderComponent: return r.newComponentProgram(), nil default: diff --git a/deploy/operator/internal/controller/dgd_workload_program_test.go b/deploy/operator/internal/controller/dgd_workload_program_test.go index ac870533fdf9..7acf8af51cf3 100644 --- a/deploy/operator/internal/controller/dgd_workload_program_test.go +++ b/deploy/operator/internal/controller/dgd_workload_program_test.go @@ -59,6 +59,11 @@ func TestDGDWorkloadProgramSelection(t *testing.T) { provider: workloadProviderGrove, wantProgram: &groveProgram{}, }, + { + name: "DisaggregatedSet provider selects DisaggregatedSet program", + provider: workloadProviderDisaggregatedSet, + wantProgram: &disaggregatedSetProgram{}, + }, } for _, tt := range tests { @@ -90,6 +95,12 @@ func TestDGDWorkloadProgramSelection(t *testing.T) { assert.NotNil(t, grove.scalingAdapters) assert.NotNil(t, grove.topology) } + if disaggregatedSet, ok := got.(*disaggregatedSetProgram); ok { + assert.NotNil(t, disaggregatedSet.sharedResources) + assert.NotNil(t, disaggregatedSet.rollout) + assert.NotNil(t, disaggregatedSet.workloads) + assert.NotNil(t, disaggregatedSet.scalingAdapters) + } }) } } @@ -114,6 +125,26 @@ func TestSelectedGroveProgramDoesNotFallbackWhenUnavailable(t *testing.T) { assert.Contains(t, ready.Message, "Grove is disabled") } +func TestSelectedDisaggregatedSetProgramDoesNotFallbackWhenUnavailable(t *testing.T) { + t.Log("Create a DGD request and an unavailable DisaggregatedSet program") + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 3}, + } + program := &disaggregatedSetProgram{gate: features.Gates{LWS: true}} + + t.Log("Reconcile the durably selected DisaggregatedSet program while prerequisites are unavailable") + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: dgd}) + require.Error(t, err) + assert.ErrorIs(t, err, reconcile.TerminalError(nil)) + + t.Log("Verify DisaggregatedSet reports provider unavailability without falling back") + ready := meta.FindStatusCondition(result.Status.Conditions, "Ready") + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, string(reasonSelectedWorkloadProviderUnavailable), ready.Reason) + assert.Contains(t, ready.Message, "DisaggregatedSet prerequisites are unavailable") +} + func TestNewWorkloadProgramResultCopiesStatus(t *testing.T) { dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ Status: nvidiacomv1beta1.DynamoGraphDeploymentStatus{ @@ -136,6 +167,25 @@ func TestNewWorkloadProgramResultCopiesStatus(t *testing.T) { assert.Equal(t, nvidiacomv1beta1.RollingUpdatePhaseInProgress, dgd.Status.RollingUpdate.Phase) } +func TestDisaggregatedSetUnsupportedIntentIsSticky(t *testing.T) { + dgd := newEnvtestDSHappyPathDGD("unsupported-intent") + dgd.Generation = 7 + dgd.Spec.Components[0].ScalingAdapter = &nvidiacomv1beta1.ScalingAdapter{} + program := &disaggregatedSetProgram{gate: features.Gates{LWS: true, DisaggregatedSet: true}} + + result, err := program.Reconcile(t.Context(), workloadProgramRequest{DGD: dgd}) + require.NoError(t, err) + condition := meta.FindStatusCondition(result.Status.Conditions, disaggregatedSetEligibleConditionType) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionFalse, condition.Status) + assert.Equal(t, "UnsupportedIntent", condition.Reason) + assert.Equal(t, int64(7), condition.ObservedGeneration) + assert.Equal(t, nvidiacomv1beta1.DGDStateFailed, result.Status.State) + assert.Equal(t, int64(7), result.Status.ObservedGeneration) + assert.Contains(t, meta.FindStatusCondition(result.Status.Conditions, "Ready").Message, "scalingAdapter") + assert.Len(t, result.Events, 1) +} + func TestPersistWorkloadProgramResultEmitsEventsAfterStatusUpdate(t *testing.T) { statusUpdateErr := errors.New("status update failed") tests := []struct { diff --git a/deploy/operator/internal/controller/dgd_workload_provider.go b/deploy/operator/internal/controller/dgd_workload_provider.go index bd2a447c311c..e2a0edfa48fe 100644 --- a/deploy/operator/internal/controller/dgd_workload_provider.go +++ b/deploy/operator/internal/controller/dgd_workload_provider.go @@ -27,6 +27,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,8 +36,9 @@ import ( type workloadProvider string const ( - workloadProviderComponent workloadProvider = consts.WorkloadProviderComponent - workloadProviderGrove workloadProvider = consts.WorkloadProviderGrove + workloadProviderComponent workloadProvider = consts.WorkloadProviderComponent + workloadProviderGrove workloadProvider = consts.WorkloadProviderGrove + workloadProviderDisaggregatedSet workloadProvider = consts.WorkloadProviderDisaggregatedSet ) var ( @@ -96,16 +98,23 @@ func providerFromOwnedWorkloads( if err != nil { return "", false, err } + hasDisaggregatedSet, err := hasOwnedDisaggregatedSet(ctx, reader, dgd) + if err != nil { + return "", false, err + } - // Adopt one unambiguous family and fail closed when both families exist. + // DisaggregatedSet and Grove are the only ambiguous combination. DS takes + // precedence over legacy DCDs; Grove takes precedence over DCDs. switch { - case hasComponents && hasGrove: + case hasGrove && hasDisaggregatedSet: return "", false, fmt.Errorf( - "%w: DynamoGraphDeployment %s/%s owns DynamoComponentDeployments and PodCliqueSets", + "%w: DynamoGraphDeployment %s/%s owns workloads from multiple providers", errConflictingWorkloadProviders, dgd.Namespace, dgd.Name, ) + case hasDisaggregatedSet: + return workloadProviderDisaggregatedSet, true, nil case hasGrove: return workloadProviderGrove, true, nil case hasComponents: @@ -115,6 +124,24 @@ func providerFromOwnedWorkloads( } } +func hasOwnedDisaggregatedSet( + ctx context.Context, + reader client.Reader, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) (bool, error) { + // A DGD owns one deterministic DisaggregatedSet. Treat an unavailable + // optional API as an empty observable workload family during adoption. + ds := newDisaggregatedSetObject() + key := client.ObjectKey{Name: disaggregatedSetName(dgd), Namespace: dgd.Namespace} + if err := reader.Get(ctx, key, ds); err != nil { + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + return false, nil + } + return false, fmt.Errorf("get owned DisaggregatedSet: %w", err) + } + return metav1.IsControlledBy(ds, dgd), nil +} + func hasOwnedComponentWorkloads( ctx context.Context, reader client.Reader, @@ -167,13 +194,16 @@ func providerFromCurrentIntent( strings.ToLower(dgd.Annotations[consts.KubeAnnotationEnableGrove]) != consts.KubeLabelValueFalse { return workloadProviderGrove } + if strings.ToLower(dgd.Annotations[consts.KubeAnnotationEnableDisaggregatedSet]) == consts.KubeLabelValueTrue { + return workloadProviderDisaggregatedSet + } return workloadProviderComponent } func parseWorkloadProvider(value string) (workloadProvider, error) { // Accept only the workload programs implemented by this controller. switch workloadProvider(value) { - case workloadProviderComponent, workloadProviderGrove: + case workloadProviderComponent, workloadProviderGrove, workloadProviderDisaggregatedSet: return workloadProvider(value), nil default: return "", fmt.Errorf("%w: %q", errUnsupportedWorkloadProvider, value) diff --git a/deploy/operator/internal/controller/dgd_workload_provider_test.go b/deploy/operator/internal/controller/dgd_workload_provider_test.go index 270419af990a..e0c8de3079c3 100644 --- a/deploy/operator/internal/controller/dgd_workload_provider_test.go +++ b/deploy/operator/internal/controller/dgd_workload_provider_test.go @@ -62,6 +62,15 @@ func TestEnsureWorkloadProvider(t *testing.T) { workloads: []providerTestWorkload{{provider: workloadProviderGrove, owned: true}}, wantProvider: workloadProviderComponent, }, + { + name: "materialized DisaggregatedSet remains authoritative when Grove is enabled", + gate: features.Gates{Grove: true}, + annotations: map[string]string{ + consts.KubeAnnotationWorkloadProvider: consts.WorkloadProviderDisaggregatedSet, + }, + workloads: []providerTestWorkload{{provider: workloadProviderGrove, owned: true}}, + wantProvider: workloadProviderDisaggregatedSet, + }, { name: "no owned workloads select Grove from current intent", gate: features.Gates{Grove: true}, @@ -75,6 +84,22 @@ func TestEnsureWorkloadProvider(t *testing.T) { }, wantProvider: workloadProviderComponent, }, + { + name: "explicit DisaggregatedSet intent selects DisaggregatedSet after Grove opt-out", + gate: features.Gates{Grove: true}, + annotations: map[string]string{ + consts.KubeAnnotationEnableGrove: consts.KubeLabelValueFalse, + consts.KubeAnnotationEnableDisaggregatedSet: consts.KubeLabelValueTrue, + }, + wantProvider: workloadProviderDisaggregatedSet, + }, + { + name: "explicit DisaggregatedSet intent is durable even when its gate is unavailable", + annotations: map[string]string{ + consts.KubeAnnotationEnableDisaggregatedSet: consts.KubeLabelValueTrue, + }, + wantProvider: workloadProviderDisaggregatedSet, + }, { name: "owned DCD adopts component despite current Grove intent", gate: features.Gates{Grove: true}, @@ -90,20 +115,50 @@ func TestEnsureWorkloadProvider(t *testing.T) { wantProvider: workloadProviderGrove, }, { - name: "mixed owned workload families fail closed", + name: "owned DisaggregatedSet adopts DS despite current component intent", + annotations: map[string]string{ + consts.KubeAnnotationEnableGrove: consts.KubeLabelValueFalse, + }, + workloads: []providerTestWorkload{{provider: workloadProviderDisaggregatedSet, owned: true}}, + wantProvider: workloadProviderDisaggregatedSet, + }, + { + name: "DisaggregatedSet adopts alongside auxiliary DCDs", + workloads: []providerTestWorkload{ + {provider: workloadProviderComponent, owned: true}, + {provider: workloadProviderDisaggregatedSet, owned: true}, + }, + wantProvider: workloadProviderDisaggregatedSet, + }, + { + name: "Grove adopts alongside auxiliary DCDs", workloads: []providerTestWorkload{ {provider: workloadProviderComponent, owned: true}, {provider: workloadProviderGrove, owned: true}, }, - wantErr: "owns DynamoComponentDeployments and PodCliqueSets", + wantProvider: workloadProviderGrove, + }, + { + name: "DisaggregatedSet and Grove fail closed", + workloads: []providerTestWorkload{ + {provider: workloadProviderGrove, owned: true}, + {provider: workloadProviderDisaggregatedSet, owned: true}, + }, + wantErr: "owns workloads from multiple providers", wantErrIs: errConflictingWorkloadProviders, }, + { + name: "unannotated DGD with only DCDs adopts component", + workloads: []providerTestWorkload{{provider: workloadProviderComponent, owned: true}}, + wantProvider: workloadProviderComponent, + }, { name: "foreign workloads are ignored", gate: features.Gates{Grove: true}, workloads: []providerTestWorkload{ {provider: workloadProviderComponent}, {provider: workloadProviderGrove}, + {provider: workloadProviderDisaggregatedSet}, }, wantProvider: workloadProviderGrove, }, @@ -228,6 +283,12 @@ func newProviderTestWorkload( return &nvidiacomv1beta1.DynamoComponentDeployment{ObjectMeta: objectMeta} case workloadProviderGrove: return &grovev1alpha1.PodCliqueSet{ObjectMeta: objectMeta} + case workloadProviderDisaggregatedSet: + ds := newDisaggregatedSetObject() + ds.SetName(disaggregatedSetName(dgd)) + ds.SetNamespace(dgd.Namespace) + ds.SetOwnerReferences(ownerReferences) + return ds default: t.Fatalf("unsupported test workload provider %q", workload.provider) return nil diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go index df19806572a5..55a0460329ec 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go @@ -228,28 +228,30 @@ func (r *DynamoComponentDeploymentReconciler) Reconcile(ctx context.Context, req } modified := componentReconcileResult.modified - // create or update api-server service - serviceModified, err := r.createOrUpdateOrDeleteServices(ctx, generateResourceOption{ - dynamoComponentDeployment: dynamoComponentDeployment, - }) + serviceModified := false + disaggregatedSetBacked, err := r.isDisaggregatedSetBacked(ctx, dynamoComponentDeployment) if err != nil { - return ctrl.Result{}, fmt.Errorf("failed to create or update the service: %w", err) - } - - // create or update headless service for model endpoint discovery - componentName := dynamo.GetDCDComponentName(dynamoComponentDeployment) - componentMap := map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ - componentName: &dynamoComponentDeployment.Spec.DynamoComponentDeploymentSharedSpec, - } - if err := dynamo.ReconcileModelServicesForComponents( - ctx, - r, - dynamoComponentDeployment, - componentMap, - dynamoComponentDeployment.Namespace, - ); err != nil { - logs.Error(err, "Failed to reconcile model service") - return ctrl.Result{}, err + return ctrl.Result{}, fmt.Errorf("failed to determine workload provider: %w", err) + } + if !disaggregatedSetBacked { + // Graph-level Services are rendered and owned by the DGD when the DS + // provider is selected. Standalone/component DCDs retain this path. + serviceModified, err = r.createOrUpdateOrDeleteServices(ctx, generateResourceOption{ + dynamoComponentDeployment: dynamoComponentDeployment, + serviceTargetReady: componentReconcileResult.status == metav1.ConditionTrue, + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create or update the service: %w", err) + } + + componentName := dynamo.GetDCDComponentName(dynamoComponentDeployment) + componentMap := map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + componentName: &dynamoComponentDeployment.Spec.DynamoComponentDeploymentSharedSpec, + } + if err := dynamo.ReconcileModelServicesForComponents(ctx, r, dynamoComponentDeployment, componentMap, dynamoComponentDeployment.Namespace); err != nil { + logs.Error(err, "Failed to reconcile model service") + return ctrl.Result{}, err + } } // create or update api-server ingresses @@ -279,6 +281,24 @@ func (r *DynamoComponentDeploymentReconciler) Reconcile(ctx context.Context, req return } +func (r *DynamoComponentDeploymentReconciler) isDisaggregatedSetBacked( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) (bool, error) { + owner := metav1.GetControllerOf(dcd) + if owner == nil || owner.Kind != dynamoGraphDeploymentKind || owner.APIVersion != nvidiacomv1beta1.GroupVersion.String() { + return false, nil + } + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{} + if err := r.Get(ctx, client.ObjectKey{Namespace: dcd.Namespace, Name: owner.Name}, dgd); err != nil { + if k8serrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return dgd.Annotations[commonconsts.KubeAnnotationWorkloadProvider] == commonconsts.WorkloadProviderDisaggregatedSet, nil +} + type ComponentReconcileResult struct { modified bool status metav1.ConditionStatus @@ -695,7 +715,11 @@ func getResourceAnnotations(dynamoComponentDeployment *nvidiacomv1beta1.DynamoCo func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteServices(ctx context.Context, opt generateResourceOption) (bool, error) { modified, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*corev1.Service, bool, error) { - return r.generateService(ctx, opt) + service, deleted, err := r.generateService(ctx, opt) + if err != nil || deleted || service == nil || opt.serviceTargetReady { + return service, deleted, err + } + return service, false, nil }) if err != nil { return false, err @@ -870,6 +894,7 @@ func getDeploymentRollingUpdateMaxSurgeAndMaxUnavailable(annotations map[string] type generateResourceOption struct { dynamoComponentDeployment *nvidiacomv1beta1.DynamoComponentDeployment + serviceTargetReady bool } func (r *DynamoComponentDeploymentReconciler) generateService(ctx context.Context, opt generateResourceOption) (*corev1.Service, bool, error) { @@ -910,7 +935,6 @@ func (r *DynamoComponentDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) UpdateFunc: func(de event.UpdateEvent) bool { return true }, GenericFunc: func(ge event.GenericEvent) bool { return true }, })). - Owns(&corev1.Service{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Owns(&networkingv1.Ingress{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). WithEventFilter(deploymentEventFilter(r.Config, r.RuntimeConfig)) diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go index 08743313e6c4..b6fb13cd5427 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go @@ -731,6 +731,50 @@ func TestDynamoComponentDeploymentReconciler_generateService_DottedDeleteStub(t require.Equal(t, testNormalizedDCDName, service.Name) } +func TestStandaloneDCDServiceRenderingCharacterization(t *testing.T) { + t.Log("Arrange a standalone frontend DCD and its service-rendering configuration") + dcd := &v1beta1.DynamoComponentDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "frontend", + Namespace: "default", + UID: "frontend-uid", + Labels: map[string]string{ + commonconsts.KubeLabelDynamoComponent: "frontend", + commonconsts.KubeLabelDynamoNamespace: "dynamo-graph", + }, + }, + Spec: v1beta1.DynamoComponentDeploymentSpec{ + DynamoComponentDeploymentSharedSpec: v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "frontend", + ComponentType: v1beta1.ComponentTypeFrontend, + }, + }, + } + r := &DynamoComponentDeploymentReconciler{ + Config: &configv1alpha1.OperatorConfiguration{}, + } + + t.Log("Act through the existing standalone DCD Service renderer") + service, toDelete, err := r.generateService(context.Background(), generateResourceOption{ + dynamoComponentDeployment: dcd, + }) + require.NoError(t, err) + require.False(t, toDelete) + + t.Log("Assert standalone DCD naming, selector, port, and propagated metadata") + require.Equal(t, "frontend", service.Name) + require.Equal(t, dcd.Namespace, service.Namespace) + require.Equal(t, map[string]string{ + commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeFrontend, + commonconsts.KubeLabelDynamoNamespace: "dynamo-graph", + commonconsts.KubeLabelDynamoComponent: "frontend", + }, service.Spec.Selector) + require.Equal(t, commonconsts.DynamoServicePortName, service.Spec.Ports[0].Name) + require.Equal(t, int32(commonconsts.DynamoServicePort), service.Spec.Ports[0].Port) + require.Equal(t, dynamo.GetDCDKubeLabels(dcd), service.Labels) + require.Equal(t, dynamo.GetDCDKubeAnnotations(dcd), service.Annotations) +} + func TestDynamoComponentDeploymentReconciler_LWSNameDoesNotCollideWithComponentService(t *testing.T) { t.Log("Build a multinode DCD and the dependencies shared by reconciliation and rendering") s := scheme.Scheme diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go b/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go index 6cf1173323e1..26effc13bf02 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go @@ -106,6 +106,126 @@ func (r *dcdWorkloadRenderer) renderMultinodePodTemplateSpecs( return leaderPodTemplateSpec, workerPodTemplateSpec, nil } +// renderMultinodePodTemplateSpecsForDGDComponent renders a selected DS role +// directly from the normalized DGD component instead of materializing a DCD. +func (r *dcdWorkloadRenderer) renderMultinodePodTemplateSpecsForDGDComponent( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + componentName string, + workloadName string, + dynamoNamespace string, + backendFramework dynamo.BackendFramework, +) (*corev1.PodTemplateSpec, *corev1.PodTemplateSpec, error) { + podLabels := dynamo.GetDGDComponentResourceLabels(dgd, componentName, component) + podAnnotations := dynamo.GetDGDComponentResourceAnnotations(dgd, componentName, component) + podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dgd.Name + podLabels[commonconsts.KubeLabelDynamoComponent] = componentName + podLabels[commonconsts.KubeLabelDynamoNamespace] = dynamoNamespace + dynamo.AddBaseModelLabel(podLabels, component.ModelRef) + dynamo.AddBaseModelAnnotation(podAnnotations, component.ModelRef) + componentType, err := r.getWorkloadComponentType( + ctx, + dgd.Namespace, + workloadName, + string(component.ComponentType), + podLabels, + ) + if err != nil { + return nil, nil, err + } + containerGPUs := dynamo.ContainerGPUCount(func() (int64, error) { + return dynamo.ResolveContainerGPUs(ctx, r.reader, dgd.Namespace, component) + }) + + leaderLabels := maps.Clone(podLabels) + leaderPodTemplateSpec, err := r.generateComponentRolePodTemplateSpec( + ctx, + component, + maps.Clone(podLabels), + maps.Clone(podAnnotations), + componentType, + workloadName, + dgd.Name, + dgd.Namespace, + componentName, + dynamoNamespace, + backendFramework, + dynamo.RoleLeader, + leaderLabels, + containerGPUs, + ) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to generate leader pod template") + } + + workerLabels := maps.Clone(podLabels) + workerPodTemplateSpec, err := r.generateComponentRolePodTemplateSpec( + ctx, + component, + maps.Clone(podLabels), + maps.Clone(podAnnotations), + componentType, + workloadName, + dgd.Name, + dgd.Namespace, + componentName, + dynamoNamespace, + backendFramework, + dynamo.RoleWorker, + workerLabels, + containerGPUs, + ) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to generate worker pod template") + } + + return leaderPodTemplateSpec, workerPodTemplateSpec, nil +} + +func (r *dcdWorkloadRenderer) generateComponentRolePodTemplateSpec( + ctx context.Context, + component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + podLabels map[string]string, + podAnnotations map[string]string, + componentType string, + workloadName string, + parentGraphDeploymentName string, + namespace string, + componentName string, + dynamoNamespace string, + backendFramework dynamo.BackendFramework, + role dynamo.Role, + labels map[string]string, + containerGPUs dynamo.ContainerGPUCount, +) (*corev1.PodTemplateSpec, error) { + podTemplate, err := r.generateComponentPodTemplateSpec( + ctx, + component, + podLabels, + podAnnotations, + componentType, + workloadName, + parentGraphDeploymentName, + namespace, + componentName, + dynamoNamespace, + backendFramework, + role, + containerGPUs, + ) + if err != nil { + return nil, err + } + maps.Copy(podTemplate.ObjectMeta.Labels, labels) + podTemplate.ObjectMeta.Labels[dcdWorkloadRoleLabel] = string(role) + delete(podTemplate.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector) + if err := checkMainContainer(&podTemplate.Spec); err != nil { + return nil, err + } + return podTemplate, nil +} + func (r *dcdWorkloadRenderer) containerGPUCount( ctx context.Context, dcd *nvidiacomv1beta1.DynamoComponentDeployment, @@ -179,7 +299,6 @@ func (r *dcdWorkloadRenderer) generatePodTemplateSpec( if podAnnotations[commonconsts.KubeAnnotationEnableMetrics] != commonconsts.KubeLabelValueFalse { podLabels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue } - if parentName := dcd.GetLabels()[commonconsts.KubeLabelDynamoGraphDeploymentName]; parentName != "" { podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = parentName } else if parentName := dcd.GetParentGraphDeploymentName(); parentName != "" { @@ -200,24 +319,14 @@ func (r *dcdWorkloadRenderer) generatePodTemplateSpec( var checkpointInfo *checkpoint.CheckpointInfo if checkpointConfig := dynamo.GetCheckpoint(component); r.runtimeConfig.Gate.Enabled(features.Checkpoint) && checkpointConfig != nil { - info, err := checkpoint.ResolveCheckpointForService( - ctx, - r.reader, - dcd.Namespace, - dynamo.ToAlphaCheckpointConfig(checkpointConfig), - ) + info, err := checkpoint.ResolveCheckpointForService(ctx, r.reader, dcd.Namespace, dynamo.ToAlphaCheckpointConfig(checkpointConfig)) if err != nil { return nil, errors.Wrap(err, "failed to resolve checkpoint") } if dynamo.IsIntraPodFailoverEnabled(&dcd.Spec.DynamoComponentDeploymentSharedSpec) { info.RestoreTargetContainers = dynamo.IntraPodFailoverEngineContainerNames() } - if err := gms.OverlayClients( - &info.GPUMemoryService, - info.CheckpointName, - info.Exists, - dynamo.GetGPUMemoryService(component), - ); err != nil { + if err := gms.OverlayClients(&info.GPUMemoryService, info.CheckpointName, info.Exists, dynamo.GetGPUMemoryService(component)); err != nil { return nil, errors.Wrap(err, "failed to apply checkpoint gpuMemoryService config") } checkpointInfo = info @@ -231,63 +340,143 @@ func (r *dcdWorkloadRenderer) generatePodTemplateSpec( commonconsts.MultinodeDeploymentTypeLWS, checkpointInfo, containerGPUs, - dynamo.GenerateBasePodSpecForControllerOptions{ - WorkloadComponentType: nvidiacomv1beta1.ComponentType(componentType), - }, + dynamo.GenerateBasePodSpecForControllerOptions{WorkloadComponentType: nvidiacomv1beta1.ComponentType(componentType)}, ) if err != nil { return nil, errors.Wrap(err, "failed to generate base pod spec") } - if r.runtimeConfig.Gate.Enabled(features.Checkpoint) { - if checkpointInfo == nil || - string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyWaitForCheckpoint) { - if err := checkpoint.InjectCheckpointIntoPodSpecWithStorageConfig( - ctx, - r.reader, - dcd.Namespace, - podSpec, - checkpointInfo, - r.config.Checkpoint.Storage, - r.config.Checkpoint.EffectiveSeccompProfile(), - ); err != nil { - return nil, errors.Wrap(err, "failed to inject checkpoint config") - } + if r.runtimeConfig.Gate.Enabled(features.Checkpoint) && (checkpointInfo == nil || string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyWaitForCheckpoint)) { + if err := checkpoint.InjectCheckpointIntoPodSpecWithStorageConfig(ctx, r.reader, dcd.Namespace, podSpec, checkpointInfo, r.config.Checkpoint.Storage, r.config.Checkpoint.EffectiveSeccompProfile()); err != nil { + return nil, errors.Wrap(err, "failed to inject checkpoint config") } } - if len(podSpec.Containers) == 0 { return nil, errors.New("no containers found in base pod spec") } - podLabels[commonconsts.KubeLabelDynamoSelector] = kubeName - if commonController.IsK8sDiscoveryEnabled(r.config.Discovery.Backend, podAnnotations) { podLabels[commonconsts.KubeLabelDynamoDiscoveryBackend] = "kubernetes" podLabels[commonconsts.KubeLabelDynamoDiscoveryEnabled] = commonconsts.KubeLabelValueTrue } - - if checkpointInfo != nil && - (checkpointInfo.StartupPolicy == "" || - string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyImmediate)) { + if checkpointInfo != nil && (checkpointInfo.StartupPolicy == "" || string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyImmediate)) { if err := checkpoint.ApplyRestoreCandidateMetadata(podLabels, podAnnotations, checkpointInfo); err != nil { return nil, errors.Wrap(err, "failed to apply checkpoint candidate metadata") } - } else if err := checkpoint.ApplyRestorePodMetadataWithStorageConfig( - podLabels, - podAnnotations, + } else if err := checkpoint.ApplyRestorePodMetadataWithStorageConfig(podLabels, podAnnotations, checkpointInfo, r.config.Checkpoint.Storage); err != nil { + return nil, errors.Wrap(err, "failed to apply checkpoint metadata") + } + if podSpec.ServiceAccountName == "" { + serviceAccounts := &corev1.ServiceAccountList{} + if err := r.reader.List(ctx, serviceAccounts, client.InNamespace(dcd.Namespace), client.MatchingLabels{commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue}); err != nil { + return nil, errors.Wrapf(err, "failed to list service accounts in namespace %s", dcd.Namespace) + } + if len(serviceAccounts.Items) > 0 { + podSpec.ServiceAccountName = serviceAccounts.Items[0].Name + } else { + podSpec.ServiceAccountName = DefaultServiceAccountName + } + } + return &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: podLabels, Annotations: podAnnotations}, Spec: *podSpec}, nil +} + +func (r *dcdWorkloadRenderer) generateComponentPodTemplateSpec( + ctx context.Context, + component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + podLabels map[string]string, + podAnnotations map[string]string, + componentType string, + workloadName string, + parentGraphDeploymentName string, + namespace string, + componentName string, + dynamoNamespace string, + backendFramework dynamo.BackendFramework, + role dynamo.Role, + containerGPUs dynamo.ContainerGPUCount, +) (*corev1.PodTemplateSpec, error) { + component = component.DeepCopy() + if componentType != "" { + component.ComponentType = nvidiacomv1beta1.ComponentType(componentType) + } + if dynamoNamespace == "" { + return nil, fmt.Errorf("expected workload %s to have a dynamoNamespace", workloadName) + } + + if podAnnotations[commonconsts.KubeAnnotationEnableMetrics] != commonconsts.KubeLabelValueFalse { + podLabels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue + } + if parentGraphDeploymentName != "" { + podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = parentGraphDeploymentName + } + if componentType != "" { + podLabels[commonconsts.KubeLabelDynamoComponentType] = componentType + } + if componentName != "" { + podLabels[commonconsts.KubeLabelDynamoComponent] = componentName + } + if dynamoNamespace != "" { + podLabels[commonconsts.KubeLabelDynamoNamespace] = dynamoNamespace + } + + var checkpointInfo *checkpoint.CheckpointInfo + if checkpointConfig := dynamo.GetCheckpoint(component); r.runtimeConfig.Gate.Enabled(features.Checkpoint) && checkpointConfig != nil { + info, err := checkpoint.ResolveCheckpointForService(ctx, r.reader, namespace, dynamo.ToAlphaCheckpointConfig(checkpointConfig)) + if err != nil { + return nil, errors.Wrap(err, "failed to resolve checkpoint") + } + if dynamo.IsIntraPodFailoverEnabled(component) { + info.RestoreTargetContainers = dynamo.IntraPodFailoverEngineContainerNames() + } + if err := gms.OverlayClients(&info.GPUMemoryService, info.CheckpointName, info.Exists, dynamo.GetGPUMemoryService(component)); err != nil { + return nil, errors.Wrap(err, "failed to apply checkpoint gpuMemoryService config") + } + checkpointInfo = info + } + + podSpec, err := dynamo.GenerateBasePodSpec( + component, + backendFramework, + r.dockerSecretRetriever, + parentGraphDeploymentName, + namespace, + role, + component.GetNumberOfNodes(), + r.config, + commonconsts.MultinodeDeploymentTypeLWS, + componentName, checkpointInfo, - r.config.Checkpoint.Storage, - ); err != nil { + nil, + containerGPUs, + ) + if err != nil { + return nil, errors.Wrap(err, "failed to generate base pod spec") + } + if r.runtimeConfig.Gate.Enabled(features.Checkpoint) && (checkpointInfo == nil || string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyWaitForCheckpoint)) { + if err := checkpoint.InjectCheckpointIntoPodSpecWithStorageConfig(ctx, r.reader, namespace, podSpec, checkpointInfo, r.config.Checkpoint.Storage, r.config.Checkpoint.EffectiveSeccompProfile()); err != nil { + return nil, errors.Wrap(err, "failed to inject checkpoint config") + } + } + if len(podSpec.Containers) == 0 { + return nil, errors.New("no containers found in base pod spec") + } + + podLabels[commonconsts.KubeLabelDynamoSelector] = workloadName + if commonController.IsK8sDiscoveryEnabled(r.config.Discovery.Backend, podAnnotations) { + podLabels[commonconsts.KubeLabelDynamoDiscoveryBackend] = "kubernetes" + podLabels[commonconsts.KubeLabelDynamoDiscoveryEnabled] = commonconsts.KubeLabelValueTrue + } + if checkpointInfo != nil && (checkpointInfo.StartupPolicy == "" || string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyImmediate)) { + if err := checkpoint.ApplyRestoreCandidateMetadata(podLabels, podAnnotations, checkpointInfo); err != nil { + return nil, errors.Wrap(err, "failed to apply checkpoint candidate metadata") + } + } else if err := checkpoint.ApplyRestorePodMetadataWithStorageConfig(podLabels, podAnnotations, checkpointInfo, r.config.Checkpoint.Storage); err != nil { return nil, errors.Wrap(err, "failed to apply checkpoint metadata") } if podSpec.ServiceAccountName == "" { serviceAccounts := &corev1.ServiceAccountList{} - err = r.reader.List(ctx, serviceAccounts, client.InNamespace(dcd.Namespace), client.MatchingLabels{ - commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue, - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to list service accounts in namespace %s", dcd.Namespace) + if err := r.reader.List(ctx, serviceAccounts, client.InNamespace(namespace), client.MatchingLabels{commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue}); err != nil { + return nil, errors.Wrapf(err, "failed to list service accounts in namespace %s", namespace) } if len(serviceAccounts.Items) > 0 { podSpec.ServiceAccountName = serviceAccounts.Items[0].Name @@ -296,13 +485,7 @@ func (r *dcdWorkloadRenderer) generatePodTemplateSpec( } } - return &corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - Annotations: podAnnotations, - }, - Spec: *podSpec, - }, nil + return &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: podLabels, Annotations: podAnnotations}, Spec: *podSpec}, nil } func (r *dcdWorkloadRenderer) generateService( @@ -352,6 +535,52 @@ func (r *dcdWorkloadRenderer) generateService( return svc, false, nil } +func (r *dcdWorkloadRenderer) generateServiceForDGDComponent( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + componentName string, + serviceName string, +) (*corev1.Service, bool, error) { + annotations := dynamo.GetDGDComponentResourceAnnotations(dgd, componentName, component) + labels := dynamo.GetDGDComponentResourceLabels(dgd, componentName, component) + labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = dgd.Name + labels[commonconsts.KubeLabelDynamoComponent] = componentName + labels[commonconsts.KubeLabelDynamoNamespace] = dynamo.GetDynamoNamespace(dgd, component) + dynamo.AddBaseModelLabel(labels, component.ModelRef) + dynamo.AddBaseModelAnnotation(annotations, component.ModelRef) + isK8sDiscovery := commonController.IsK8sDiscoveryEnabled(r.config.Discovery.Backend, annotations) + componentType := string(component.ComponentType) + if !isK8sDiscovery && componentType != commonconsts.ComponentTypeFrontend { + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: dynamo.NormalizeKubeResourceName(serviceName), Namespace: dgd.Namespace}}, true, nil + } + dynamoNamespace := dynamo.GetDynamoNamespace(dgd, component) + if dynamoNamespace == "" { + return nil, false, fmt.Errorf("expected component %s to have a dynamoNamespace", componentName) + } + workloadComponentType, err := r.getWorkloadComponentType(ctx, dgd.Namespace, serviceName, componentType, labels) + if err != nil { + return nil, false, err + } + service, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ + ServiceName: serviceName, + Namespace: dgd.Namespace, + ComponentType: workloadComponentType, + DynamoNamespace: dynamoNamespace, + ComponentName: componentName, + Labels: labels, + Annotations: annotations, + IsK8sDiscovery: isK8sDiscovery, + }) + if err != nil { + return nil, false, err + } + if component.IsMultinode() { + service.Spec.Selector[dcdWorkloadRoleLabel] = string(dynamo.RoleLeader) + } + return service, false, nil +} + func (r *dcdWorkloadRenderer) getDCDWorkloadPodLabels( ctx context.Context, dcd *nvidiacomv1beta1.DynamoComponentDeployment, @@ -386,16 +615,31 @@ func (r *dcdWorkloadRenderer) getDCDWorkloadComponentType( return "", nil } - componentType := dynamo.GetDCDWorkloadComponentType(dcd) + return r.getWorkloadComponentType( + ctx, + dcd.Namespace, + dcd.Name, + dynamo.GetDCDWorkloadComponentType(dcd), + dcd.GetLabels(), + ) +} + +func (r *dcdWorkloadRenderer) getWorkloadComponentType( + ctx context.Context, + namespace string, + workloadName string, + componentType string, + labels map[string]string, +) (string, error) { if componentType == commonconsts.ComponentTypeWorker || !dynamo.IsWorkerComponent(componentType) { return componentType, nil } - if hasLegacyWorkerSelector(dcd.GetLabels(), componentType) { + if hasLegacyWorkerSelector(labels, componentType) { return commonconsts.ComponentTypeWorker, nil } - legacy, err := r.hasExistingLegacyWorkerSelector(ctx, dcd, componentType) + legacy, err := r.hasExistingLegacyWorkerSelector(ctx, namespace, workloadName, componentType) if err != nil { return "", err } @@ -408,26 +652,27 @@ func (r *dcdWorkloadRenderer) getDCDWorkloadComponentType( func (r *dcdWorkloadRenderer) hasExistingLegacyWorkerSelector( ctx context.Context, - dcd *nvidiacomv1beta1.DynamoComponentDeployment, + namespace string, + workloadName string, componentType string, ) (bool, error) { - if dcd == nil || r == nil || r.reader == nil { + if r == nil || r.reader == nil { return false, nil } deployment := &appsv1.Deployment{} - if err := r.reader.Get(ctx, types.NamespacedName{Name: dcd.Name, Namespace: dcd.Namespace}, deployment); err == nil { + if err := r.reader.Get(ctx, types.NamespacedName{Name: workloadName, Namespace: namespace}, deployment); err == nil { if hasLegacyWorkerSelector(deployment.Spec.Template.Labels, componentType) { return true, nil } } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get deployment %s/%s: %w", dcd.Namespace, dcd.Name, err) + return false, fmt.Errorf("failed to get deployment %s/%s: %w", namespace, workloadName, err) } if r.runtimeConfig.Gate.Enabled(features.LWS) { - lwsName := leaderWorkerSetName(dcd) + lwsName := fmt.Sprintf("%s-0", workloadName) leaderWorkerSet := &leaderworkersetv1.LeaderWorkerSet{} - if err := r.reader.Get(ctx, types.NamespacedName{Name: lwsName, Namespace: dcd.Namespace}, leaderWorkerSet); err == nil { + if err := r.reader.Get(ctx, types.NamespacedName{Name: lwsName, Namespace: namespace}, leaderWorkerSet); err == nil { template := leaderWorkerSet.Spec.LeaderWorkerTemplate if template.LeaderTemplate != nil && hasLegacyWorkerSelector(template.LeaderTemplate.Labels, componentType) { return true, nil @@ -436,16 +681,16 @@ func (r *dcdWorkloadRenderer) hasExistingLegacyWorkerSelector( return true, nil } } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get leaderworkerset %s/%s: %w", dcd.Namespace, lwsName, err) + return false, fmt.Errorf("failed to get leaderworkerset %s/%s: %w", namespace, lwsName, err) } } - serviceName := dynamo.NormalizeKubeResourceName(dcd.Name) + serviceName := dynamo.NormalizeKubeResourceName(workloadName) service := &corev1.Service{} - if err := r.reader.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: dcd.Namespace}, service); err == nil { + if err := r.reader.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: namespace}, service); err == nil { return hasLegacyWorkerSelector(service.Spec.Selector, componentType), nil } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get service %s/%s: %w", dcd.Namespace, serviceName, err) + return false, fmt.Errorf("failed to get service %s/%s: %w", namespace, serviceName, err) } return false, nil diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 72498c3e234d..799c789442de 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" @@ -38,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" @@ -90,6 +92,7 @@ type DynamoGraphDeploymentReconciler struct { // +kubebuilder:rbac:groups=grove.io,resources=podcliquescalinggroups,verbs=get;list;watch // +kubebuilder:rbac:groups=grove.io,resources=podcliquescalinggroups/scale,verbs=get;update;patch // +kubebuilder:rbac:groups=grove.io,resources=clustertopologybindings,verbs=get;list;watch +// +kubebuilder:rbac:groups=disaggregatedset.x-k8s.io,resources=disaggregatedsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=scheduling.run.ai,resources=queues,verbs=get;list // +kubebuilder:rbac:groups=inference.networking.k8s.io,resources=inferencepools,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.istio.io,resources=destinationrules,verbs=get;list;watch;create;update;patch;delete @@ -228,6 +231,19 @@ func (r *DynamoGraphDeploymentReconciler) FinalizeResource(ctx context.Context, ).deleteAutoCheckpointsForDGD(ctx, dynamoDeployment) } +func workloadRoutingAnnotationsChanged(update event.UpdateEvent) bool { + oldDGD, oldOK := update.ObjectOld.(*nvidiacomv1beta1.DynamoGraphDeployment) + newDGD, newOK := update.ObjectNew.(*nvidiacomv1beta1.DynamoGraphDeployment) + if !oldOK || !newOK { + return false + } + annotationValue := func(dgd *nvidiacomv1beta1.DynamoGraphDeployment, key string) string { + return strings.ToLower(dgd.GetAnnotations()[key]) + } + return annotationValue(oldDGD, consts.KubeAnnotationEnableDisaggregatedSet) != annotationValue(newDGD, consts.KubeAnnotationEnableDisaggregatedSet) || + annotationValue(oldDGD, consts.KubeAnnotationEnableGrove) != annotationValue(newDGD, consts.KubeAnnotationEnableGrove) +} + // SetupWithManager sets up the controller with the Manager. func (r *DynamoGraphDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField( @@ -241,7 +257,10 @@ func (r *DynamoGraphDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) err ctrlBuilder := ctrl.NewControllerManagedBy(mgr). For(&nvidiacomv1beta1.DynamoGraphDeployment{}, builder.WithPredicates( - generationOrDeletionChangedPredicate(), + predicate.Or( + generationOrDeletionChangedPredicate(), + predicate.Funcs{UpdateFunc: workloadRoutingAnnotationsChanged}, + ), )). Named(consts.ResourceTypeDynamoGraphDeployment). Watches( @@ -314,8 +333,24 @@ func (r *DynamoGraphDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) err GenericFunc: func(ge event.GenericEvent) bool { return false }, })) } - - // Register Grove-owned workload watches only when the Grove feature is enabled. + if r.RuntimeConfig.Gate.Enabled(features.DisaggregatedSet) { + disaggregatedSetWatches := newDisaggregatedSetWatchMapper(mgr.GetClient()) + ctrlBuilder = ctrlBuilder.Owns(newDisaggregatedSetObject(), builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(ce event.CreateEvent) bool { return false }, + DeleteFunc: func(de event.DeleteEvent) bool { return true }, + UpdateFunc: func(ue event.UpdateEvent) bool { return disaggregatedSetStatusChanged(ue.ObjectOld, ue.ObjectNew) }, + GenericFunc: func(ge event.GenericEvent) bool { return true }, + })).Watches( + &leaderworkersetv1.LeaderWorkerSet{}, + handler.EnqueueRequestsFromMapFunc(disaggregatedSetWatches.MapChildLWSToDGD), + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(ce event.CreateEvent) bool { return true }, + DeleteFunc: func(de event.DeleteEvent) bool { return true }, + UpdateFunc: func(ue event.UpdateEvent) bool { return leaderWorkerSetStatusChanged(ue.ObjectOld, ue.ObjectNew) }, + GenericFunc: func(ge event.GenericEvent) bool { return false }, + }), + ) + } if r.RuntimeConfig.Gate.Enabled(features.Grove) { ctrlBuilder = newGroveWatchSetup(r.Client).addTo(ctrlBuilder) } diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index fb3d01a4e14b..3f393e56569c 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -53,6 +53,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/event" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" ) func newDynamoGraphDeploymentControllerTestScheme(t testing.TB) *runtime.Scheme { @@ -66,6 +68,8 @@ func newDynamoGraphDeploymentControllerTestScheme(t testing.TB) *runtime.Scheme v1alpha1.AddToScheme, v1beta1.AddToScheme, grovev1alpha1.AddToScheme, + leaderworkersetv1.AddToScheme, + disaggregatedsetv1.AddToScheme, } { if err := addToScheme(s); err != nil { t.Fatalf("failed to add type to scheme: %v", err) diff --git a/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset.go b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset.go new file mode 100644 index 000000000000..7d8f9e151a92 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset.go @@ -0,0 +1,1475 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "maps" + "sort" + "strconv" + "strings" + + nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/checkpoint" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" + disaggregatedsetutils "sigs.k8s.io/lws/pkg/utils/disaggregatedset" +) + +var disaggregatedSetGVK = schema.GroupVersionKind{ + Group: "disaggregatedset.x-k8s.io", + Version: "v1", + Kind: "DisaggregatedSet", +} + +const ( + maxDisaggregatedSetRoles = 10 + disaggregatedSetRevisionLength = 8 + maxDisaggregatedSetNameLength = 31 + // The name budget assumes at most two slice-index digits (slices <= 99). + // The transitional pathway always renders one slice; when a future + // grouping API raises the slice cardinality, extend this budget first. + maxDisaggregatedSetSliceIndexLength = len("99") + disaggregatedSetServiceSuffixLength = len("-prv") + maxDisaggregatedSetRoleNameLength = 63 - maxDisaggregatedSetNameLength - maxDisaggregatedSetSliceIndexLength - disaggregatedSetRevisionLength - 3 - disaggregatedSetServiceSuffixLength + disaggregatedSetNameHashLength = 8 + dynamoGraphDeploymentKind = "DynamoGraphDeployment" + dynamoComponentDeploymentKind = "DynamoComponentDeployment" + resourceNotFoundReason = "resource not found" +) + +type disaggregatedSetSelection struct { + componentToRole map[string]string + desiredReplicas map[string]int32 +} + +type disaggregatedSetChildIdentity struct { + slice int + role string +} + +func newDisaggregatedSetObject() *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(disaggregatedSetGVK) + return obj +} + +func disaggregatedSetEligibilityReason( + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + gate features.Gate, +) string { + if dgd == nil { + return "DynamoGraphDeployment is nil" + } + selection, reason := selectDisaggregatedSetComponents(dgd) + if reason != "" { + return reason + } + if len(selection.componentToRole) < 2 { + return "DisaggregatedSet requires at least two eligible multinode worker roles" + } + if gate == nil || !gate.Enabled(features.LWS) { + for i := range dgd.Spec.Components { + component := &dgd.Spec.Components[i] + if component.GetNumberOfNodes() <= 1 { + continue + } + if _, selected := selection.componentToRole[component.ComponentName]; !selected { + return fmt.Sprintf( + "multinode component %q is not eligible for DisaggregatedSet and requires LeaderWorkerSet support", + component.ComponentName, + ) + } + } + } + return "" +} + +// coalesceDisaggregatedSetRestartState treats all selected DS roles as one +// restart unit. A DisaggregatedSet revision covers the complete role list, so +// annotating only one selected role would still roll every role. +func coalesceDisaggregatedSetRestartState( + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + restartState *dynamo.RestartState, +) *dynamo.RestartState { + if restartState == nil || dynamo.IsParallelRestart(dgd) { + return restartState + } + selection, reason := selectDisaggregatedSetComponents(dgd) + if reason != "" { + return restartState + } + selectedRestarting := false + for componentName := range selection.componentToRole { + if restartState.ShouldAnnotateComponent(componentName) { + selectedRestarting = true + break + } + } + if !selectedRestarting { + return restartState + } + for componentName := range selection.componentToRole { + restartState.ComponentsToAnnotate[componentName] = true + } + return restartState +} + +func selectDisaggregatedSetComponents(dgd *nvidiacomv1beta1.DynamoGraphDeployment) (disaggregatedSetSelection, string) { + selection := disaggregatedSetSelection{ + componentToRole: make(map[string]string), + desiredReplicas: make(map[string]int32), + } + if dgd == nil { + return selection, "DynamoGraphDeployment is nil" + } + + usedRoles := make(map[string]struct{}) + zeroReplicas := 0 + positiveReplicas := 0 + for i := range dgd.Spec.Components { + component := &dgd.Spec.Components[i] + if !isDisaggregatedSetEligibleComponent(component) { + continue + } + if component.ScalingAdapter != nil { + return selection, fmt.Sprintf( + "component %q uses scalingAdapter, but DisaggregatedSet does not support scale subresource integration", + component.ComponentName, + ) + } + roleName := disaggregatedSetRoleName(component, usedRoles) + usedRoles[roleName] = struct{}{} + selection.componentToRole[component.ComponentName] = roleName + + desiredReplicas := desiredComponentReplicas(component) + selection.desiredReplicas[component.ComponentName] = desiredReplicas + if desiredReplicas == 0 { + zeroReplicas++ + } else { + positiveReplicas++ + } + } + + if len(selection.componentToRole) == 0 { + return selection, "no eligible multinode worker roles found" + } + if len(selection.componentToRole) > maxDisaggregatedSetRoles { + return selection, fmt.Sprintf("DisaggregatedSet supports at most %d roles", maxDisaggregatedSetRoles) + } + if zeroReplicas > 0 && positiveReplicas > 0 { + return selection, "DisaggregatedSet requires replicas to be zero for all selected roles or positive for all selected roles" + } + return selection, "" +} + +func isDisaggregatedSetEligibleComponent(component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) bool { + return component != nil && component.GetNumberOfNodes() > 1 && dynamo.IsWorkerComponent(string(component.ComponentType)) +} + +func desiredComponentReplicas(component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) int32 { + if component == nil || component.Replicas == nil { + return 1 + } + return *component.Replicas +} + +func disaggregatedSetRoleName(component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, used map[string]struct{}) string { + preferred := strings.ToLower(string(component.ComponentType)) + if preferred != consts.ComponentTypePrefill && preferred != consts.ComponentTypeDecode { + preferred = "" + } + if preferred == "" || roleNameUsed(preferred, used) { + preferred = dynamo.NormalizeKubeResourceName(component.ComponentName) + } + preferred = truncateDNSLabelWithHash(preferred, maxDisaggregatedSetRoleNameLength) + roleName := preferred + for i := 2; roleNameUsed(roleName, used); i++ { + suffix := fmt.Sprintf("-%d", i) + roleName = truncateDNSLabel(preferred, maxDisaggregatedSetRoleNameLength-len(suffix)) + suffix + } + return roleName +} + +func roleNameUsed(roleName string, used map[string]struct{}) bool { + _, ok := used[roleName] + return ok +} + +func truncateDNSLabel(value string, maxLength int) string { + if len(value) <= maxLength { + return value + } + return strings.TrimRight(value[:maxLength], "-") +} + +func truncateDNSLabelWithHash(value string, maxLength int) string { + if len(value) <= maxLength { + return value + } + hash := sha256.Sum256([]byte(value)) + hashText := hex.EncodeToString(hash[:])[:disaggregatedSetNameHashLength] + if maxLength <= len(hashText) { + return hashText[:maxLength] + } + suffix := "-" + hashText + prefix := strings.TrimRight(value[:maxLength-len(suffix)], "-") + return prefix + suffix +} + +func disaggregatedSetName(dgd *nvidiacomv1beta1.DynamoGraphDeployment) string { + return truncateDNSLabelWithHash(dynamo.NormalizeKubeResourceName(dgd.Name), maxDisaggregatedSetNameLength) +} + +func (r *disaggregatedSetWorkloadsReconciler) reconcileDisaggregatedSetResources( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + restartState *dynamo.RestartState, + checkpointInfos map[string]*checkpoint.CheckpointInfo, +) (ReconcileResult, error) { + resources := []Resource{} + logger := log.FromContext(ctx) + + rollingUpdateCtx, err := r.rollout.buildRollingUpdateContext(ctx, dgd) + if err != nil { + return ReconcileResult{}, fmt.Errorf("failed to build rolling update context: %w", err) + } + selection, reason := selectDisaggregatedSetComponents(dgd) + if reason != "" { + return ReconcileResult{}, fmt.Errorf("failed to select DisaggregatedSet roles: %s", reason) + } + + existingRestartAnnotations, err := getExistingRestartAnnotationsDCD(ctx, r.Client, dgd) + if err != nil { + logger.Error(err, "failed to get existing restart annotations") + return ReconcileResult{}, fmt.Errorf("failed to get existing restart annotations: %w", err) + } + existingDSRestartAnnotations, err := r.getExistingRestartAnnotationsDisaggregatedSet(ctx, dgd, selection) + if err != nil { + logger.Error(err, "failed to get existing DisaggregatedSet restart annotations") + return ReconcileResult{}, fmt.Errorf("failed to get existing DisaggregatedSet restart annotations: %w", err) + } + maps.Copy(existingRestartAnnotations, existingDSRestartAnnotations) + + normalizedComponents, err := dynamo.NormalizeDynamoGraphDeploymentComponents( + dgd, + restartState, + existingRestartAnnotations, + rollingUpdateCtx, + ) + if err != nil { + return ReconcileResult{}, fmt.Errorf("failed to normalize components for DisaggregatedSet path: %w", err) + } + + checkpointGated, err := r.applyDisaggregatedSetCheckpointStartupPolicies(normalizedComponents, checkpointInfos, selection) + if err != nil { + return ReconcileResult{}, err + } + + desiredDS, err := r.generateDisaggregatedSetFromNormalized(ctx, dgd, normalizedComponents, selection, rollingUpdateCtx) + if err != nil { + return ReconcileResult{}, err + } + dsModified, syncedDS, err := r.syncDisaggregatedSet(ctx, dgd, desiredDS) + if err != nil { + return ReconcileResult{}, err + } + + targetReady, dsReason, dsStatuses, err := r.checkDisaggregatedSetReadiness(ctx, syncedDS, selection) + if err != nil { + return ReconcileResult{}, err + } + dsReady := targetReady && !dsModified && !checkpointGated + targetRevision, err := disaggregatedSetTargetRevision(syncedDS) + if err != nil { + return ReconcileResult{}, err + } + nonSelectedComponents := make(map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) + for componentName, component := range normalizedComponents { + if _, selected := selection.componentToRole[componentName]; !selected { + nonSelectedComponents[componentName] = component + } + } + dcds, err := dynamo.GenerateDynamoComponentsDeploymentsFromNormalized(dgd, nonSelectedComponents, rollingUpdateCtx) + if err != nil { + return ReconcileResult{}, fmt.Errorf("failed to lower non-DisaggregatedSet components: %w", err) + } + selectedServiceNames, err := r.reconcileDisaggregatedSetSideResources( + ctx, + dgd, + dcds, + normalizedComponents, + selection, + targetRevision, + dsReady, + rollingUpdateCtx, + ) + if err != nil { + return ReconcileResult{}, err + } + + syncedDSResource, err := commoncontroller.NewResourceWithComponentStatuses( + syncedDS, + func() (bool, string, map[string]nvidiacomv1beta1.ComponentReplicaStatus) { + if dsModified { + return false, "DisaggregatedSet spec was updated; waiting for controller status", dsStatuses + } + if checkpointGated { + return false, "DisaggregatedSet roles are waiting for checkpoint readiness", dsStatuses + } + return dsReady, dsReason, dsStatuses + }, + ) + if err != nil { + return ReconcileResult{}, err + } + resources = append(resources, syncedDSResource) + + nonSelectedResources, err := r.reconcileDisaggregatedSetNonSelectedDCDs(ctx, dgd, dcds, selection) + if err != nil { + return ReconcileResult{}, err + } + resources = append(resources, nonSelectedResources...) + desiredServiceNames := selectedServiceNames + + if dsReady { + if err := r.deleteOwnedSelectedDCDs(ctx, dgd, selection); err != nil { + return ReconcileResult{}, err + } + } + + result := checkResourcesReadiness(resources) + if result.State == nvidiacomv1beta1.DGDStateSuccessful { + if err := r.deleteStaleDisaggregatedSetServices(ctx, dgd, desiredServiceNames); err != nil { + return ReconcileResult{}, err + } + } + return result, nil +} + +func (r *disaggregatedSetWorkloadsReconciler) getExistingRestartAnnotationsDisaggregatedSet( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + selection disaggregatedSetSelection, +) (map[string]string, error) { + ds := newDisaggregatedSetObject() + key := types.NamespacedName{Name: disaggregatedSetName(dgd), Namespace: dgd.Namespace} + if err := r.Get(ctx, key, ds); err != nil { + if apierrors.IsNotFound(err) { + return map[string]string{}, nil + } + return nil, fmt.Errorf("failed to get DisaggregatedSet %s: %w", key, err) + } + return restartAnnotationsFromDisaggregatedSet(ds, selection) +} + +func restartAnnotationsFromDisaggregatedSet( + ds *unstructured.Unstructured, + selection disaggregatedSetSelection, +) (map[string]string, error) { + restartAnnotations := make(map[string]string) + if ds == nil { + return restartAnnotations, nil + } + spec, found, err := unstructured.NestedMap(ds.Object, "spec") + if err != nil { + return nil, fmt.Errorf("failed to read DisaggregatedSet spec: %w", err) + } + if !found { + return restartAnnotations, nil + } + typedSpec := disaggregatedsetv1.DisaggregatedSetSpec{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(spec, &typedSpec); err != nil { + return nil, fmt.Errorf("failed to decode DisaggregatedSet spec: %w", err) + } + roleToComponent := make(map[string]string, len(selection.componentToRole)) + for componentName, roleName := range selection.componentToRole { + roleToComponent[roleName] = componentName + } + for i := range typedSpec.Roles { + role := &typedSpec.Roles[i] + componentName, selected := roleToComponent[role.Name] + if !selected { + continue + } + if role.Spec.LeaderWorkerTemplate.LeaderTemplate != nil { + if timestamp := role.Spec.LeaderWorkerTemplate.LeaderTemplate.Annotations[consts.RestartAnnotation]; timestamp != "" { + restartAnnotations[componentName] = timestamp + continue + } + } + if timestamp := role.Spec.LeaderWorkerTemplate.WorkerTemplate.Annotations[consts.RestartAnnotation]; timestamp != "" { + restartAnnotations[componentName] = timestamp + } + } + return restartAnnotations, nil +} + +func (r *disaggregatedSetWorkloadsReconciler) applyDisaggregatedSetCheckpointStartupPolicies( + components map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + checkpointInfos map[string]*checkpoint.CheckpointInfo, + selection disaggregatedSetSelection, +) (bool, error) { + for _, componentName := range sortedComponentNames(components) { + if err := applyCheckpointStartupPolicy(components[componentName], checkpointInfos[componentName]); err != nil { + return false, fmt.Errorf("failed to apply checkpoint startup policy for %s: %w", componentName, err) + } + } + + gateSelectedRoles := false + for componentName := range selection.componentToRole { + info := checkpointInfos[componentName] + if info != nil && + info.Enabled && + info.StartupPolicy == nvidiacomv1alpha1.CheckpointStartupPolicyWaitForCheckpoint && + !info.Ready { + gateSelectedRoles = true + break + } + } + for componentName := range selection.componentToRole { + component := components[componentName] + if component == nil { + return false, fmt.Errorf("generated DynamoComponentDeployment missing for selected component %q", componentName) + } + if gateSelectedRoles { + component.Replicas = ptr.To(int32(0)) + } + selection.desiredReplicas[componentName] = desiredComponentReplicas(component) + } + return gateSelectedRoles, nil +} + +func sortedComponentNames(components map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec) []string { + names := make([]string, 0, len(components)) + for name := range components { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func (r *disaggregatedSetWorkloadsReconciler) reconcileDisaggregatedSetNonSelectedDCDs( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + dcds map[string]*nvidiacomv1beta1.DynamoComponentDeployment, + selection disaggregatedSetSelection, +) ([]Resource, error) { + resources := []Resource{} + for _, componentName := range sortedDCDKeys(dcds) { + dcd := dcds[componentName] + if _, selected := selection.componentToRole[componentName]; selected { + continue + } + if err := preserveExistingBackendFramework(ctx, r.Client, dcd); err != nil { + return nil, fmt.Errorf("failed to preserve existing DynamoComponentDeployment backendFramework: %w", err) + } + _, syncedDCD, err := commoncontroller.SyncResource(ctx, r, dgd, func(context.Context) (*nvidiacomv1beta1.DynamoComponentDeployment, bool, error) { + return dcd, false, nil + }) + if err != nil { + return nil, fmt.Errorf("failed to sync non-DisaggregatedSet DynamoComponentDeployment %s: %w", dcd.Name, err) + } + dcds[componentName] = syncedDCD + resources = append(resources, syncedDCD) + } + return resources, nil +} + +// generateDisaggregatedSet keeps the narrow legacy test/helper API for callers +// that already have materialized DCDs. The reconciler uses the normalized path +// below and does not materialize selected DCDs. +func (r *disaggregatedSetWorkloadsReconciler) generateDisaggregatedSet( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + dcds map[string]*nvidiacomv1beta1.DynamoComponentDeployment, + selection disaggregatedSetSelection, +) (*unstructured.Unstructured, error) { + components := make(map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, len(dcds)) + for name, dcd := range dcds { + if dcd != nil { + components[name] = &dcd.Spec.DynamoComponentDeploymentSharedSpec + } + } + return r.generateDisaggregatedSetFromNormalized(ctx, dgd, components, selection, dynamo.RollingUpdateContext{}) +} + +func (r *disaggregatedSetWorkloadsReconciler) generateDisaggregatedSetFromNormalized( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + components map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + selection disaggregatedSetSelection, + rollingUpdateCtx dynamo.RollingUpdateContext, +) (*unstructured.Unstructured, error) { + ds := newDisaggregatedSetObject() + ds.SetName(disaggregatedSetName(dgd)) + ds.SetNamespace(dgd.Namespace) + ds.SetLabels(map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: dgd.Name, + consts.KubeLabelDynamoSelector: disaggregatedSetName(dgd), + }) + if ownerRef := dgdControllerOwnerReference(dgd); ownerRef != nil { + ds.SetOwnerReferences([]metav1.OwnerReference{*ownerRef}) + } + + roles := make([]any, 0, len(selection.componentToRole)) + for i := range dgd.Spec.Components { + componentName := dgd.Spec.Components[i].ComponentName + roleName, ok := selection.componentToRole[componentName] + if !ok { + continue + } + component := components[componentName] + if component == nil { + return nil, fmt.Errorf("normalized component missing for selected component %q", componentName) + } + backendFramework, err := dynamo.BackendFrameworkForComponent(component, dgd) + if err != nil { + return nil, fmt.Errorf("failed to determine backend framework for selected component %q: %w", componentName, err) + } + workloadName := dynamo.GetDCDResourceName(dgd, componentName, rollingUpdateCtx.NewWorkerHash) + role, err := r.buildDisaggregatedSetRole( + ctx, + dgd, + component, + componentName, + workloadName, + dynamo.GetDynamoNamespace(dgd, component), + backendFramework, + ) + if err != nil { + return nil, fmt.Errorf("failed to build DisaggregatedSet role %q: %w", roleName, err) + } + role["name"] = roleName + roles = append(roles, role) + } + if len(roles) < 2 { + return nil, fmt.Errorf("DisaggregatedSet requires at least two roles, got %d", len(roles)) + } + ds.Object["spec"] = map[string]any{"roles": roles} + return ds, nil +} + +func disaggregatedSetTargetRevision(ds *unstructured.Unstructured) (string, error) { + typedDS := &disaggregatedsetv1.DisaggregatedSet{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(ds.Object, typedDS); err != nil { + return "", fmt.Errorf("failed to decode DisaggregatedSet for target revision: %w", err) + } + return disaggregatedsetutils.ComputeRevision(typedDS.Spec.Roles), nil +} + +func setDisaggregatedSetServiceSelector(service *corev1.Service, setName, roleName, revision string) { + // Dynamo's stable component Service intentionally omits the slice label so it + // aggregates every slice for this role and revision. LWS owns separate + // slice-local -prv Services for intra-slice discovery. + service.Spec.Selector = map[string]string{ + disaggregatedsetv1.SetNameLabelKey: setName, + disaggregatedsetv1.RoleLabelKey: roleName, + disaggregatedsetv1.RevisionLabelKey: revision, + } +} + +func setDesiredDisaggregatedSetServiceSelector( + service *corev1.Service, + existingService *corev1.Service, + hasExistingService bool, + setName string, + roleName string, + revision string, + targetReady bool, +) { + if targetReady || !hasExistingService { + setDisaggregatedSetServiceSelector(service, setName, roleName, revision) + return + } + service.Spec.Selector = maps.Clone(existingService.Spec.Selector) +} + +// buildDisaggregatedSetRole reuses the shared DCD workload renderer. +func (r *disaggregatedSetWorkloadsReconciler) buildDisaggregatedSetRole( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + component *nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + componentName string, + workloadName string, + dynamoNamespace string, + backendFramework dynamo.BackendFramework, +) (map[string]any, error) { + leaderPodTemplateSpec, workerPodTemplateSpec, err := r.renderer.renderMultinodePodTemplateSpecsForDGDComponent( + ctx, + dgd, + component, + componentName, + workloadName, + dynamoNamespace, + backendFramework, + ) + if err != nil { + return nil, err + } + + desiredReplicas := int32(1) + if component.Replicas != nil { + desiredReplicas = *component.Replicas + } + groupSize := component.GetNumberOfNodes() + + lwsSpec := leaderworkersetv1.LeaderWorkerSetSpec{ + Replicas: &desiredReplicas, + StartupPolicy: leaderworkersetv1.LeaderCreatedStartupPolicy, + RolloutStrategy: leaderworkersetv1.RolloutStrategy{ + Type: leaderworkersetv1.RollingUpdateStrategyType, + }, + LeaderWorkerTemplate: leaderworkersetv1.LeaderWorkerTemplate{ + LeaderTemplate: leaderPodTemplateSpec, + WorkerTemplate: *workerPodTemplateSpec, + Size: &groupSize, + RestartPolicy: leaderworkersetv1.RecreateGroupOnPodRestart, + }, + } + lwsSpecUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&lwsSpec) + if err != nil { + return nil, fmt.Errorf("failed to convert LeaderWorkerSet spec: %w", err) + } + return map[string]any{"spec": lwsSpecUnstructured}, nil +} + +func (r *disaggregatedSetWorkloadsReconciler) reconcileDisaggregatedSetSideResources( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + dcds map[string]*nvidiacomv1beta1.DynamoComponentDeployment, + normalizedComponents map[string]*nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec, + selection disaggregatedSetSelection, + targetRevision string, + targetReady bool, + rollingUpdateCtx dynamo.RollingUpdateContext, +) (map[string]struct{}, error) { + desiredServiceNames := map[string]struct{}{} + componentNames := make([]string, 0, len(normalizedComponents)) + for componentName := range normalizedComponents { + componentNames = append(componentNames, componentName) + } + sort.Strings(componentNames) + for _, componentName := range componentNames { + component := normalizedComponents[componentName] + serviceName := dynamo.GetDCDResourceName(dgd, componentName, "") + if dcd := dcds[componentName]; dcd != nil { + serviceName = dcd.Name + } + if _, selected := selection.componentToRole[componentName]; selected { + serviceName = dynamo.GetDCDResourceName(dgd, componentName, rollingUpdateCtx.NewWorkerHash) + } + service, deleted, err := r.renderer.generateServiceForDGDComponent(ctx, dgd, component, componentName, serviceName) + if err != nil { + return nil, fmt.Errorf("failed to render component service for %q: %w", componentName, err) + } + if deleted { + continue + } + if roleName, selected := selection.componentToRole[componentName]; selected && targetRevision != "" { + existing := &corev1.Service{} + existingErr := r.Get(ctx, types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, existing) + if existingErr != nil && !apierrors.IsNotFound(existingErr) { + return nil, fmt.Errorf("failed to get existing component service for %q: %w", componentName, existingErr) + } + setDesiredDisaggregatedSetServiceSelector(service, existing, existingErr == nil, disaggregatedSetName(dgd), roleName, targetRevision, targetReady) + } + if err := r.syncDGDStableService(ctx, dgd, service); err != nil { + return nil, fmt.Errorf("failed to reconcile component service for %q: %w", componentName, err) + } + desiredServiceNames[service.Name] = struct{}{} + } + + modelNames := map[string]struct{}{} + for _, component := range normalizedComponents { + if component.ModelRef == nil || component.ModelRef.Name == "" { + continue + } + modelNames[component.ModelRef.Name] = struct{}{} + } + modelNamesSorted := make([]string, 0, len(modelNames)) + for modelName := range modelNames { + modelNamesSorted = append(modelNamesSorted, modelName) + } + sort.Strings(modelNamesSorted) + for _, modelName := range modelNamesSorted { + annotations := maps.Clone(dgd.Spec.Annotations) + service := dynamo.GenerateModelServiceForGraph(dgd.Namespace, modelName, dgd.Name, annotations) + if err := r.syncDGDStableService(ctx, dgd, service); err != nil { + return nil, fmt.Errorf("failed to reconcile model service for %q: %w", modelName, err) + } + desiredServiceNames[service.Name] = struct{}{} + } + return desiredServiceNames, nil +} + +// syncDGDStableService renders the complete graph-level Service and performs +// at most one API write for it. In particular, ownership and managed metadata +// are composed before the write instead of being patched after SyncResource. +func (r *disaggregatedSetWorkloadsReconciler) syncDGDStableService( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + desired *corev1.Service, +) error { + key := types.NamespacedName{Name: desired.Name, Namespace: desired.Namespace} + existing := &corev1.Service{} + if err := r.Get(ctx, key, existing); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + setDGDControllerOwnerReference(dgd, desired) + if err := r.Create(ctx, desired); err != nil { + return err + } + return nil + } + + if owner := metav1.GetControllerOf(existing); owner != nil && !ownerReferenceMatchesDGD(owner, dgd) { + if owner.APIVersion != nvidiacomv1beta1.GroupVersion.String() || owner.Kind != dynamoComponentDeploymentKind { + return fmt.Errorf("Service %s/%s is controlled by %s/%s %q", existing.Namespace, existing.Name, owner.APIVersion, owner.Kind, owner.Name) + } + currentOwner := &nvidiacomv1beta1.DynamoComponentDeployment{} + if err := r.Get(ctx, types.NamespacedName{Name: owner.Name, Namespace: existing.Namespace}, currentOwner); err != nil { + return fmt.Errorf("failed to get current Service owner %s/%s: %w", existing.Namespace, owner.Name, err) + } + if !isControlledByBetaDGD(currentOwner, dgd) { + return fmt.Errorf("Service %s/%s is controlled by unrelated DynamoComponentDeployment %s", existing.Namespace, existing.Name, currentOwner.Name) + } + } + + updated := existing.DeepCopy() + updated.Spec = *desired.Spec.DeepCopy() + normalizeDGDStableServiceSpec(&updated.Spec) + updated.Labels = maps.Clone(existing.Labels) + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + maps.Copy(updated.Labels, desired.Labels) + updated.Annotations = maps.Clone(existing.Annotations) + if updated.Annotations == nil { + updated.Annotations = map[string]string{} + } + maps.Copy(updated.Annotations, desired.Annotations) + setDGDControllerOwnerReference(dgd, updated) + + // These fields are allocated by the apiserver and must survive a desired + // Service render. + updated.Spec.ClusterIP = existing.Spec.ClusterIP + updated.Spec.ClusterIPs = existing.Spec.ClusterIPs + updated.Spec.IPFamilies = existing.Spec.IPFamilies + updated.Spec.IPFamilyPolicy = existing.Spec.IPFamilyPolicy + updated.Spec.HealthCheckNodePort = existing.Spec.HealthCheckNodePort + + if equality.Semantic.DeepEqual(existing.Spec, updated.Spec) && + equality.Semantic.DeepEqual(existing.Labels, updated.Labels) && + equality.Semantic.DeepEqual(existing.Annotations, updated.Annotations) && + equality.Semantic.DeepEqual(existing.OwnerReferences, updated.OwnerReferences) { + return nil + } + if err := r.Update(ctx, updated); err != nil { + return err + } + return nil +} + +// normalizeDGDStableServiceSpec applies the defaults that the apiserver adds +// to a Service when the rendered object leaves those fields unspecified. This +// keeps the desired object comparable with the object read on the next +// reconcile and prevents a defaulting-induced update loop. +func normalizeDGDStableServiceSpec(spec *corev1.ServiceSpec) { + if spec.Type == "" { + spec.Type = corev1.ServiceTypeClusterIP + } + if spec.SessionAffinity == "" { + spec.SessionAffinity = corev1.ServiceAffinityNone + } + if spec.SessionAffinity == corev1.ServiceAffinityNone { + spec.SessionAffinityConfig = nil + } + if spec.Type == corev1.ServiceTypeNodePort || spec.Type == corev1.ServiceTypeLoadBalancer { + if spec.ExternalTrafficPolicy == "" { + spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeCluster + } + } + if spec.InternalTrafficPolicy == nil && + (spec.Type == corev1.ServiceTypeClusterIP || + spec.Type == corev1.ServiceTypeNodePort || + spec.Type == corev1.ServiceTypeLoadBalancer) { + spec.InternalTrafficPolicy = ptr.To(corev1.ServiceInternalTrafficPolicyCluster) + } + for i := range spec.Ports { + port := &spec.Ports[i] + if port.Protocol == "" { + port.Protocol = corev1.ProtocolTCP + } + if port.TargetPort == intstr.FromInt32(0) || port.TargetPort == intstr.FromString("") { + port.TargetPort = intstr.FromInt32(port.Port) + } + } + if spec.Type == corev1.ServiceTypeLoadBalancer && spec.AllocateLoadBalancerNodePorts == nil { + spec.AllocateLoadBalancerNodePorts = ptr.To(true) + } +} + +func sortedDCDKeys(dcds map[string]*nvidiacomv1beta1.DynamoComponentDeployment) []string { + keys := make([]string, 0, len(dcds)) + for key := range dcds { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func (r *disaggregatedSetWorkloadsReconciler) syncDisaggregatedSet( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + desired *unstructured.Unstructured, +) (bool, *unstructured.Unstructured, error) { + current := newDisaggregatedSetObject() + key := types.NamespacedName{Name: desired.GetName(), Namespace: desired.GetNamespace()} + err := r.Get(ctx, key, current) + if apierrors.IsNotFound(err) { + if err := r.Create(ctx, desired); err != nil { + return false, nil, fmt.Errorf("failed to create DisaggregatedSet %s: %w", key, err) + } + return true, desired, nil + } + if err != nil { + return false, nil, fmt.Errorf("failed to get DisaggregatedSet %s: %w", key, err) + } + if !isControlledByBetaDGD(current, dgd) { + return false, nil, fmt.Errorf( + "refusing to reconcile DisaggregatedSet %s because it is not controlled by DynamoGraphDeployment %s/%s", + key, + dgd.Namespace, + dgd.Name, + ) + } + + original := current.DeepCopy() + labels := maps.Clone(current.GetLabels()) + if labels == nil { + labels = map[string]string{} + } + maps.Copy(labels, desired.GetLabels()) + current.SetLabels(labels) + annotations := maps.Clone(current.GetAnnotations()) + if annotations == nil && len(desired.GetAnnotations()) > 0 { + annotations = map[string]string{} + } + maps.Copy(annotations, desired.GetAnnotations()) + current.SetAnnotations(annotations) + setDGDControllerOwnerReference(dgd, current) + current.Object["spec"] = desired.Object["spec"] + if disaggregatedSetDesiredStateEqual(original, current) { + return false, current, nil + } + + // Let the API server apply structural defaults before deciding whether the + // controller-owned desired state differs from the live object. + if err := r.Patch(ctx, current, client.MergeFrom(original), client.DryRunAll); err != nil { + return false, nil, fmt.Errorf("failed to dry-run patch DisaggregatedSet %s: %w", key, err) + } + if disaggregatedSetDesiredStateEqual(original, current) { + return false, current, nil + } + if err := r.Patch(ctx, current, client.MergeFrom(original)); err != nil { + return false, nil, fmt.Errorf("failed to patch DisaggregatedSet %s: %w", key, err) + } + return true, current, nil +} + +func disaggregatedSetDesiredStateEqual(a, b *unstructured.Unstructured) bool { + if a == nil || b == nil { + return a == b + } + return equality.Semantic.DeepEqual(a.Object["spec"], b.Object["spec"]) && + equality.Semantic.DeepEqual(a.GetLabels(), b.GetLabels()) && + equality.Semantic.DeepEqual(a.GetAnnotations(), b.GetAnnotations()) && + equality.Semantic.DeepEqual(a.GetOwnerReferences(), b.GetOwnerReferences()) +} + +func (r *disaggregatedSetWorkloadsReconciler) deleteOwnedSelectedDCDs( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + selection disaggregatedSetSelection, +) error { + dcds, err := r.listOwnedSelectedDCDs(ctx, dgd, selection) + if err != nil { + return err + } + for i := range dcds { + dcd := &dcds[i] + if err := r.Delete(ctx, dcd); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete selected DynamoComponentDeployment %s/%s: %w", dcd.Namespace, dcd.Name, err) + } + } + return nil +} + +func (r *disaggregatedSetWorkloadsReconciler) deleteStaleDisaggregatedSetServices( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + desiredServiceNames map[string]struct{}, +) error { + serviceList := &corev1.ServiceList{} + if err := r.List(ctx, serviceList, client.InNamespace(dgd.Namespace)); err != nil { + return fmt.Errorf("failed to list DisaggregatedSet Services: %w", err) + } + + for i := range serviceList.Items { + service := &serviceList.Items[i] + if !isControlledByBetaDGD(service, dgd) { + continue + } + // The controller owner is the authoritative scope; labels only classify + // the owned Service as a component or model endpoint. + componentName := service.Labels[consts.KubeLabelDynamoComponent] + modelHash := service.Labels[consts.KubeLabelDynamoBaseModelHash] + if componentName == "" && modelHash == "" { + continue + } + if _, desired := desiredServiceNames[service.Name]; desired { + continue + } + if err := r.Delete(ctx, service); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete stale Service %s/%s: %w", service.Namespace, service.Name, err) + } + } + return nil +} + +func (r *disaggregatedSetWorkloadsReconciler) listOwnedSelectedDCDs( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + selection disaggregatedSetSelection, +) ([]nvidiacomv1beta1.DynamoComponentDeployment, error) { + dcdList := &nvidiacomv1beta1.DynamoComponentDeploymentList{} + if err := r.List(ctx, dcdList, client.InNamespace(dgd.Namespace)); err != nil { + return nil, fmt.Errorf("failed to list DynamoComponentDeployments for DisaggregatedSet cleanup: %w", err) + } + selectedDCDs := []nvidiacomv1beta1.DynamoComponentDeployment{} + for _, dcd := range dcdList.Items { + if !isControlledByBetaDGD(&dcd, dgd) { + continue + } + componentName := dynamo.GetDCDComponentName(&dcd) + if _, selected := selection.componentToRole[componentName]; selected { + selectedDCDs = append(selectedDCDs, dcd) + } + } + return selectedDCDs, nil +} + +func ownerReferenceMatchesDGD(owner *metav1.OwnerReference, dgd *nvidiacomv1beta1.DynamoGraphDeployment) bool { + return owner != nil && + dgd != nil && + owner.APIVersion == nvidiacomv1beta1.GroupVersion.String() && + owner.Kind == dynamoGraphDeploymentKind && + owner.Name == dgd.Name && + owner.UID != "" && + owner.UID == dgd.UID +} + +func dgdControllerOwnerReference(dgd *nvidiacomv1beta1.DynamoGraphDeployment) *metav1.OwnerReference { + if dgd == nil || dgd.UID == "" { + return nil + } + return &metav1.OwnerReference{ + APIVersion: nvidiacomv1beta1.GroupVersion.String(), + Kind: dynamoGraphDeploymentKind, + Name: dgd.Name, + UID: dgd.UID, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + } +} + +func setDGDControllerOwnerReference(dgd *nvidiacomv1beta1.DynamoGraphDeployment, obj client.Object) { + ownerRef := dgdControllerOwnerReference(dgd) + if ownerRef == nil { + return + } + ownerRefs := make([]metav1.OwnerReference, 0, len(obj.GetOwnerReferences())+1) + for _, ref := range obj.GetOwnerReferences() { + if ptr.Deref(ref.Controller, false) { + continue + } + if ref.APIVersion == ownerRef.APIVersion && ref.Kind == ownerRef.Kind && ref.Name == ownerRef.Name { + continue + } + ownerRefs = append(ownerRefs, ref) + } + ownerRefs = append(ownerRefs, *ownerRef) + obj.SetOwnerReferences(ownerRefs) +} + +func isControlledByBetaDGD(obj client.Object, dgd *nvidiacomv1beta1.DynamoGraphDeployment) bool { + if obj == nil || dgd == nil { + return false + } + if dgd.UID != "" { + return metav1.IsControlledBy(obj, dgd) + } + controllerOwner := metav1.GetControllerOf(obj) + return controllerOwner != nil && + controllerOwner.APIVersion == nvidiacomv1beta1.GroupVersion.String() && + controllerOwner.Kind == dynamoGraphDeploymentKind && + controllerOwner.Name == dgd.Name +} + +func checkDisaggregatedSetReadiness( + ds *unstructured.Unstructured, + selection disaggregatedSetSelection, +) (bool, string, map[string]nvidiacomv1beta1.ComponentReplicaStatus) { + sliceCount := disaggregatedSetSliceCount(ds) + statuses := make(map[string]nvidiacomv1beta1.ComponentReplicaStatus, len(selection.componentToRole)) + roleStatuses := disaggregatedSetRoleStatuses(ds) + notReadyReasons := []string{} + for componentName, roleName := range selection.componentToRole { + desiredReplicas := selection.desiredReplicas[componentName] * sliceCount + componentStatus := nvidiacomv1beta1.ComponentReplicaStatus{ + ComponentKind: nvidiacomv1beta1.ComponentKindLeaderWorkerSet, + } + roleStatus, found := roleStatuses[roleName] + if found { + componentStatus.Replicas = nestedInt32(roleStatus, "replicas") + componentStatus.UpdatedReplicas = nestedInt32(roleStatus, "updatedReplicas") + readyReplicas := nestedInt32(roleStatus, "readyReplicas") + componentStatus.ReadyReplicas = &readyReplicas + } + statuses[componentName] = componentStatus + if !found { + notReadyReasons = append(notReadyReasons, fmt.Sprintf("%s role %q has no status yet", componentName, roleName)) + continue + } + if desiredReplicas == 0 { + if componentStatus.Replicas != 0 || componentStatus.UpdatedReplicas != 0 || ptr.Deref(componentStatus.ReadyReplicas, 0) != 0 { + notReadyReasons = append(notReadyReasons, fmt.Sprintf("%s role %q has not scaled to zero", componentName, roleName)) + } + continue + } + if componentStatus.Replicas != desiredReplicas || + componentStatus.UpdatedReplicas != desiredReplicas || + componentStatus.ReadyReplicas == nil || + *componentStatus.ReadyReplicas != desiredReplicas { + notReadyReasons = append(notReadyReasons, fmt.Sprintf( + "%s role %q replicas not ready (desired=%d replicas=%d updated=%d ready=%d)", + componentName, + roleName, + desiredReplicas, + componentStatus.Replicas, + componentStatus.UpdatedReplicas, + ptr.Deref(componentStatus.ReadyReplicas, 0), + )) + } + } + if current, reason := disaggregatedSetStatusObserved(ds); !current { + return false, reason, statuses + } + if len(notReadyReasons) > 0 { + sort.Strings(notReadyReasons) + return false, strings.Join(notReadyReasons, "; "), statuses + } + return true, "All DisaggregatedSet roles are ready", statuses +} + +func disaggregatedSetSliceCount(ds *unstructured.Unstructured) int32 { + if ds == nil { + return 1 + } + value, found := nestedInt64FromObject(ds.Object, "spec", "slices") + if !found || value < 1 { + return 1 + } + return int32(value) +} + +func disaggregatedSetChildSlice(labels map[string]string) (int, bool) { + value := labels[disaggregatedsetv1.SliceLabelKey] + if value == "" { + return 0, true + } + slice, err := strconv.Atoi(value) + return slice, err == nil && slice >= 0 +} + +func (r *disaggregatedSetWorkloadsReconciler) checkDisaggregatedSetReadiness( + ctx context.Context, + ds *unstructured.Unstructured, + selection disaggregatedSetSelection, +) (bool, string, map[string]nvidiacomv1beta1.ComponentReplicaStatus, error) { + children := &leaderworkersetv1.LeaderWorkerSetList{} + if err := r.List(ctx, children, client.InNamespace(ds.GetNamespace()), client.MatchingLabels{ + disaggregatedsetv1.SetNameLabelKey: ds.GetName(), + }); err != nil { + return false, "", nil, fmt.Errorf("failed to list DisaggregatedSet child LeaderWorkerSets: %w", err) + } + targetRevision, err := disaggregatedSetTargetRevision(ds) + if err != nil { + return false, "", nil, err + } + targetByIdentity := make(map[disaggregatedSetChildIdentity][]*leaderworkersetv1.LeaderWorkerSet) + childrenByRole := make(map[string][]*leaderworkersetv1.LeaderWorkerSet) + sliceCount := int(disaggregatedSetSliceCount(ds)) + for i := range children.Items { + child := &children.Items[i] + if !metav1.IsControlledBy(child, ds) { + continue + } + roleName := child.Labels[disaggregatedsetv1.RoleLabelKey] + childrenByRole[roleName] = append(childrenByRole[roleName], child) + slice, validSlice := disaggregatedSetChildSlice(child.Labels) + if !validSlice || slice >= sliceCount || child.Labels[disaggregatedsetv1.RevisionLabelKey] != targetRevision { + continue + } + identity := disaggregatedSetChildIdentity{slice: slice, role: roleName} + targetByIdentity[identity] = append(targetByIdentity[identity], child) + } + if len(disaggregatedSetRoleStatuses(ds)) > 0 && disaggregatedSetStatusHasObservation(ds) { + ready, reason, statuses := checkDisaggregatedSetReadiness(ds, selection) + if !ready { + return ready, reason, statuses, nil + } + } + ready, reason, statuses := checkDisaggregatedSetChildLWSReadiness(selection, sliceCount, targetByIdentity, childrenByRole) + return ready, reason, statuses, nil +} + +func checkDisaggregatedSetChildLWSReadiness( + selection disaggregatedSetSelection, + sliceCount int, + targetByIdentity map[disaggregatedSetChildIdentity][]*leaderworkersetv1.LeaderWorkerSet, + childrenByRole map[string][]*leaderworkersetv1.LeaderWorkerSet, +) (bool, string, map[string]nvidiacomv1beta1.ComponentReplicaStatus) { + statuses := make(map[string]nvidiacomv1beta1.ComponentReplicaStatus, len(selection.componentToRole)) + notReadyReasons := []string{} + for componentName, roleName := range selection.componentToRole { + desiredReplicas := selection.desiredReplicas[componentName] + status := nvidiacomv1beta1.ComponentReplicaStatus{ComponentKind: nvidiacomv1beta1.ComponentKindLeaderWorkerSet} + children := childrenByRole[roleName] + sort.Slice(children, func(i, j int) bool { return children[i].Name < children[j].Name }) + readyReplicas := int32(0) + for _, roleChild := range children { + status.ComponentNames = append(status.ComponentNames, roleChild.Name) + status.Replicas += roleChild.Status.Replicas + readyReplicas += roleChild.Status.ReadyReplicas + } + status.ReadyReplicas = ptr.To(readyReplicas) + for slice := range sliceCount { + identity := disaggregatedSetChildIdentity{slice: slice, role: roleName} + targets := targetByIdentity[identity] + if len(targets) != 1 { + notReadyReasons = append(notReadyReasons, fmt.Sprintf( + "%s role %q slice %d has %d target LeaderWorkerSets, expected 1", + componentName, roleName, slice, len(targets), + )) + continue + } + child := targets[0] + status.UpdatedReplicas += child.Status.UpdatedReplicas + if child.Status.ObservedGeneration < child.Generation { + notReadyReasons = append(notReadyReasons, fmt.Sprintf("%s child LeaderWorkerSet %q has not observed generation %d", componentName, child.Name, child.Generation)) + continue + } + if child.Status.Replicas != desiredReplicas || child.Status.UpdatedReplicas != desiredReplicas || child.Status.ReadyReplicas != desiredReplicas { + notReadyReasons = append(notReadyReasons, fmt.Sprintf( + "%s child LeaderWorkerSet %q for slice %d replicas not ready (desired=%d replicas=%d updated=%d ready=%d)", + componentName, + child.Name, + slice, + desiredReplicas, + child.Status.Replicas, + child.Status.UpdatedReplicas, + child.Status.ReadyReplicas, + )) + } + } + statuses[componentName] = status + } + + notReadyReasons = append(notReadyReasons, staleDisaggregatedSetChildLWSNotReadyReasons(targetByIdentity, childrenByRole)...) + if len(notReadyReasons) > 0 { + sort.Strings(notReadyReasons) + return false, strings.Join(notReadyReasons, "; "), statuses + } + return true, "All DisaggregatedSet child LeaderWorkerSets are ready", statuses +} + +func staleDisaggregatedSetChildLWSNotReadyReasons( + targetByIdentity map[disaggregatedSetChildIdentity][]*leaderworkersetv1.LeaderWorkerSet, + childrenByRole map[string][]*leaderworkersetv1.LeaderWorkerSet, +) []string { + targets := make(map[*leaderworkersetv1.LeaderWorkerSet]struct{}) + for _, children := range targetByIdentity { + if len(children) == 1 { + targets[children[0]] = struct{}{} + } + } + notReadyReasons := []string{} + for roleName, children := range childrenByRole { + sort.Slice(children, func(i, j int) bool { return children[i].Name < children[j].Name }) + for _, child := range children { + if _, target := targets[child]; target { + continue + } + if ptr.Deref(child.Spec.Replicas, 1) == 0 && + child.Status.Replicas == 0 && + child.Status.UpdatedReplicas == 0 && + child.Status.ReadyReplicas == 0 { + continue + } + notReadyReasons = append(notReadyReasons, fmt.Sprintf( + "stale role %q child LeaderWorkerSet %q has not scaled to zero", + roleName, + child.Name, + )) + } + } + sort.Strings(notReadyReasons) + return notReadyReasons +} + +func disaggregatedSetStatusObserved(ds *unstructured.Unstructured) (bool, string) { + if ds == nil || ds.GetGeneration() == 0 { + return true, "" + } + if observedGeneration, found := nestedInt64FromObject(ds.Object, "status", "observedGeneration"); found && observedGeneration < ds.GetGeneration() { + return false, fmt.Sprintf("DisaggregatedSet status has not observed generation %d (observedGeneration=%d)", ds.GetGeneration(), observedGeneration) + } + conditions, found, _ := unstructured.NestedSlice(ds.Object, "status", "conditions") + if !found { + return true, "" + } + for _, item := range conditions { + condition, ok := item.(map[string]any) + if !ok { + continue + } + observedGeneration, ok := nestedInt64(condition, "observedGeneration") + if !ok || observedGeneration >= ds.GetGeneration() { + continue + } + conditionType, _ := condition["type"].(string) + return false, fmt.Sprintf("DisaggregatedSet condition %q has not observed generation %d (observedGeneration=%d)", conditionType, ds.GetGeneration(), observedGeneration) + } + return true, "" +} + +func disaggregatedSetStatusHasObservation(ds *unstructured.Unstructured) bool { + if ds == nil || ds.GetGeneration() == 0 { + return true + } + if _, found := nestedInt64FromObject(ds.Object, "status", "observedGeneration"); found { + return true + } + conditions, found, _ := unstructured.NestedSlice(ds.Object, "status", "conditions") + if !found { + return false + } + for _, item := range conditions { + condition, ok := item.(map[string]any) + if !ok { + continue + } + if _, found := nestedInt64(condition, "observedGeneration"); found { + return true + } + } + return false +} + +func disaggregatedSetRoleStatuses(ds *unstructured.Unstructured) map[string]map[string]any { + out := map[string]map[string]any{} + roleStatuses, found, _ := unstructured.NestedSlice(ds.Object, "status", "roleStatuses") + if !found { + return out + } + for _, item := range roleStatuses { + roleStatus, ok := item.(map[string]any) + if !ok { + continue + } + name, ok := roleStatus["name"].(string) + if !ok || name == "" { + continue + } + out[name] = roleStatus + } + return out +} + +func nestedInt32(obj map[string]any, key string) int32 { + value, _ := nestedInt64(obj, key) + return int32(value) +} + +func nestedInt64FromObject(obj map[string]any, fields ...string) (int64, bool) { + value, found, err := unstructured.NestedFieldNoCopy(obj, fields...) + if err != nil || !found { + return 0, false + } + return int64Value(value) +} + +func nestedInt64(obj map[string]any, key string) (int64, bool) { + return int64Value(obj[key]) +} + +func int64Value(value any) (int64, bool) { + switch v := value.(type) { + case int32: + return int64(v), true + case int64: + return v, true + case int: + return int64(v), true + case float64: + return int64(v), true + default: + return 0, false + } +} + +func disaggregatedSetStatusChanged(oldObj, newObj client.Object) bool { + oldDS, okOld := oldObj.(*unstructured.Unstructured) + newDS, okNew := newObj.(*unstructured.Unstructured) + if !okOld || !okNew { + return false + } + return oldDS.GetGeneration() != newDS.GetGeneration() || + !equality.Semantic.DeepEqual(oldDS.Object["status"], newDS.Object["status"]) || + !equality.Semantic.DeepEqual(oldDS.GetLabels(), newDS.GetLabels()) || + !equality.Semantic.DeepEqual(oldDS.GetOwnerReferences(), newDS.GetOwnerReferences()) +} + +func leaderWorkerSetStatusChanged(oldObj, newObj client.Object) bool { + oldLWS, okOld := oldObj.(*leaderworkersetv1.LeaderWorkerSet) + newLWS, okNew := newObj.(*leaderworkersetv1.LeaderWorkerSet) + if !okOld || !okNew { + return false + } + return oldLWS.Generation != newLWS.Generation || + !equality.Semantic.DeepEqual(oldLWS.Status, newLWS.Status) || + !equality.Semantic.DeepEqual(oldLWS.GetLabels(), newLWS.GetLabels()) || + !equality.Semantic.DeepEqual(oldLWS.GetOwnerReferences(), newLWS.GetOwnerReferences()) +} + +type disaggregatedSetWatchMapper struct { + reader client.Reader +} + +func newDisaggregatedSetWatchMapper(reader client.Reader) *disaggregatedSetWatchMapper { + return &disaggregatedSetWatchMapper{reader: reader} +} + +func (r *disaggregatedSetWatchMapper) MapChildLWSToDGD(ctx context.Context, obj client.Object) []ctrl.Request { + setName := obj.GetLabels()[disaggregatedsetv1.SetNameLabelKey] + if setName == "" { + return nil + } + ds := newDisaggregatedSetObject() + if err := r.reader.Get(ctx, types.NamespacedName{Name: setName, Namespace: obj.GetNamespace()}, ds); err != nil { + if !apierrors.IsNotFound(err) { + log.FromContext(ctx).Error(err, "failed to map DisaggregatedSet child LeaderWorkerSet", "leaderWorkerSet", obj.GetName()) + } + return nil + } + owner := metav1.GetControllerOf(ds) + if owner == nil || owner.APIVersion != nvidiacomv1beta1.GroupVersion.String() || owner.Kind != dynamoGraphDeploymentKind { + return nil + } + return []ctrl.Request{{NamespacedName: types.NamespacedName{Name: owner.Name, Namespace: ds.GetNamespace()}}} +} + +func (r *disaggregatedSetWorkloadsReconciler) getUpdatedInProgressForDisaggregatedSet( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + inProgress []string, +) []string { + logger := log.FromContext(ctx) + selection, reason := selectDisaggregatedSetComponents(dgd) + if reason != "" { + logger.V(1).Info("failed to select DisaggregatedSet components for restart progress", "reason", reason) + return inProgress + } + + ds := newDisaggregatedSetObject() + dsErr := r.Get(ctx, types.NamespacedName{Name: disaggregatedSetName(dgd), Namespace: dgd.Namespace}, ds) + if dsErr != nil && !apierrors.IsNotFound(dsErr) { + logger.V(1).Info("failed to get DisaggregatedSet for restart progress", "error", dsErr) + } + dsReady := false + dsReason := resourceNotFoundReason + if dsErr == nil { + var err error + dsReady, dsReason, _, err = r.checkDisaggregatedSetReadiness(ctx, ds, selection) + if err != nil { + dsReason = err.Error() + } + } + + updatedInProgress := make([]string, 0, len(inProgress)) + for _, componentName := range inProgress { + if _, selected := selection.componentToRole[componentName]; !selected { + isFullyUpdated, reason := r.componentRestartProgress.checkComponentFullyUpdated(ctx, dgd, componentName) + if !isFullyUpdated { + logger.V(1).Info("component not fully updated", "componentName", componentName, "reason", reason) + updatedInProgress = append(updatedInProgress, componentName) + } + continue + } + + if dsErr != nil { + reason := resourceNotFoundReason + if !apierrors.IsNotFound(dsErr) { + reason = dsErr.Error() + } + logger.V(1).Info("DisaggregatedSet component not fully updated", "componentName", componentName, "reason", reason) + updatedInProgress = append(updatedInProgress, componentName) + continue + } + + if !dsReady { + logger.V(1).Info("DisaggregatedSet component not fully updated", "componentName", componentName, "reason", dsReason) + updatedInProgress = append(updatedInProgress, componentName) + } + } + return updatedInProgress +} diff --git a/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_clustertest_helpers_test.go b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_clustertest_helpers_test.go new file mode 100644 index 000000000000..a1b1769cc593 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_clustertest_helpers_test.go @@ -0,0 +1,75 @@ +//go:build clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "fmt" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" +) + +func setupProductionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptions) error { + return webhooksetup.Setup(mgr, webhooksetup.Options{ + Config: opts.OperatorConfig, + RuntimeConfig: opts.RuntimeConfig, + OperatorVersion: opts.OperatorVersion, + OperatorPrincipal: opts.OperatorPrincipal, + }) +} + +func newEnvtestDSHappyPathDGD(name string) *nvidiacomv1beta1.DynamoGraphDeployment { + return &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: types.UID(fmt.Sprintf("%s-uid", name)), + Annotations: map[string]string{ + consts.KubeAnnotationEnableDisaggregatedSet: consts.KubeLabelValueTrue, + }, + }, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + BackendFramework: "vllm", + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "prefill", + ComponentType: nvidiacomv1beta1.ComponentTypePrefill, + RuntimeVersionOverride: "1.0.0", + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + PodTemplate: envtestDSTestPodTemplate(), + }, + { + ComponentName: "decode", + ComponentType: nvidiacomv1beta1.ComponentTypeDecode, + RuntimeVersionOverride: "1.0.0", + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + PodTemplate: envtestDSTestPodTemplate(), + }, + }, + }, + } +} + +func envtestDSTestPodTemplate() *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: consts.MainContainerName, + Image: "busybox:1.36", + Command: []string{"sh"}, + Args: []string{"-c", "sleep 3600"}, + }}, + }, + } +} diff --git a/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_envtest_test.go b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_envtest_test.go new file mode 100644 index 000000000000..b21ed9d12f68 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_envtest_test.go @@ -0,0 +1,448 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "fmt" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apiMeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" + disaggregatedsetutils "sigs.k8s.io/lws/pkg/utils/disaggregatedset" +) + +var _ = Describe("DisaggregatedSet envtest semantics", func() { + It("propagates graph metadata into both DS roles and rotates the revision", func() { + ctx := context.Background() + dgd := newEnvtestDSHappyPathDGD("demo-ds-metadata") + dgd.Spec.Labels = map[string]string{"e2e.dynamo/metadata": "initial"} + dgd.Spec.Annotations = map[string]string{"e2e.dynamo/metadata": "initial"} + Expect(k8sClient.Create(ctx, dgd)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, dgd) }) + + reconciler := newEnvtestDSReconcilers() + + By("creating the initial DisaggregatedSet") + _, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + initialDS := fetchTypedDisaggregatedSet(ctx, current) + initialRevision := disaggregatedsetutils.ComputeRevision(initialDS.Spec.Roles) + Expect(initialDS.Spec.Roles).To(HaveLen(2)) + + By("updating graph-level metadata and reconciling again") + current.Spec.Labels["e2e.dynamo/metadata"] = "updated" + current.Spec.Annotations["e2e.dynamo/metadata"] = "updated" + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + + _, current = reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + updatedDS := fetchTypedDisaggregatedSet(ctx, current) + updatedRevision := disaggregatedsetutils.ComputeRevision(updatedDS.Spec.Roles) + Expect(updatedRevision).NotTo(Equal(initialRevision)) + for i := range updatedDS.Spec.Roles { + role := &updatedDS.Spec.Roles[i] + Expect(role.Spec.LeaderWorkerTemplate.LeaderTemplate).NotTo(BeNil()) + Expect(role.Spec.LeaderWorkerTemplate.LeaderTemplate.Labels).To(HaveKeyWithValue("e2e.dynamo/metadata", "updated")) + Expect(role.Spec.LeaderWorkerTemplate.LeaderTemplate.Annotations).To(HaveKeyWithValue("e2e.dynamo/metadata", "updated")) + Expect(role.Spec.LeaderWorkerTemplate.WorkerTemplate.Labels).To(HaveKeyWithValue("e2e.dynamo/metadata", "updated")) + Expect(role.Spec.LeaderWorkerTemplate.WorkerTemplate.Annotations).To(HaveKeyWithValue("e2e.dynamo/metadata", "updated")) + } + }) + + It("coalesces sequential restart across both DS roles and completes", func() { + ctx := context.Background() + dgd := newEnvtestDSHappyPathDGD("demo-ds-restart") + Expect(k8sClient.Create(ctx, dgd)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, dgd) }) + + reconciler := newEnvtestDSReconcilers() + + By("creating a ready baseline DisaggregatedSet revision") + _, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + baselineDS := fetchTypedDisaggregatedSet(ctx, current) + baselineRevision := disaggregatedsetutils.ComputeRevision(baselineDS.Spec.Roles) + markDisaggregatedSetReady(ctx, current) + baselineResult, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(baselineResult.Status.State).To(Equal(nvidiacomv1beta1.DGDStateSuccessful)) + + By("requesting a sequential restart through the DGD") + current.Spec.Restart = &nvidiacomv1beta1.Restart{ + ID: "ds-envtest-restart", + Strategy: &nvidiacomv1beta1.RestartStrategy{ + Type: nvidiacomv1beta1.RestartStrategyTypeSequential, + Order: []string{"prefill", "decode"}, + }, + } + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + + restartResult, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(restartResult.Status.Restart).NotTo(BeNil()) + Expect(restartResult.Status.Restart.ObservedID).To(Equal("ds-envtest-restart")) + Expect(restartResult.Status.Restart.Phase).To(Equal(nvidiacomv1beta1.RestartPhaseRestarting)) + Expect(restartResult.Status.Restart.InProgress).To(Equal([]string{"prefill"})) + + restartedDS := fetchTypedDisaggregatedSet(ctx, current) + restartedRevision := disaggregatedsetutils.ComputeRevision(restartedDS.Spec.Roles) + Expect(restartedRevision).NotTo(Equal(baselineRevision)) + restartValues := map[string]bool{} + for i := range restartedDS.Spec.Roles { + role := &restartedDS.Spec.Roles[i] + Expect(role.Spec.LeaderWorkerTemplate.LeaderTemplate).NotTo(BeNil()) + leaderRestart := role.Spec.LeaderWorkerTemplate.LeaderTemplate.Annotations[consts.RestartAnnotation] + workerRestart := role.Spec.LeaderWorkerTemplate.WorkerTemplate.Annotations[consts.RestartAnnotation] + Expect(leaderRestart).NotTo(BeEmpty()) + Expect(workerRestart).To(Equal(leaderRestart)) + restartValues[leaderRestart] = true + } + Expect(restartValues).To(HaveLen(1), "both DS roles must share one restart revision") + + By("persisting restart status and reconciling after the DS becomes ready") + current = persistWorkloadProgramStatus(ctx, current, restartResult.Status) + markDisaggregatedSetReady(ctx, current) + completedResult, _ := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(completedResult.Status.Restart).NotTo(BeNil()) + Expect(completedResult.Status.Restart.ObservedID).To(Equal("ds-envtest-restart")) + Expect(completedResult.Status.Restart.Phase).To(Equal(nvidiacomv1beta1.RestartPhaseCompleted)) + }) + + It("does not switch the durable provider when routing annotations change", func() { + ctx := context.Background() + dgd := newEnvtestDSHappyPathDGD("demo-ds-immutable-provider") + dgd.Annotations = nil + Expect(k8sClient.Create(ctx, dgd)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, dgd) }) + + reconciler := newEnvtestDSReconcilers() + _, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(current.Annotations[consts.KubeAnnotationWorkloadProvider]).To(Equal(consts.WorkloadProviderComponent)) + + current.Annotations[consts.KubeAnnotationEnableDisaggregatedSet] = consts.KubeLabelValueTrue + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + _, current = reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + + Expect(current.Annotations[consts.KubeAnnotationWorkloadProvider]).To(Equal(consts.WorkloadProviderComponent)) + Expect(apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: disaggregatedSetName(current), Namespace: current.Namespace}, newDisaggregatedSetObject()))).To(BeTrue()) + }) + + It("does not rewrite API-defaulted stable Services on a no-op reconcile", func() { + ctx := context.Background() + dgd := newEnvtestDSHappyPathDGD("demo-ds-service-idempotence") + for i := range dgd.Spec.Components { + dgd.Spec.Components[i].ModelRef = &nvidiacomv1beta1.ModelReference{Name: "llama-3"} + } + Expect(k8sClient.Create(ctx, dgd)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, dgd) }) + + reconciler := newEnvtestDSReconcilers() + By("creating the DisaggregatedSet and its graph-level Services") + _, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + + services := &corev1.ServiceList{} + Expect(k8sClient.List(ctx, services, client.InNamespace(current.Namespace))).To(Succeed()) + Expect(services.Items).NotTo(BeEmpty()) + resourceVersions := make(map[string]string, len(services.Items)) + for i := range services.Items { + resourceVersions[services.Items[i].Name] = services.Items[i].ResourceVersion + } + + By("reconciling the unchanged graph again") + _, current = reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(k8sClient.List(ctx, services, client.InNamespace(current.Namespace))).To(Succeed()) + for i := range services.Items { + Expect(services.Items[i].ResourceVersion).To(Equal(resourceVersions[services.Items[i].Name]), services.Items[i].Name) + } + }) + + It("scopes model Services to each graph and cleans only removed references", func() { + ctx := context.Background() + first := newEnvtestDSHappyPathDGD("demo-ds-model-first") + second := newEnvtestDSHappyPathDGD("demo-ds-model-second") + for i := range first.Spec.Components { + first.Spec.Components[i].ModelRef = &nvidiacomv1beta1.ModelReference{Name: "llama-3"} + second.Spec.Components[i].ModelRef = &nvidiacomv1beta1.ModelReference{Name: "llama-3"} + } + Expect(k8sClient.Create(ctx, first)).To(Succeed()) + Expect(k8sClient.Create(ctx, second)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, first); _ = k8sClient.Delete(ctx, second) }) + + reconciler := newEnvtestDSReconcilers() + By("reconciling both graphs with the same model reference") + _, firstCurrent := reconcileCurrentDGDProgram(ctx, reconciler, first.Name, first.Namespace) + _, secondCurrent := reconcileCurrentDGDProgram(ctx, reconciler, second.Name, second.Namespace) + markDisaggregatedSetReady(ctx, firstCurrent) + markDisaggregatedSetReady(ctx, secondCurrent) + _, firstCurrent = reconcileCurrentDGDProgram(ctx, reconciler, first.Name, first.Namespace) + _, secondCurrent = reconcileCurrentDGDProgram(ctx, reconciler, second.Name, second.Namespace) + + services := &corev1.ServiceList{} + Expect(k8sClient.List(ctx, services, client.InNamespace(first.Namespace))).To(Succeed()) + modelServices := map[string]corev1.Service{} + for i := range services.Items { + service := services.Items[i] + if service.Labels[consts.KubeLabelDynamoBaseModelHash] == dynamo.HashModelName("llama-3") && + service.Labels[consts.KubeLabelDynamoGraphDeploymentName] != "" { + modelServices[service.Labels[consts.KubeLabelDynamoGraphDeploymentName]] = service + } + } + Expect(modelServices).To(HaveLen(2)) + Expect(modelServices[first.Name].Name).NotTo(Equal(modelServices[second.Name].Name)) + firstService := modelServices[first.Name] + secondService := modelServices[second.Name] + Expect(metav1.IsControlledBy(&firstService, firstCurrent)).To(BeTrue()) + Expect(metav1.IsControlledBy(&secondService, secondCurrent)).To(BeTrue()) + + By("removing the first graph's final model references") + for i := range firstCurrent.Spec.Components { + firstCurrent.Spec.Components[i].ModelRef = nil + } + Expect(k8sClient.Update(ctx, firstCurrent)).To(Succeed()) + _, firstCurrent = reconcileCurrentDGDProgram(ctx, reconciler, first.Name, first.Namespace) + markDisaggregatedSetReady(ctx, firstCurrent) + _, _ = reconcileCurrentDGDProgram(ctx, reconciler, first.Name, first.Namespace) + + firstServiceKey := types.NamespacedName{Name: modelServices[first.Name].Name, Namespace: first.Namespace} + secondServiceKey := types.NamespacedName{Name: modelServices[second.Name].Name, Namespace: second.Namespace} + Expect(apierrors.IsNotFound(k8sClient.Get(ctx, firstServiceKey, &corev1.Service{}))).To(BeTrue()) + Expect(k8sClient.Get(ctx, secondServiceKey, &corev1.Service{})).To(Succeed()) + + By("removing the final remaining model reference") + for i := range secondCurrent.Spec.Components { + secondCurrent.Spec.Components[i].ModelRef = nil + } + Expect(k8sClient.Update(ctx, secondCurrent)).To(Succeed()) + _, secondCurrent = reconcileCurrentDGDProgram(ctx, reconciler, second.Name, second.Namespace) + markDisaggregatedSetReady(ctx, secondCurrent) + _, _ = reconcileCurrentDGDProgram(ctx, reconciler, second.Name, second.Namespace) + Expect(apierrors.IsNotFound(k8sClient.Get(ctx, secondServiceKey, &corev1.Service{}))).To(BeTrue()) + }) + + It("keeps the selected DisaggregatedSet when later intent is unsupported", func() { + ctx := context.Background() + dgd := newEnvtestDSHappyPathDGD("demo-ds-fallback-gating") + Expect(k8sClient.Create(ctx, dgd)).To(Succeed()) + DeferCleanup(func() { _ = k8sClient.Delete(ctx, dgd) }) + + reconciler := newEnvtestDSReconcilers() + + By("creating a ready DisaggregatedSet") + _, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + markDisaggregatedSetReady(ctx, current) + result, current := reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(result.Status.State).To(Equal(nvidiacomv1beta1.DGDStateSuccessful)) + + By("making the deployment ineligible for DisaggregatedSet while keeping the annotation") + current.Spec.Components[0].ScalingAdapter = &nvidiacomv1beta1.ScalingAdapter{} + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + + By("reporting unsupported intent without switching workload pathways") + result, current = reconcileCurrentDGDProgram(ctx, reconciler, dgd.Name, dgd.Namespace) + Expect(result.Status.State).To(Equal(nvidiacomv1beta1.DGDStateFailed)) + eligibility := apiMeta.FindStatusCondition(result.Status.Conditions, disaggregatedSetEligibleConditionType) + Expect(eligibility).NotTo(BeNil()) + Expect(eligibility.Status).To(Equal(metav1.ConditionFalse)) + Expect(eligibility.Reason).To(Equal("UnsupportedIntent")) + Expect(eligibility.Message).To(ContainSubstring("scalingAdapter")) + Expect(ownedEnvtestCutoverDCDs(ctx, current)).To(BeEmpty()) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: disaggregatedSetName(current), Namespace: current.Namespace}, newDisaggregatedSetObject())).To(Succeed()) + }) +}) + +func newEnvtestDSReconcilers() *DynamoGraphDeploymentReconciler { + runtimeConfig := &commoncontroller.RuntimeConfig{ + Gate: features.Gates{LWS: true, DisaggregatedSet: true}, + } + operatorConfig := &configv1alpha1.OperatorConfiguration{ + Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, + Namespace: configv1alpha1.NamespaceConfiguration{Restricted: envtestNamespace}, + } + reconciler := &DynamoGraphDeploymentReconciler{ + Client: k8sClient, + Recorder: events.NewFakeRecorder(100), + Config: operatorConfig, + RuntimeConfig: runtimeConfig, + } + return reconciler +} + +func reconcileCurrentDGDProgram( + ctx context.Context, + reconciler *DynamoGraphDeploymentReconciler, + name string, + namespace string, +) (workloadProgramResult, *nvidiacomv1beta1.DynamoGraphDeployment) { + current := &nvidiacomv1beta1.DynamoGraphDeployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, current)).To(Succeed()) + provider, err := reconciler.ensureWorkloadProvider(ctx, current) + Expect(err).NotTo(HaveOccurred()) + program, err := reconciler.selectWorkloadProgram(provider) + Expect(err).NotTo(HaveOccurred()) + result, err := program.Reconcile(ctx, workloadProgramRequest{DGD: current}) + Expect(err).NotTo(HaveOccurred()) + return result, current +} + +func persistWorkloadProgramStatus( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, + status nvidiacomv1beta1.DynamoGraphDeploymentStatus, +) *nvidiacomv1beta1.DynamoGraphDeployment { + current := &nvidiacomv1beta1.DynamoGraphDeployment{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dgd), current)).To(Succeed()) + current.Status = status + Expect(k8sClient.Status().Update(ctx, current)).To(Succeed()) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dgd), current)).To(Succeed()) + return current +} + +func fetchTypedDisaggregatedSet( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) *disaggregatedsetv1.DisaggregatedSet { + raw := newDisaggregatedSetObject() + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: disaggregatedSetName(dgd), Namespace: dgd.Namespace}, raw)).To(Succeed()) + typed := &disaggregatedsetv1.DisaggregatedSet{} + Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(raw.Object, typed)).To(Succeed()) + return typed +} + +func markDisaggregatedSetReady( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) { + ds := newDisaggregatedSetObject() + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: disaggregatedSetName(dgd), Namespace: dgd.Namespace}, ds)).To(Succeed()) + typedDS := &disaggregatedsetv1.DisaggregatedSet{} + Expect(runtime.DefaultUnstructuredConverter.FromUnstructured(ds.Object, typedDS)).To(Succeed()) + revision := disaggregatedsetutils.ComputeRevision(typedDS.Spec.Roles) + sliceCount := int(disaggregatedsetutils.GetSlices(typedDS)) + + // Simulate LWS v0.10 convergence: each (slice, role, revision) has one ready + // child, and children from the previous revision have been removed. + existing := &leaderworkersetv1.LeaderWorkerSetList{} + Expect(k8sClient.List(ctx, existing, client.InNamespace(ds.GetNamespace()), client.MatchingLabels{ + disaggregatedsetv1.SetNameLabelKey: ds.GetName(), + })).To(Succeed()) + for i := range existing.Items { + Expect(k8sClient.Delete(ctx, &existing.Items[i])).To(Succeed()) + } + + roleStatuses := make([]any, 0, len(typedDS.Spec.Roles)) + for i := range typedDS.Spec.Roles { + role := &typedDS.Spec.Roles[i] + desiredReplicas := ptr.Deref(role.Spec.Replicas, int32(1)) + for slice := range sliceCount { + child := &leaderworkersetv1.LeaderWorkerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: disaggregatedsetutils.GenerateName(ds.GetName(), slice, revision, role.Name), + Namespace: ds.GetNamespace(), + Labels: disaggregatedsetutils.GenerateLabels(ds.GetName(), slice, revision, role.Name), + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: disaggregatedsetv1.GroupVersion.String(), + Kind: "DisaggregatedSet", + Name: ds.GetName(), + UID: ds.GetUID(), + Controller: ptr.To(true), + }}, + }, + Spec: role.Spec, + } + Expect(k8sClient.Create(ctx, child)).To(Succeed()) + child.Status = leaderworkersetv1.LeaderWorkerSetStatus{ + ObservedGeneration: child.Generation, + Replicas: desiredReplicas, + UpdatedReplicas: desiredReplicas, + ReadyReplicas: desiredReplicas, + } + Expect(k8sClient.Status().Update(ctx, child)).To(Succeed()) + } + totalReplicas := int64(desiredReplicas) * int64(sliceCount) + roleStatuses = append(roleStatuses, map[string]any{ + "name": role.Name, "replicas": totalReplicas, "updatedReplicas": totalReplicas, "readyReplicas": totalReplicas, + }) + } + ds.Object["status"] = map[string]any{ + "observedGeneration": ds.GetGeneration(), + "roleStatuses": roleStatuses, + } + Expect(k8sClient.Status().Update(ctx, ds)).To(Succeed()) +} + +func ownedEnvtestCutoverDCDs( + ctx context.Context, + dgd *nvidiacomv1beta1.DynamoGraphDeployment, +) []nvidiacomv1beta1.DynamoComponentDeployment { + list := &nvidiacomv1beta1.DynamoComponentDeploymentList{} + Expect(k8sClient.List(ctx, list, client.InNamespace(dgd.Namespace))).To(Succeed()) + owned := make([]nvidiacomv1beta1.DynamoComponentDeployment, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], dgd) { + owned = append(owned, list.Items[i]) + } + } + return owned +} + +func newEnvtestDSHappyPathDGD(name string) *nvidiacomv1beta1.DynamoGraphDeployment { + return &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: envtestNamespace, + UID: types.UID(fmt.Sprintf("%s-uid", name)), + Annotations: map[string]string{ + consts.KubeAnnotationEnableDisaggregatedSet: consts.KubeLabelValueTrue, + }, + }, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + BackendFramework: "vllm", + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "prefill", + ComponentType: nvidiacomv1beta1.ComponentTypePrefill, + RuntimeVersionOverride: "1.0.0", + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + PodTemplate: envtestDSTestPodTemplate(), + }, + { + ComponentName: "decode", + ComponentType: nvidiacomv1beta1.ComponentTypeDecode, + RuntimeVersionOverride: "1.0.0", + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + PodTemplate: envtestDSTestPodTemplate(), + }, + }, + }, + } +} + +func envtestDSTestPodTemplate() *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: consts.MainContainerName, + Image: "busybox:1.36", + Command: []string{"sh"}, + Args: []string{"-c", "sleep 3600"}, + }}, + }, + } +} diff --git a/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_real_crd_test.go b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_real_crd_test.go new file mode 100644 index 000000000000..c300766ed576 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_real_crd_test.go @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "os/exec" + "path/filepath" + "strings" + "testing" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/events" +) + +func TestDisaggregatedSetRealLWSCRDValidationAndConvergence(t *testing.T) { + t.Log("Start envtest with Dynamo and the pinned real LWS CRDs") + operatorRoot, err := filepath.Abs(filepath.Join("..", "..")) + require.NoError(t, err) + cmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", "sigs.k8s.io/lws") + cmd.Dir = operatorRoot + output, err := cmd.Output() + require.NoError(t, err) + lwsModuleRoot := strings.TrimSpace(string(output)) + + env := operatorenv.New(operatorenv.Options{ + SetupWebhooks: setupProductionWebhooks, + CRDDirectoryPaths: []string{ + filepath.Join(operatorRoot, "config", "crd", "bases"), + filepath.Join(lwsModuleRoot, "config", "crd", "bases"), + }, + }) + testEnv := env.RunT(t) + + t.Log("Create a two-role DisaggregatedSet through the real API schema") + dgd := newEnvtestDSHappyPathDGD("real-crd-convergence") + dgd.Namespace = testEnv.Namespace() + dcds, err := dynamo.GenerateDynamoComponentsDeployments(dgd, nil, nil, dynamo.RollingUpdateContext{}) + require.NoError(t, err) + selection, reason := selectDisaggregatedSetComponents(dgd) + require.Empty(t, reason) + + runtimeConfig := &commoncontroller.RuntimeConfig{ + Gate: features.Gates{LWS: true, DisaggregatedSet: true}, + } + reconciler := &DynamoGraphDeploymentReconciler{ + Client: testEnv.Client(), + Recorder: events.NewFakeRecorder(10), + Config: &configv1alpha1.OperatorConfiguration{}, + RuntimeConfig: runtimeConfig, + } + workloads := reconciler.newDisaggregatedSetWorkloadsReconciler( + newDGDWorkerRolloutReconciler(reconciler.Client, reconciler.Recorder), + ) + desired, err := workloads.generateDisaggregatedSet(t.Context(), dgd, dcds, selection) + require.NoError(t, err) + _, found, err := unstructured.NestedFieldNoCopy(desired.Object, "spec", "slices") + require.NoError(t, err) + require.False(t, found, "DGD does not expose slice cardinality before the grouping API lands") + modified, current, err := workloads.syncDisaggregatedSet(t.Context(), dgd, desired) + require.NoError(t, err) + require.True(t, modified) + slices, found, err := unstructured.NestedInt64(current.Object, "spec", "slices") + require.NoError(t, err) + require.True(t, found) + require.Equal(t, int64(1), slices, "the LWS v0.10 CRD defaults the transitional pathway to one slice") + + t.Log("Reconcile identical desired state after API defaulting") + desired, err = workloads.generateDisaggregatedSet(t.Context(), dgd, dcds, selection) + require.NoError(t, err) + modified, _, err = workloads.syncDisaggregatedSet(t.Context(), dgd, desired) + require.NoError(t, err) + require.False(t, modified, "an identical desired DisaggregatedSet must converge after API defaulting") +} diff --git a/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_test.go b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_test.go new file mode 100644 index 000000000000..48b940a5a6f9 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeployment_disaggregatedset_test.go @@ -0,0 +1,708 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "maps" + "strings" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + disaggregatedsetv1 "sigs.k8s.io/lws/api/disaggregatedset/v1" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" + disaggregatedsetutils "sigs.k8s.io/lws/pkg/utils/disaggregatedset" +) + +func TestDisaggregatedSetEligibilityDoesNotSelectAProvider(t *testing.T) { + dgd := newEnvtestDSHappyPathDGD("selection-eligibility") + tests := []struct { + name string + gate features.Gate + wantReason string + }{ + { + name: "LWS gate supports all eligible roles", + gate: features.Gates{LWS: true}, + wantReason: "", + }, + { + name: "selection validation rejects scaling adapter", + gate: features.Gates{LWS: true}, + wantReason: "scalingAdapter", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantReason != "" { + dgd.Spec.Components[0].ScalingAdapter = &nvidiacomv1beta1.ScalingAdapter{} + } + require.Contains(t, disaggregatedSetEligibilityReason(dgd, tt.gate), tt.wantReason) + }) + } +} + +func TestSyncDisaggregatedSetPreservesUnmanagedMetadata(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) + scheme.AddKnownTypeWithName(disaggregatedSetGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(disaggregatedSetGVK.GroupVersion().WithKind("DisaggregatedSetList"), &unstructured.UnstructuredList{}) + + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "default", UID: "demo-uid"}, + } + current := newDisaggregatedSetObject() + current.SetName(disaggregatedSetName(dgd)) + current.SetNamespace(dgd.Namespace) + current.SetLabels(map[string]string{"example.com/keep": "label"}) + current.SetAnnotations(map[string]string{"example.com/keep": "annotation"}) + current.SetOwnerReferences([]metav1.OwnerReference{ + *dgdControllerOwnerReference(dgd), + {APIVersion: "v1", Kind: "ConfigMap", Name: "keep", UID: "keep-uid"}, + }) + current.Object["spec"] = map[string]any{"roles": []any{}} + desired := current.DeepCopy() + desired.SetLabels(map[string]string{consts.KubeLabelDynamoGraphDeploymentName: dgd.Name}) + desired.SetAnnotations(map[string]string{"example.com/desired": "annotation"}) + desired.SetOwnerReferences([]metav1.OwnerReference{*dgdControllerOwnerReference(dgd)}) + desired.Object["spec"] = map[string]any{"roles": []any{ + map[string]any{"name": "prefill"}, + map[string]any{"name": "decode"}, + }} + k8sClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(dgd, current).Build() + recorder := events.NewFakeRecorder(10) + workloads := newDisaggregatedSetWorkloadsReconciler(k8sClient, recorder, nil, nil, nil, newDGDWorkerRolloutReconciler(k8sClient, recorder)) + + modified, synced, err := workloads.syncDisaggregatedSet(t.Context(), dgd, desired) + require.NoError(t, err) + require.True(t, modified) + require.Equal(t, "label", synced.GetLabels()["example.com/keep"]) + require.Equal(t, "annotation", synced.GetAnnotations()["example.com/keep"]) + require.Equal(t, "annotation", synced.GetAnnotations()["example.com/desired"]) + require.Len(t, synced.GetOwnerReferences(), 2) + persisted := newDisaggregatedSetObject() + require.NoError(t, workloads.Get(t.Context(), client.ObjectKeyFromObject(current), persisted)) + require.Equal(t, "label", persisted.GetLabels()["example.com/keep"]) + require.Equal(t, "annotation", persisted.GetAnnotations()["example.com/desired"]) +} + +func TestDisaggregatedSetServiceSelectorIsRevisionScoped(t *testing.T) { + service := &corev1.Service{} + setDisaggregatedSetServiceSelector(service, "demo-ds", "prefill", "abc12345") + + require.Equal(t, map[string]string{ + disaggregatedsetv1.SetNameLabelKey: "demo-ds", + disaggregatedsetv1.RoleLabelKey: "prefill", + disaggregatedsetv1.RevisionLabelKey: "abc12345", + }, service.Spec.Selector, "the stable component Service must aggregate every slice") +} + +func TestDisaggregatedSetServiceSelectorCutover(t *testing.T) { + existingDCDSelector := map[string]string{consts.KubeLabelDynamoSelector: "demo-prefill"} + existingDSSelector := map[string]string{ + disaggregatedsetv1.SetNameLabelKey: "demo-ds", + disaggregatedsetv1.RoleLabelKey: "prefill", + disaggregatedsetv1.RevisionLabelKey: "old12345", + } + tests := []struct { + name string + hasExisting bool + targetReady bool + existing map[string]string + want map[string]string + }{ + { + name: "DCD selector remains active while the first DS revision is pending", + hasExisting: true, + existing: existingDCDSelector, + want: existingDCDSelector, + }, + { + name: "old DS revision remains active while the target revision is pending", + hasExisting: true, + existing: existingDSSelector, + want: existingDSSelector, + }, + { + name: "a new service selects the target revision immediately", + want: map[string]string{ + disaggregatedsetv1.SetNameLabelKey: "demo-ds", + disaggregatedsetv1.RoleLabelKey: "prefill", + disaggregatedsetv1.RevisionLabelKey: "new12345", + }, + }, + { + name: "a ready target replaces the active selector", + hasExisting: true, + targetReady: true, + existing: existingDCDSelector, + want: map[string]string{ + disaggregatedsetv1.SetNameLabelKey: "demo-ds", + disaggregatedsetv1.RoleLabelKey: "prefill", + disaggregatedsetv1.RevisionLabelKey: "new12345", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + service := &corev1.Service{} + existing := &corev1.Service{Spec: corev1.ServiceSpec{Selector: maps.Clone(tt.existing)}} + + setDesiredDisaggregatedSetServiceSelector( + service, + existing, + tt.hasExisting, + "demo-ds", + "prefill", + "new12345", + tt.targetReady, + ) + + require.Equal(t, tt.want, service.Spec.Selector) + }) + } +} + +func TestDeleteStaleDisaggregatedSetServicesRemovesUndesiredModelService(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) + require.NoError(t, corev1.AddToScheme(scheme)) + + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "default", UID: "demo-uid"}, + } + controlledService := func(name string, labels map[string]string) *corev1.Service { + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: dgd.Namespace, + Labels: labels, + OwnerReferences: []metav1.OwnerReference{*dgdControllerOwnerReference(dgd)}, + }} + } + desiredComponent := controlledService("demo-prefill", map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: dgd.Name, + consts.KubeLabelDynamoComponent: "prefill", + }) + staleComponent := controlledService("demo-removed", map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: dgd.Name, + consts.KubeLabelDynamoComponent: "removed", + }) + staleModel := controlledService(dynamo.GenerateServiceName("removed-model"), map[string]string{ + consts.KubeLabelDynamoBaseModelHash: dynamo.HashModelName("removed-model"), + }) + foreignModel := controlledService(dynamo.GenerateServiceName("foreign-model"), map[string]string{ + consts.KubeLabelDynamoBaseModelHash: dynamo.HashModelName("foreign-model"), + }) + foreignModel.OwnerReferences = nil + + k8sClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(dgd, desiredComponent, staleComponent, staleModel, foreignModel). + Build() + workloads := newDisaggregatedSetWorkloadsReconciler( + k8sClient, + events.NewFakeRecorder(10), + nil, + nil, + nil, + newDGDWorkerRolloutReconciler(k8sClient, events.NewFakeRecorder(10)), + ) + + t.Log("stale component and model services are removed while desired and foreign services remain") + require.NoError(t, workloads.deleteStaleDisaggregatedSetServices(t.Context(), dgd, map[string]struct{}{ + desiredComponent.Name: {}, + })) + for _, name := range []string{staleComponent.Name, staleModel.Name} { + err := k8sClient.Get(t.Context(), client.ObjectKey{Name: name, Namespace: dgd.Namespace}, &corev1.Service{}) + require.True(t, apierrors.IsNotFound(err)) + } + for _, name := range []string{desiredComponent.Name, foreignModel.Name} { + require.NoError(t, k8sClient.Get(t.Context(), client.ObjectKey{Name: name, Namespace: dgd.Namespace}, &corev1.Service{})) + } +} + +func TestDeleteOwnedSelectedDCDsUsesOwnerReference(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, nvidiacomv1beta1.AddToScheme(scheme)) + + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "default", UID: "demo-uid"}, + } + owned := func(name string, labels map[string]string, owner *metav1.OwnerReference) *nvidiacomv1beta1.DynamoComponentDeployment { + var ownerReferences []metav1.OwnerReference + if owner != nil { + ownerReferences = []metav1.OwnerReference{*owner} + } + return &nvidiacomv1beta1.DynamoComponentDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: dgd.Namespace, + Labels: labels, + OwnerReferences: ownerReferences, + }, + Spec: nvidiacomv1beta1.DynamoComponentDeploymentSpec{ + DynamoComponentDeploymentSharedSpec: nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "prefill", + }, + }, + } + } + ownedByDGD := dgdControllerOwnerReference(dgd) + foreignOwner := ownedByDGD.DeepCopy() + foreignOwner.UID = "other-dgd-uid" + selected := disaggregatedSetSelection{componentToRole: map[string]string{"prefill": "prefill"}} + ownedWithoutGraphLabel := owned("owned-without-graph-label", nil, ownedByDGD) + ownedWithStaleGraphLabel := owned("owned-with-stale-graph-label", map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: "other-graph", + }, ownedByDGD) + foreignLabeled := owned("foreign-labeled", map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: dgd.Name, + }, foreignOwner) + foreignUnlabeled := owned("foreign-unlabeled", nil, foreignOwner) + + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(dgd, ownedWithoutGraphLabel, ownedWithStaleGraphLabel, foreignLabeled, foreignUnlabeled). + Build() + workloads := newDisaggregatedSetWorkloadsReconciler( + kubeClient, + events.NewFakeRecorder(10), + nil, + nil, + nil, + newDGDWorkerRolloutReconciler(kubeClient, events.NewFakeRecorder(10)), + ) + + require.NoError(t, workloads.deleteOwnedSelectedDCDs(t.Context(), dgd, selected)) + for _, name := range []string{ownedWithoutGraphLabel.Name, ownedWithStaleGraphLabel.Name} { + err := kubeClient.Get(t.Context(), client.ObjectKey{Name: name, Namespace: dgd.Namespace}, &nvidiacomv1beta1.DynamoComponentDeployment{}) + require.True(t, apierrors.IsNotFound(err), "owned selected DCD %s should be deleted", name) + } + for _, name := range []string{foreignLabeled.Name, foreignUnlabeled.Name} { + require.NoError(t, kubeClient.Get(t.Context(), client.ObjectKey{Name: name, Namespace: dgd.Namespace}, &nvidiacomv1beta1.DynamoComponentDeployment{})) + } +} + +func TestSelectDisaggregatedSetComponents(t *testing.T) { + t.Run("selects multinode worker roles", func(t *testing.T) { + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "prefill", + ComponentType: nvidiacomv1beta1.ComponentTypePrefill, + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + Replicas: ptr.To(int32(2)), + }, + { + ComponentName: "decode", + ComponentType: nvidiacomv1beta1.ComponentTypeDecode, + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + Replicas: ptr.To(int32(2)), + }, + { + ComponentName: "frontend", + ComponentType: nvidiacomv1beta1.ComponentTypeFrontend, + }, + }, + }, + } + + selection, reason := selectDisaggregatedSetComponents(dgd) + require.Empty(t, reason) + require.Equal(t, "prefill", selection.componentToRole["prefill"]) + require.Equal(t, "decode", selection.componentToRole["decode"]) + require.Len(t, selection.componentToRole, 2) + }) + + t.Run("rejects scaling adapter", func(t *testing.T) { + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: "prefill", + ComponentType: nvidiacomv1beta1.ComponentTypePrefill, + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + ScalingAdapter: &nvidiacomv1beta1.ScalingAdapter{}, + Replicas: ptr.To(int32(2)), + }, + { + ComponentName: "decode", + ComponentType: nvidiacomv1beta1.ComponentTypeDecode, + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + Replicas: ptr.To(int32(2)), + }, + }, + }, + } + + _, reason := selectDisaggregatedSetComponents(dgd) + require.Contains(t, reason, "scalingAdapter") + }) +} + +func TestDisaggregatedSetChildNamesFitDNSLabelLimit(t *testing.T) { + dgd := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: strings.Repeat("d", 63)}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentSpec{ + Components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + { + ComponentName: strings.Repeat("p", 63), + ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + }, + { + ComponentName: strings.Repeat("q", 63), + ComponentType: nvidiacomv1beta1.ComponentTypeWorker, + Multinode: &nvidiacomv1beta1.MultinodeSpec{NodeCount: 2}, + }, + }, + }, + } + + selection, reason := selectDisaggregatedSetComponents(dgd) + require.Empty(t, reason) + require.Len(t, selection.componentToRole, 2) + setName := disaggregatedSetName(dgd) + require.LessOrEqual(t, len(setName), maxDisaggregatedSetNameLength) + for _, roleName := range selection.componentToRole { + require.LessOrEqual(t, len(roleName), maxDisaggregatedSetRoleNameLength) + childName := disaggregatedsetutils.GenerateName(setName, 99, strings.Repeat("a", disaggregatedSetRevisionLength), roleName) + require.LessOrEqual(t, len(childName), 63) + serviceName := childName + "-prv" + require.LessOrEqual(t, len(serviceName), 63) + } +} + +func TestCheckDisaggregatedSetReadiness(t *testing.T) { + ds := newDisaggregatedSetObject() + ds.SetName("demo") + ds.SetGeneration(3) + ds.Object["status"] = map[string]any{ + "observedGeneration": int64(2), + "roleStatuses": []any{ + map[string]any{"name": "prefill", "replicas": int64(2), "updatedReplicas": int64(2), "readyReplicas": int64(2)}, + map[string]any{"name": "decode", "replicas": int64(2), "updatedReplicas": int64(2), "readyReplicas": int64(1)}, + }, + } + selection := disaggregatedSetSelection{ + componentToRole: map[string]string{"prefill": "prefill", "decode": "decode"}, + desiredReplicas: map[string]int32{"prefill": 2, "decode": 2}, + } + + t.Log("stale observedGeneration keeps the DisaggregatedSet unready") + ready, reason, statuses := checkDisaggregatedSetReadiness(ds, selection) + require.False(t, ready) + require.Contains(t, reason, "observed generation") + require.Equal(t, int32(2), ptr.Deref(statuses["prefill"].ReadyReplicas, 0)) + + t.Log("lagging decode role readiness keeps the DisaggregatedSet unready") + ds.Object["status"].(map[string]any)["observedGeneration"] = int64(3) + ready, reason, statuses = checkDisaggregatedSetReadiness(ds, selection) + require.False(t, ready) + require.Contains(t, reason, "decode") + require.Equal(t, int32(1), ptr.Deref(statuses["decode"].ReadyReplicas, 0)) + + t.Log("all roles at desired ready replicas report ready") + ds.Object["status"].(map[string]any)["roleStatuses"] = []any{ + map[string]any{"name": "prefill", "replicas": int64(2), "updatedReplicas": int64(2), "readyReplicas": int64(2)}, + map[string]any{"name": "decode", "replicas": int64(2), "updatedReplicas": int64(2), "readyReplicas": int64(2)}, + } + ready, _, _ = checkDisaggregatedSetReadiness(ds, selection) + require.True(t, ready) +} + +func TestCheckDisaggregatedSetChildLWSReadinessWaitsForRemovedRoles(t *testing.T) { + selection := disaggregatedSetSelection{ + componentToRole: map[string]string{"prefill": "prefill", "decode": "decode"}, + desiredReplicas: map[string]int32{"prefill": 1, "decode": 1}, + } + readyChild := func(name string) *leaderworkersetv1.LeaderWorkerSet { + return &leaderworkersetv1.LeaderWorkerSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Generation: 1}, + Spec: leaderworkersetv1.LeaderWorkerSetSpec{Replicas: ptr.To[int32](1)}, + Status: leaderworkersetv1.LeaderWorkerSetStatus{ + ObservedGeneration: 1, + Replicas: 1, + UpdatedReplicas: 1, + ReadyReplicas: 1, + }, + } + } + prefill := readyChild("demo-prefill-target") + decode := readyChild("demo-decode-target") + removed := readyChild("demo-legacy-worker-old") + targetByIdentity := map[disaggregatedSetChildIdentity][]*leaderworkersetv1.LeaderWorkerSet{ + {slice: 0, role: "prefill"}: {prefill}, + {slice: 0, role: "decode"}: {decode}, + } + childrenByRole := map[string][]*leaderworkersetv1.LeaderWorkerSet{ + "prefill": {prefill}, + "decode": {decode}, + "legacy-worker": {removed}, + } + + t.Log("a removed role with live replicas keeps the DisaggregatedSet unready") + ready, reason, _ := checkDisaggregatedSetChildLWSReadiness(selection, 1, targetByIdentity, childrenByRole) + require.False(t, ready) + require.Contains(t, reason, removed.Name) + + t.Log("the target becomes ready after the removed role is fully drained") + removed.Spec.Replicas = ptr.To[int32](0) + removed.Status.Replicas = 0 + removed.Status.UpdatedReplicas = 0 + removed.Status.ReadyReplicas = 0 + ready, _, _ = checkDisaggregatedSetChildLWSReadiness(selection, 1, targetByIdentity, childrenByRole) + require.True(t, ready) +} + +func TestCheckDisaggregatedSetReadinessTracksEverySlice(t *testing.T) { + typedDS := &disaggregatedsetv1.DisaggregatedSet{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: testNamespace, UID: "demo-uid"}, + Spec: disaggregatedsetv1.DisaggregatedSetSpec{ + Slices: ptr.To[int32](2), + Roles: []disaggregatedsetv1.DisaggregatedRoleSpec{ + {Name: "prefill"}, + {Name: "decode"}, + }, + }, + } + dsObject, err := runtime.DefaultUnstructuredConverter.ToUnstructured(typedDS) + require.NoError(t, err) + ds := newDisaggregatedSetObject() + ds.Object = dsObject + ds.SetGroupVersionKind(disaggregatedSetGVK) + revision := disaggregatedsetutils.ComputeRevision(typedDS.Spec.Roles) + + readyChild := func(slice int, role string) *leaderworkersetv1.LeaderWorkerSet { + return &leaderworkersetv1.LeaderWorkerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: disaggregatedsetutils.GenerateName(ds.GetName(), slice, revision, role), + Namespace: ds.GetNamespace(), + Generation: 1, + Labels: disaggregatedsetutils.GenerateLabels(ds.GetName(), slice, revision, role), + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: disaggregatedsetv1.GroupVersion.String(), + Kind: "DisaggregatedSet", + Name: ds.GetName(), + UID: ds.GetUID(), + Controller: ptr.To(true), + }}, + }, + Spec: leaderworkersetv1.LeaderWorkerSetSpec{Replicas: ptr.To[int32](1)}, + Status: leaderworkersetv1.LeaderWorkerSetStatus{ + ObservedGeneration: 1, + Replicas: 1, + UpdatedReplicas: 1, + ReadyReplicas: 1, + }, + } + } + children := []client.Object{ + readyChild(0, "prefill"), + readyChild(0, "decode"), + readyChild(1, "prefill"), + readyChild(1, "decode"), + } + // A label-less pre-v0.10 child is slice 0 during an in-place upgrade. + delete(children[0].GetLabels(), disaggregatedsetv1.SliceLabelKey) + + scheme := runtime.NewScheme() + require.NoError(t, leaderworkersetv1.AddToScheme(scheme)) + workloads := &disaggregatedSetWorkloadsReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(children...).Build(), + } + selection := disaggregatedSetSelection{ + componentToRole: map[string]string{"prefill": "prefill", "decode": "decode"}, + desiredReplicas: map[string]int32{"prefill": 1, "decode": 1}, + } + + ready, reason, statuses, err := workloads.checkDisaggregatedSetReadiness(t.Context(), ds, selection) + require.NoError(t, err) + require.True(t, ready, reason) + require.Equal(t, int32(2), statuses["prefill"].Replicas) + require.Equal(t, int32(2), statuses["prefill"].UpdatedReplicas) + require.Equal(t, int32(2), ptr.Deref(statuses["prefill"].ReadyReplicas, 0)) + + t.Log("legacy and slice-aware children cannot both claim slice 0") + duplicate := readyChild(0, "prefill") + duplicate.Name += "-duplicate" + require.NoError(t, workloads.Create(t.Context(), duplicate)) + ready, reason, _, err = workloads.checkDisaggregatedSetReadiness(t.Context(), ds, selection) + require.NoError(t, err) + require.False(t, ready) + require.Contains(t, reason, "2 target LeaderWorkerSets") + require.NoError(t, workloads.Delete(t.Context(), duplicate)) + + t.Log("one missing slice role cannot be hidden by aggregate replica counts") + missing := children[3].(*leaderworkersetv1.LeaderWorkerSet) + require.NoError(t, workloads.Delete(t.Context(), missing)) + ready, reason, _, err = workloads.checkDisaggregatedSetReadiness(t.Context(), ds, selection) + require.NoError(t, err) + require.False(t, ready) + require.Contains(t, reason, "slice 1") +} + +func TestDisaggregatedSetWatchMapperMapsNonzeroSlice(t *testing.T) { + ds := newDisaggregatedSetObject() + ds.SetName("demo-ds") + ds.SetNamespace(testNamespace) + ds.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta1.GroupVersion.String(), + Kind: dynamoGraphDeploymentKind, + Name: "demo", + Controller: ptr.To(true), + }}) + scheme := runtime.NewScheme() + scheme.AddKnownTypeWithName(disaggregatedSetGVK, &unstructured.Unstructured{}) + scheme.AddKnownTypeWithName(disaggregatedSetGVK.GroupVersion().WithKind("DisaggregatedSetList"), &unstructured.UnstructuredList{}) + mapper := newDisaggregatedSetWatchMapper(fake.NewClientBuilder().WithScheme(scheme).WithObjects(ds).Build()) + child := &leaderworkersetv1.LeaderWorkerSet{ObjectMeta: metav1.ObjectMeta{ + Name: "demo-ds-1-abc12345-prefill", + Namespace: testNamespace, + Labels: map[string]string{ + disaggregatedsetv1.SetNameLabelKey: "demo-ds", + disaggregatedsetv1.SliceLabelKey: "1", + }, + }} + + require.Equal(t, []ctrl.Request{{NamespacedName: types.NamespacedName{ + Name: "demo", Namespace: testNamespace, + }}}, mapper.MapChildLWSToDGD(t.Context(), child)) +} + +func TestDisaggregatedSetStatusReadinessWaitsForRemovedRoleChildren(t *testing.T) { + ds := newDisaggregatedSetObject() + ds.SetName("demo") + ds.SetNamespace(testNamespace) + ds.SetUID("demo-uid") + ds.SetGeneration(2) + ds.Object["status"] = map[string]any{ + "observedGeneration": int64(2), + "roleStatuses": []any{ + map[string]any{"name": "prefill", "replicas": int64(1), "updatedReplicas": int64(1), "readyReplicas": int64(1)}, + map[string]any{"name": "decode", "replicas": int64(1), "updatedReplicas": int64(1), "readyReplicas": int64(1)}, + }, + } + removed := &leaderworkersetv1.LeaderWorkerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-old-legacy-worker", + Namespace: testNamespace, + Labels: map[string]string{ + disaggregatedsetv1.SetNameLabelKey: "demo", + disaggregatedsetv1.RoleLabelKey: "legacy-worker", + }, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: disaggregatedsetv1.GroupVersion.String(), + Kind: "DisaggregatedSet", + Name: ds.GetName(), + UID: ds.GetUID(), + Controller: ptr.To(true), + }}, + }, + Spec: leaderworkersetv1.LeaderWorkerSetSpec{Replicas: ptr.To[int32](1)}, + Status: leaderworkersetv1.LeaderWorkerSetStatus{Replicas: 1, ReadyReplicas: 1}, + } + scheme := runtime.NewScheme() + require.NoError(t, leaderworkersetv1.AddToScheme(scheme)) + workloads := &disaggregatedSetWorkloadsReconciler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(removed).Build()} + selection := disaggregatedSetSelection{ + componentToRole: map[string]string{"prefill": "prefill", "decode": "decode"}, + desiredReplicas: map[string]int32{"prefill": 1, "decode": 1}, + } + + ready, reason, _, err := workloads.checkDisaggregatedSetReadiness(t.Context(), ds, selection) + require.NoError(t, err) + require.False(t, ready) + require.Contains(t, reason, removed.Name) +} + +func TestDisaggregatedSetPredicatesObserveRoutingMetadata(t *testing.T) { + baseDS := newDisaggregatedSetObject() + baseDS.SetLabels(map[string]string{consts.KubeLabelDynamoGraphDeploymentName: "demo"}) + relabeledDS := baseDS.DeepCopy() + relabeledDS.SetLabels(map[string]string{consts.KubeLabelDynamoGraphDeploymentName: "other"}) + reownedDS := baseDS.DeepCopy() + reownedDS.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta1.GroupVersion.String(), + Kind: dynamoGraphDeploymentKind, + Name: "demo", + UID: "demo-uid", + Controller: ptr.To(true), + }}) + statusUpdatedDS := baseDS.DeepCopy() + statusUpdatedDS.Object["status"] = map[string]any{"observedGeneration": int64(1)} + + require.False(t, disaggregatedSetStatusChanged(baseDS, baseDS.DeepCopy())) + require.True(t, disaggregatedSetStatusChanged(baseDS, relabeledDS)) + require.True(t, disaggregatedSetStatusChanged(baseDS, reownedDS)) + require.True(t, disaggregatedSetStatusChanged(baseDS, statusUpdatedDS)) + + baseLWS := &leaderworkersetv1.LeaderWorkerSet{ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: "demo", + }, + }} + relabeledLWS := baseLWS.DeepCopy() + relabeledLWS.SetLabels(map[string]string{ + consts.KubeLabelDynamoGraphDeploymentName: "other", + }) + statusUpdatedLWS := baseLWS.DeepCopy() + statusUpdatedLWS.Status.Conditions = []metav1.Condition{{Type: "Available", Status: metav1.ConditionTrue}} + + require.False(t, leaderWorkerSetStatusChanged(baseLWS, baseLWS.DeepCopy())) + require.True(t, leaderWorkerSetStatusChanged(baseLWS, relabeledLWS)) + require.True(t, leaderWorkerSetStatusChanged(baseLWS, statusUpdatedLWS)) +} + +func TestWorkloadRoutingAnnotationsChanged(t *testing.T) { + t.Log("no change in routing annotations does not trigger update predicate") + oldDGD := &nvidiacomv1beta1.DynamoGraphDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + consts.KubeAnnotationEnableGrove: consts.KubeLabelValueFalse, + consts.KubeAnnotationEnableDisaggregatedSet: consts.KubeLabelValueFalse, + }, + }, + } + newDGD := oldDGD.DeepCopy() + require.False(t, workloadRoutingAnnotationsChanged(updateEvent(oldDGD, newDGD))) + + t.Log("enabling DisaggregatedSet triggers update predicate") + newDGD = oldDGD.DeepCopy() + newDGD.Annotations[consts.KubeAnnotationEnableDisaggregatedSet] = consts.KubeLabelValueTrue + require.True(t, workloadRoutingAnnotationsChanged(updateEvent(oldDGD, newDGD))) + + t.Log("disabling DisaggregatedSet triggers update predicate") + oldDGD = newDGD.DeepCopy() + newDGD = oldDGD.DeepCopy() + newDGD.Annotations[consts.KubeAnnotationEnableDisaggregatedSet] = consts.KubeLabelValueFalse + require.True(t, workloadRoutingAnnotationsChanged(updateEvent(oldDGD, newDGD))) + + t.Log("removing DisaggregatedSet triggers update predicate") + newDGD = oldDGD.DeepCopy() + delete(newDGD.Annotations, consts.KubeAnnotationEnableDisaggregatedSet) + require.True(t, workloadRoutingAnnotationsChanged(updateEvent(oldDGD, newDGD))) +} + +func updateEvent(oldObj, newObj *nvidiacomv1beta1.DynamoGraphDeployment) event.UpdateEvent { + return event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObj} +} diff --git a/deploy/operator/internal/controller/testing/disaggregatedset/disaggregatedsets.yaml b/deploy/operator/internal/controller/testing/disaggregatedset/disaggregatedsets.yaml new file mode 100644 index 000000000000..f1bc416a0f7f --- /dev/null +++ b/deploy/operator/internal/controller/testing/disaggregatedset/disaggregatedsets.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Minimal DisaggregatedSet CRD used by envtest. The real schema is owned by +# the upstream LWS project; this fixture declares only the fields the operator +# reads and writes during controller tests. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: disaggregatedsets.disaggregatedset.x-k8s.io +spec: + group: disaggregatedset.x-k8s.io + scope: Namespaced + names: + plural: disaggregatedsets + singular: disaggregatedset + kind: DisaggregatedSet + listKind: DisaggregatedSetList + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/operator/internal/controller/testing/disaggregatedset/leaderworkersets.yaml b/deploy/operator/internal/controller/testing/disaggregatedset/leaderworkersets.yaml new file mode 100644 index 000000000000..24ae3070743d --- /dev/null +++ b/deploy/operator/internal/controller/testing/disaggregatedset/leaderworkersets.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: leaderworkersets.leaderworkerset.x-k8s.io +spec: + group: leaderworkerset.x-k8s.io + names: + kind: LeaderWorkerSet + listKind: LeaderWorkerSetList + plural: leaderworkersets + singular: leaderworkerset + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index db8924998e84..d78c9b7b467d 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -294,11 +294,30 @@ func GenerateDynamoComponentsDeployments( existingRestartAnnotations map[string]string, rollingUpdateCtx RollingUpdateContext, ) (map[string]*v1beta1.DynamoComponentDeployment, error) { - deployments := make(map[string]*v1beta1.DynamoComponentDeployment) - backendFramework, err := backendFrameworkForGeneratedDCDs(parentDGD) + components, err := NormalizeDynamoGraphDeploymentComponents( + parentDGD, + restartState, + existingRestartAnnotations, + rollingUpdateCtx, + ) if err != nil { return nil, err } + return GenerateDynamoComponentsDeploymentsFromNormalized(parentDGD, components, rollingUpdateCtx) +} + +// NormalizeDynamoGraphDeploymentComponents applies the component-level +// defaults and rollout mutations shared by all workload lowerings. +func NormalizeDynamoGraphDeploymentComponents( + parentDGD *v1beta1.DynamoGraphDeployment, + restartState *RestartState, + existingRestartAnnotations map[string]string, + rollingUpdateCtx RollingUpdateContext, +) (map[string]*v1beta1.DynamoComponentDeploymentSharedSpec, error) { + components := make(map[string]*v1beta1.DynamoComponentDeploymentSharedSpec) + if _, err := backendFrameworkForGeneratedDCDs(parentDGD); err != nil { + return nil, err + } // Generate DCDs for each component. for i := range parentDGD.Spec.Components { @@ -317,14 +336,55 @@ func GenerateDynamoComponentsDeployments( } } + normalized, err := normalizeDGDComponent( + parentDGD, + componentName, + component, + restartState, + existingRestartAnnotations, + rollingUpdateCtx, + ) + if err != nil { + return nil, err + } + components[componentName] = normalized + } + + return components, nil +} + +// GenerateDynamoComponentsDeploymentsFromNormalized lowers the supplied +// normalized component specs into real DCD API objects. Callers that own a +// different provider can lower only the components that remain on the DCD +// path without first materializing DCDs for their own components. +func GenerateDynamoComponentsDeploymentsFromNormalized( + parentDGD *v1beta1.DynamoGraphDeployment, + components map[string]*v1beta1.DynamoComponentDeploymentSharedSpec, + rollingUpdateCtx RollingUpdateContext, +) (map[string]*v1beta1.DynamoComponentDeployment, error) { + deployments := make(map[string]*v1beta1.DynamoComponentDeployment, len(components)) + backendFramework, err := backendFrameworkForGeneratedDCDs(parentDGD) + if err != nil { + return nil, err + } + for componentName, component := range components { + if component == nil { + return nil, fmt.Errorf("normalized component %q is nil", componentName) + } dynamoNamespace := parentDGD.GetDynamoNamespaceForComponent(component) - dcd, err := generateSingleDCD(parentDGD, componentName, component, dynamoNamespace, backendFramework, restartState, existingRestartAnnotations, rollingUpdateCtx) + dcd, err := generateSingleDCDFromNormalized( + parentDGD, + componentName, + component, + dynamoNamespace, + backendFramework, + rollingUpdateCtx, + ) if err != nil { return nil, err } deployments[componentName] = dcd } - return deployments, nil } @@ -413,16 +473,68 @@ func GetDynamoNamespace(object metav1.Object, service *v1beta1.DynamoComponentDe return v1beta1.ComputeDynamoNamespace(service.GlobalDynamoNamespace, object.GetNamespace(), object.GetName()) } -// generateSingleDCD creates a DynamoComponentDeployment for a single service. -func generateSingleDCD( +func normalizeDGDComponent( parentDGD *v1beta1.DynamoGraphDeployment, componentName string, component *v1beta1.DynamoComponentDeploymentSharedSpec, - dynamoNamespace string, - backendFramework string, restartState *RestartState, existingRestartAnnotations map[string]string, rollingUpdateCtx RollingUpdateContext, +) (*v1beta1.DynamoComponentDeploymentSharedSpec, error) { + normalized := component.DeepCopy() + + // Keep the EPP mTLS safety default in the component input before graph-wide + // metadata is merged so a graph annotation cannot override it. + if normalized.ComponentType == commonconsts.ComponentTypeEPP { + podTemplate := ensurePodTemplate(normalized) + if _, exists := podTemplate.Annotations[commonconsts.KubeAnnotationIstioSidecarInject]; !exists { + podTemplate.Annotations[commonconsts.KubeAnnotationIstioSidecarInject] = "false" + } + } + + applyDGDTemplateDefaults(normalized, parentDGD, nil) + + if IsWorkerComponent(string(normalized.ComponentType)) { + podTemplate := ensurePodTemplate(normalized) + podTemplate.Labels[commonconsts.KubeLabelDynamoWorkerHash] = rollingUpdateCtx.NewWorkerHash + if parentDGD.HasEPPComponent() { + podTemplate.Labels[commonconsts.KubeLabelDynamoComponentClass] = commonconsts.ComponentClassWorker + } + } + + if restartState.ShouldAnnotateComponent(componentName) { + ensurePodTemplate(normalized).Annotations[commonconsts.RestartAnnotation] = restartState.Timestamp + } else if existingRestartAt := existingRestartAnnotations[componentName]; existingRestartAt != "" { + ensurePodTemplate(normalized).Annotations[commonconsts.RestartAnnotation] = existingRestartAt + } + + if normalized.ComponentType == commonconsts.ComponentTypePlanner { + ensurePodTemplate(normalized).Spec.ServiceAccountName = commonconsts.PlannerServiceAccountName + } + + if err := applyDynDeploymentConfigToComponent( + normalized, + componentName, + normalized.ComponentType == commonconsts.ComponentTypeFrontend, + commonconsts.DynamoServicePort, + ); err != nil { + return nil, err + } + + if newReplicas, ok := rollingUpdateCtx.NewWorkerReplicaTargetsByComponent[componentName]; rollingUpdateCtx.InProgress() && IsWorkerComponent(string(normalized.ComponentType)) && ok { + normalized.Replicas = ptr.To(newReplicas) + } + + return normalized, nil +} + +func generateSingleDCDFromNormalized( + parentDGD *v1beta1.DynamoGraphDeployment, + componentName string, + component *v1beta1.DynamoComponentDeploymentSharedSpec, + dynamoNamespace string, + backendFramework string, + rollingUpdateCtx RollingUpdateContext, ) (*v1beta1.DynamoComponentDeployment, error) { deployment := &v1beta1.DynamoComponentDeployment{} deployment.Spec.DynamoComponentDeploymentSharedSpec = *component.DeepCopy() @@ -456,31 +568,6 @@ func generateSingleDCD( } } - // Stamp sidecar.istio.io/inject: "false" on EPP pod templates before - // DGD-level annotations are merged in. EPP serves its own TLS on port 9002 - // (--secure-serving true); an Istio sidecar intercepting that port causes a - // double-TLS handshake failure when the namespace has STRICT mTLS, which - // surfaces as cx_connect_fail / HTTP 500 on the gateway. - // - // Placement before applyDGDTemplateDefaults is intentional: the merge - // function (mergeLowPriorityMetadata) does not overwrite keys already - // present in the destination map, so a graph-wide DGD Spec.Annotations - // entry of sidecar.istio.io/inject: "true" cannot silently bypass the - // EPP opt-out. An explicit per-EPP podTemplate annotation set by the user - // is preserved by the !exists guard. - if component.ComponentType == commonconsts.ComponentTypeEPP { - podTemplate := ensurePodTemplate(&deployment.Spec.DynamoComponentDeploymentSharedSpec) - if _, exists := podTemplate.Annotations[commonconsts.KubeAnnotationIstioSidecarInject]; !exists { - podTemplate.Annotations[commonconsts.KubeAnnotationIstioSidecarInject] = "false" - } - } - - applyDGDTemplateDefaults( - &deployment.Spec.DynamoComponentDeploymentSharedSpec, - parentDGD, - nil, // no topology domains for DCDs (only applies for Grove pathway) - ) - // Topology label controller marker: set on the DCD so it propagates to pods. if shouldApplyKvTransferPolicyToWorkerComponent(component, parentDGD) { if deployment.Annotations == nil { @@ -489,29 +576,7 @@ func generateSingleDCD( applyKvTransferPolicyTopologyAnnotations(deployment.Annotations, parentDGD.Spec.Experimental.KvTransferPolicy) } - // Apply restart annotation if this component should be restarted. - if restartState.ShouldAnnotateComponent(componentName) { - podTemplate := ensurePodTemplate(&deployment.Spec.DynamoComponentDeploymentSharedSpec) - podTemplate.Annotations[commonconsts.RestartAnnotation] = restartState.Timestamp - } else if existingRestartAnnotations != nil { - if existingRestartAt, ok := existingRestartAnnotations[componentName]; ok && existingRestartAt != "" { - podTemplate := ensurePodTemplate(&deployment.Spec.DynamoComponentDeploymentSharedSpec) - podTemplate.Annotations[commonconsts.RestartAnnotation] = existingRestartAt - } - } - - if component.ComponentType == commonconsts.ComponentTypePlanner { - ensurePodTemplate(&deployment.Spec.DynamoComponentDeploymentSharedSpec).Spec.ServiceAccountName = commonconsts.PlannerServiceAccountName - } - - if err := applyDynDeploymentConfig(deployment, commonconsts.DynamoServicePort); err != nil { - return nil, err - } - - // during a rolling update, the replica count is determined by the rollingUpdateCtx instead of the component spec - if newReplicas, ok := rollingUpdateCtx.NewWorkerReplicaTargetsByComponent[componentName]; rollingUpdateCtx.InProgress() && IsWorkerComponent(string(component.ComponentType)) && ok { - deployment.Spec.Replicas = ptr.To(newReplicas) - } else if component.Replicas != nil { + if component.Replicas != nil { deployment.Spec.Replicas = component.Replicas } @@ -590,7 +655,21 @@ func GetDGDComponentPreservedIngressSpec(dgd *v1beta1.DynamoGraphDeployment, com } func applyDynDeploymentConfig(dcd *v1beta1.DynamoComponentDeployment, frontendPort int) error { - main := GetMainContainer(&dcd.Spec.DynamoComponentDeploymentSharedSpec) + return applyDynDeploymentConfigToComponent( + &dcd.Spec.DynamoComponentDeploymentSharedSpec, + GetDCDComponentName(dcd), + dcd.IsFrontendComponent(), + frontendPort, + ) +} + +func applyDynDeploymentConfigToComponent( + component *v1beta1.DynamoComponentDeploymentSharedSpec, + componentName string, + isFrontend bool, + frontendPort int, +) error { + main := GetMainContainer(component) if main == nil { return nil } @@ -598,8 +677,7 @@ func applyDynDeploymentConfig(dcd *v1beta1.DynamoComponentDeployment, frontendPo if rawConfig == nil { return nil } - componentName := GetDCDComponentName(dcd) - if dcd.IsFrontendComponent() { + if isFrontend { updatedConfig, err := updateDynDeploymentConfigBytes(rawConfig, componentName, frontendPort) if err != nil { return err @@ -611,12 +689,12 @@ func applyDynDeploymentConfig(dcd *v1beta1.DynamoComponentDeployment, frontendPo if err != nil { return err } - serviceConfig := getDynDeploymentServiceConfig(config, componentName, dcd.IsFrontendComponent()) + serviceConfig := getDynDeploymentServiceConfig(config, componentName, isFrontend) if serviceConfig == nil || serviceConfig.ServiceArgs == nil { return nil } - if dcd.Spec.Replicas == nil && serviceConfig.ServiceArgs.Workers != nil { - dcd.Spec.Replicas = serviceConfig.ServiceArgs.Workers + if component.Replicas == nil && serviceConfig.ServiceArgs.Workers != nil { + component.Replicas = serviceConfig.ServiceArgs.Workers } return applyDynDeploymentResources(main, serviceConfig.ServiceArgs.Resources) } diff --git a/deploy/operator/internal/dynamo/model_service.go b/deploy/operator/internal/dynamo/model_service.go index 6e16cea8aa82..984ba497577f 100644 --- a/deploy/operator/internal/dynamo/model_service.go +++ b/deploy/operator/internal/dynamo/model_service.go @@ -124,6 +124,20 @@ func ReconcileModelServicesForComponents( return nil } +// GenerateModelServiceForModel returns the stable headless Service for a model. +// Controllers that need to compose the complete desired Service before writing +// it can use this without going through the legacy sync-and-then-patch path. +func GenerateModelServiceForModel(namespace, baseModelName string, annotations map[string]string) *corev1.Service { + return generateHeadlessServiceForModel(namespace, baseModelName, annotations) +} + +// GenerateModelServiceForGraph returns a model Service private to one graph. +// Graph-scoped callers must use this identity because the Service is owned by +// the graph deployment. The legacy model-only helper remains for DCD/Grove. +func GenerateModelServiceForGraph(namespace, baseModelName, graphName string, annotations map[string]string) *corev1.Service { + return generateHeadlessServiceForGraph(namespace, baseModelName, graphName, annotations) +} + // GenerateHeadlessServiceForModel creates a headless service for model endpoint discovery // Service name is generated deterministically from the base model name using a hash // The base model name hash is stored as a label for efficient discovery @@ -185,6 +199,16 @@ func generateHeadlessServiceForModel( return service } +func generateHeadlessServiceForGraph(namespace, baseModelName, graphName string, annotations map[string]string) *corev1.Service { + service := generateHeadlessServiceForModel(namespace, baseModelName, annotations) + modelHash := HashModelName(baseModelName) + graphHash := HashGraphName(graphName) + service.Name = fmt.Sprintf("dynamo-model-%s-%s", modelHash, graphHash) + service.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = graphName + service.Spec.Selector[commonconsts.KubeLabelDynamoGraphDeploymentName] = graphName + return service +} + // HashModelName creates a deterministic hash from a base model name for use in labels // Returns an 8-character hex string (always valid as a Kubernetes label value) func HashModelName(baseModelName string) string { @@ -193,6 +217,12 @@ func HashModelName(baseModelName string) string { return hex.EncodeToString(hash[:])[:8] } +// HashGraphName creates a deterministic, label-safe identity for a graph. +func HashGraphName(graphName string) string { + hash := sha256.Sum256([]byte(graphName)) + return hex.EncodeToString(hash[:])[:8] +} + // GenerateServiceName creates a deterministic, DNS-safe service name from a base model name // Format: dynamo-model-{8-char-hash} func GenerateServiceName(baseModelName string) string { diff --git a/deploy/operator/internal/dynamo/model_service_test.go b/deploy/operator/internal/dynamo/model_service_test.go new file mode 100644 index 000000000000..bcb62cdaac81 --- /dev/null +++ b/deploy/operator/internal/dynamo/model_service_test.go @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dynamo + +import ( + "context" + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type modelServiceTestReconciler struct { + client.Client + recorder events.EventRecorder +} + +func (r *modelServiceTestReconciler) GetRecorder() events.EventRecorder { + return r.recorder +} + +func TestModelServiceReconciliationCharacterization(t *testing.T) { + t.Log("Arrange a DGD with two components sharing one model reference") + dgd := &v1beta1.DynamoGraphDeployment{ + TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.GroupVersion.String(), Kind: "DynamoGraphDeployment"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "graph", + Namespace: "inference", + UID: "graph-uid", + }, + Spec: v1beta1.DynamoGraphDeploymentSpec{ + Annotations: map[string]string{"example.com/graph": "true"}, + }, + } + components := map[string]*v1beta1.DynamoComponentDeploymentSharedSpec{ + "prefill": {ComponentName: "prefill", ModelRef: &v1beta1.ModelReference{Name: "llama-3"}}, + "decode": {ComponentName: "decode", ModelRef: &v1beta1.ModelReference{Name: "llama-3"}}, + } + s := scheme.Scheme + require.NoError(t, v1beta1.AddToScheme(s)) + reconciler := &modelServiceTestReconciler{ + Client: fake.NewClientBuilder().WithScheme(s).WithObjects(dgd).Build(), + recorder: events.NewFakeRecorder(10), + } + + t.Log("Act through the existing model-service reconciliation API") + require.NoError(t, ReconcileModelServicesForComponents( + context.Background(), reconciler, dgd, components, dgd.Namespace, + )) + + t.Log("Assert one deterministic headless Service with the current graph contract") + service := &corev1.Service{} + require.NoError(t, reconciler.Get(context.Background(), client.ObjectKey{ + Namespace: dgd.Namespace, + Name: GenerateServiceName("llama-3"), + }, service)) + require.Equal(t, dgd.Namespace, service.Namespace) + require.Equal(t, corev1.ClusterIPNone, service.Spec.ClusterIP) + require.Equal(t, map[string]string{ + commonconsts.KubeLabelDynamoBaseModelHash: HashModelName("llama-3"), + }, service.Spec.Selector) + require.Equal(t, commonconsts.DynamoSystemPortName, service.Spec.Ports[0].Name) + require.Equal(t, int32(commonconsts.DynamoSystemPort), service.Spec.Ports[0].Port) + require.Equal(t, HashModelName("llama-3"), service.Labels[commonconsts.KubeLabelDynamoBaseModelHash]) + require.Equal(t, "dynamo-operator", service.Labels["nvidia.com/managed-by"]) + require.Equal(t, "llama-3", service.Annotations[commonconsts.KubeAnnotationDynamoBaseModel]) + require.Equal(t, "true", service.Annotations["example.com/graph"]) + require.Len(t, service.OwnerReferences, 1) + require.Equal(t, dgd.UID, service.OwnerReferences[0].UID) + require.True(t, *service.OwnerReferences[0].Controller) +} + +func TestGenerateModelServiceForGraphIsPrivateToGraph(t *testing.T) { + first := GenerateModelServiceForGraph("inference", "llama-3", "graph-a", nil) + second := GenerateModelServiceForGraph("inference", "llama-3", "graph-b", nil) + + require.NotEqual(t, first.Name, second.Name) + require.Equal(t, "graph-a", first.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName]) + require.Equal(t, "graph-a", first.Spec.Selector[commonconsts.KubeLabelDynamoGraphDeploymentName]) + require.Equal(t, HashModelName("llama-3"), first.Labels[commonconsts.KubeLabelDynamoBaseModelHash]) + require.LessOrEqual(t, len(first.Name), 63) + require.LessOrEqual(t, len(second.Name), 63) +} diff --git a/deploy/operator/internal/features/gates.go b/deploy/operator/internal/features/gates.go index b9d08ba9b2ce..d8b5eee903fb 100644 --- a/deploy/operator/internal/features/gates.go +++ b/deploy/operator/internal/features/gates.go @@ -65,6 +65,18 @@ const ( // Default: true when both API groups are detected; false otherwise LWS Name = "lws" + // DisaggregatedSet enables the opt-in DisaggregatedSet workload pathway. + // + // Owner: @kay-yan + // Experimental since: v1.4.0 + // Beta since: N/A + // GA since: N/A + // Configuration: nvidia.com/enable-disaggregatedset annotation + // Auto-detection: disaggregatedset.x-k8s.io/v1 API + // Requires: LWS serving disaggregatedset.x-k8s.io/v1 + // Default: true when the API is detected; false otherwise + DisaggregatedSet Name = "disaggregatedSet" + // KaiScheduler enables Kai Scheduler integration. // // Owner: @julienmancuso @@ -131,6 +143,7 @@ var allNames = [...]Name{ Checkpoint, Grove, LWS, + DisaggregatedSet, KaiScheduler, VolcanoScheduler, DRA, @@ -148,6 +161,7 @@ type Gates struct { Checkpoint bool `json:"checkpoint"` Grove bool `json:"grove"` LWS bool `json:"lws"` + DisaggregatedSet bool `json:"disaggregatedSet"` KaiScheduler bool `json:"kaiScheduler"` VolcanoScheduler bool `json:"volcanoScheduler"` DRA bool `json:"dra"` @@ -163,7 +177,11 @@ func Defaults() Gates { } // New detects cluster capabilities and resolves them with operator configuration. -func New(ctx context.Context, mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration) (Gates, error) { +func New( + ctx context.Context, + mgr ctrl.Manager, + config *configv1alpha1.OperatorConfiguration, +) (Gates, error) { gates := Defaults() gates.GPUDiscovery = config.Namespace.Restricted == "" || ptr.Deref(config.GPU.DiscoveryEnabled, true) @@ -194,7 +212,6 @@ func New(ctx context.Context, mgr ctrl.Manager, config *configv1alpha1.OperatorC "Grove is explicitly enabled in config but the Grove API group was not detected in the cluster"); err != nil { return Gates{}, err } - lwsAvailable, err := detectAPIAvailability(ctx, mgr.GetConfig(), "leaderworkerset.x-k8s.io", "", "") if err != nil { return Gates{}, err @@ -203,6 +220,14 @@ func New(ctx context.Context, mgr ctrl.Manager, config *configv1alpha1.OperatorC if err != nil { return Gates{}, err } + disaggregatedSetAvailable, err := detectAPIAvailability(ctx, mgr.GetConfig(), "disaggregatedset.x-k8s.io", "v1", "") + if err != nil { + return Gates{}, err + } + // The DS pathway lists and watches the LWS children created by the DS + // controller. Do not register those watches when the LWS API is absent. + lwsOptedOut := config.Orchestrators.LWS.Enabled != nil && !*config.Orchestrators.LWS.Enabled + gates.DisaggregatedSet = !lwsOptedOut && lwsAvailable && disaggregatedSetAvailable if ptr.Deref(config.Orchestrators.LWS.Enabled, lwsAvailable && volcanoAvailable) { if !lwsAvailable { return Gates{}, fmt.Errorf("LWS is explicitly enabled in config but the LWS API group was not detected in the cluster") diff --git a/deploy/operator/internal/features/gates_test.go b/deploy/operator/internal/features/gates_test.go index c1decd1fcb36..8bd6506bede8 100644 --- a/deploy/operator/internal/features/gates_test.go +++ b/deploy/operator/internal/features/gates_test.go @@ -58,6 +58,7 @@ func allEnabledGates() Gates { Checkpoint: true, Grove: true, LWS: true, + DisaggregatedSet: true, KaiScheduler: true, VolcanoScheduler: true, DRA: true, diff --git a/deploy/operator/internal/testing/operatorenv/env.go b/deploy/operator/internal/testing/operatorenv/env.go index c3b5389c72c9..a985d5d3db19 100644 --- a/deploy/operator/internal/testing/operatorenv/env.go +++ b/deploy/operator/internal/testing/operatorenv/env.go @@ -482,6 +482,7 @@ func crdDirectoryPaths(opts Options) []string { root := operatorRoot() return []string{ filepath.Join(root, "config", "crd", "bases"), + filepath.Join(root, "internal", "controller", "testing", "disaggregatedset"), filepath.Join(root, "internal", "controller", "testing", "prometheus"), filepath.Join(root, "internal", "controller", "testing", "volcano.sh"), filepath.Join(root, "internal", "controller", "testing", "run.ai"), diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go index 3ce73dc8c050..ed55145a728b 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler.go @@ -170,6 +170,10 @@ func defaultWorkloadProvider( strings.ToLower(dgd.Annotations[consts.KubeAnnotationEnableGrove]) != consts.KubeLabelValueFalse { provider = consts.WorkloadProviderGrove } + if provider == consts.WorkloadProviderComponent && + strings.ToLower(dgd.Annotations[consts.KubeAnnotationEnableDisaggregatedSet]) == consts.KubeLabelValueTrue { + provider = consts.WorkloadProviderDisaggregatedSet + } // Allocate annotation storage before materializing the selected provider. if dgd.Annotations == nil { diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler_test.go index 5079ab635302..8cea49189206 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeployment_handler_test.go @@ -384,6 +384,22 @@ func TestDGDDefaulter_DefaultsGroveMinAvailable(t *testing.T) { "Idle": ptr.To(int32(1)), }, }, + { + name: "CREATE selects DisaggregatedSet after Grove opt-out", + op: admissionv1.Create, + groveEnabled: true, + annotations: map[string]string{ + consts.KubeAnnotationEnableGrove: consts.KubeLabelValueFalse, + consts.KubeAnnotationEnableDisaggregatedSet: consts.KubeLabelValueTrue, + }, + components: []nvidiacomv1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "Worker", Replicas: ptr.To(int32(3))}, + }, + wantMinAvailable: map[string]*int32{ + "Worker": nil, + }, + wantProvider: consts.WorkloadProviderDisaggregatedSet, + }, { name: "preserves explicit minAvailable", op: admissionv1.Create, diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment.go b/deploy/operator/internal/webhook/validation/dynamographdeployment.go index edf8d49e2d1f..c980c00acfdd 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment.go @@ -202,11 +202,11 @@ func (v *dynamoGraphDeploymentValidation) validateObjectMeta( // Restrict the durable workload provider to programs implemented by the controller. if value, exists := objectMeta.Annotations[consts.KubeAnnotationWorkloadProvider]; exists && - value != consts.WorkloadProviderComponent && value != consts.WorkloadProviderGrove { + value != consts.WorkloadProviderComponent && value != consts.WorkloadProviderGrove && value != consts.WorkloadProviderDisaggregatedSet { allErrs = append(allErrs, field.NotSupported( annotationsPath.Key(consts.KubeAnnotationWorkloadProvider), value, - []string{consts.WorkloadProviderComponent, consts.WorkloadProviderGrove}, + []string{consts.WorkloadProviderComponent, consts.WorkloadProviderGrove, consts.WorkloadProviderDisaggregatedSet}, )) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go index 9c98f2ff49c8..555aa95f4343 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_helpers.go @@ -149,6 +149,11 @@ func grovePathwayForDynamoGraphDeployment( "requires the Grove pathway, but workload provider %q is selected", provider, ) + case consts.WorkloadProviderDisaggregatedSet: + return false, fmt.Sprintf( + "requires the Grove pathway, but workload provider %q is selected", + provider, + ) default: return false, fmt.Sprintf( "requires the Grove pathway, but annotation %q has unsupported value %q", diff --git a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go index 8642ac1e782e..f184cde9fec6 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeployment_validation_envtest_test.go @@ -1815,7 +1815,7 @@ func TestDynamoGraphDeploymentValidator_Validate(t *testing.T) { } dgd.Labels = map[string]string{"updated": "true"} }), - wantWebhookErrs: []string{`metadata.annotations[nvidia.com/workload-provider]: Unsupported value: "unknown": supported values: "component", "grove"`}, + wantWebhookErrs: []string{`metadata.annotations[nvidia.com/workload-provider]: Unsupported value: "unknown": supported values: "component", "grove", "disaggregatedset"`}, }, { name: "user cannot materialize a legacy workload provider", diff --git a/docs/fern/pages/developer-guide/knowledge-base/kubernetes/multinode/lws.md b/docs/fern/pages/developer-guide/knowledge-base/kubernetes/multinode/lws.md index 206c2120d0a1..051815a969e3 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/kubernetes/multinode/lws.md +++ b/docs/fern/pages/developer-guide/knowledge-base/kubernetes/multinode/lws.md @@ -12,22 +12,29 @@ Use LWS when you want a simpler multinode orchestrator than Grove, or when your ## Prerequisites - Kubernetes cluster with GPU nodes. -- LWS version `0.7.0` or newer. -- Volcano installed for gang scheduling. +- LWS version `0.7.0` or newer for the standard DCD pathway. +- Volcano installed for gang scheduling on the standard DCD pathway. - Dynamo Kubernetes Platform installed. The installation guide includes the exact Helm commands for [LWS and Volcano](../../../../kubernetes/installation/install-dynamo.md#lws--volcano). ## Orchestrator Selection -For multinode deployments, the Dynamo operator selects an orchestrator based on what is installed: +For multinode deployments, the operator applies this routing precedence: + +1. Grove, when its API is available and `nvidia.com/enable-grove` is not `"false"`. +2. Opt-in DisaggregatedSet (DS), when Grove is not selected and the DGD sets `nvidia.com/enable-disaggregatedset: "true"`. +3. The standard DynamoComponentDeployment (DCD) pathway, which requires LWS and Volcano for multinode components. + +Installing the DS API does not move existing DGDs to DS. DS requires explicit opt-in, and Grove has higher priority during initial selection. Dynamo persists the selected provider in `nvidia.com/workload-provider`; the enable annotations are creation/adoption-time intent only. Changing them does not migrate an existing DGD. To change providers, delete and recreate the DGD. | Cluster state | Operator behavior | | --- | --- | -| Grove and LWS installed | Uses Grove by default. | -| Grove and LWS installed, DGD has `nvidia.com/enable-grove: "false"` | Uses LWS. | -| Only LWS installed | Uses LWS. | -| Neither Grove nor LWS installed | Rejects multinode deployments. | +| Grove is available and `nvidia.com/enable-grove` is not `"false"` | Uses Grove. | +| Grove is disabled or unavailable and DS is explicitly requested | Uses DS when the DS API and requested roles are supported. | +| Grove and DS are not selected | Uses the standard DCD pathway. | +| A selected provider is unavailable | Reports `Ready=False` with `selected_workload_provider_unavailable`; it does not fall back to another provider. | +| No selected pathway supports the multinode components | Rejects the deployment. | To force the LWS path when Grove is also present: @@ -42,6 +49,14 @@ spec: # ... ``` +## DisaggregatedSet Path + +Use DS when one object should own multiple multinode worker roles. Install an LWS release that serves `disaggregatedset.x-k8s.io/v1` (the current Dynamo operator dependency is LWS `v0.10.0`), then add `nvidia.com/enable-disaggregatedset: "true"` to a new DGD. If Grove is available and enabled, also set `nvidia.com/enable-grove: "false"`. + +Once DS is selected, Dynamo keeps it as the provider even if the annotations or cluster capabilities change. If the DS API or its prerequisites become unavailable, the operator reports `Ready=False` with reason `selected_workload_provider_unavailable`; it does not fall back to DCD or Grove. Unsupported DS roles, including a selected worker using `scalingAdapter`, report `DisaggregatedSetEligible=False`. Delete and recreate the DGD to select a different pathway. + +DS supports two to ten eligible multinode worker roles. All selected roles must use either zero replicas or positive replicas. Dynamo's component discovery Services select one DS revision at a time; during a rollout they continue serving the active revision until the target revision is ready. + ## Multinode Spec Set `multinode.nodeCount` on the service that should span nodes. The total GPU count is `multinode.nodeCount` multiplied by the per-node GPU limit: diff --git a/docs/fern/pages/kubernetes/installation/install-dynamo.md b/docs/fern/pages/kubernetes/installation/install-dynamo.md index 3a197fa8a887..beb43364139a 100644 --- a/docs/fern/pages/kubernetes/installation/install-dynamo.md +++ b/docs/fern/pages/kubernetes/installation/install-dynamo.md @@ -185,6 +185,8 @@ The Dynamo install command above includes commented flags for each optional comp Multinode deployments require either Grove + KAI Scheduler or an alternative orchestrator setup (LeaderWorkerSet + Volcano) to enable gang scheduling for workloads that span multiple nodes. See the [Multinode Deployment Guide](../model-deployment/multinode-deployments.md) for details on orchestrator selection and configuration. +DisaggregatedSet is an opt-in multinode pathway below Grove. To request it for a new DGD, install an LWS release that serves `disaggregatedset.x-k8s.io/v1` and set `nvidia.com/enable-disaggregatedset: "true"`. When Grove is available and enabled, also set `nvidia.com/enable-grove: "false"`. Dynamo persists the selected provider in `nvidia.com/workload-provider`; installing the DS API or changing the enable annotations does not change existing deployments. If the selected provider becomes unavailable, Dynamo reports `Ready=False` instead of silently switching providers. Delete and recreate the DGD to choose a different pathway. + #### Grove + KAI Scheduler There are two ways to enable Grove and KAI Scheduler, controlled by which flags you uncomment in the Dynamo install command: @@ -220,10 +222,10 @@ helm repo update helm install volcano volcano-sh/volcano -n volcano-system --create-namespace ``` -2. Install LWS (>= v0.7.0) with Volcano gang scheduling enabled: +2. Install LWS with Volcano gang scheduling enabled. Use an LWS release that serves `disaggregatedset.x-k8s.io/v1` (the current Dynamo operator dependency is LWS `v0.10.0`) when enabling DS; the standard DCD pathway supports LWS `v0.7.0` or newer: ```bash -export LWS_VERSION=0.8.0 +export LWS_VERSION=0.10.0 helm install lws oci://registry.k8s.io/lws/charts/lws \ --version=$LWS_VERSION \ --namespace lws-system \ diff --git a/docs/fern/pages/kubernetes/installation/multinode-orchestration.md b/docs/fern/pages/kubernetes/installation/multinode-orchestration.md index f864d7e14023..0e407d05f752 100644 --- a/docs/fern/pages/kubernetes/installation/multinode-orchestration.md +++ b/docs/fern/pages/kubernetes/installation/multinode-orchestration.md @@ -54,7 +54,7 @@ Grove is the default and recommended orchestrator for multinode deployments. It ## Option 2: LeaderWorkerSet + Volcano -If you are not using Grove for multinode, you can use [LeaderWorkerSet (LWS)](https://lws.sigs.k8s.io/docs/installation/) (>= v0.7.0) with [Volcano](https://github.com/volcano-sh/volcano#quick-start-guide) for gang scheduling. Both must be installed before deploying multinode workloads. +If you are not using Grove for multinode, you can use [LeaderWorkerSet (LWS)](https://lws.sigs.k8s.io/docs/installation/) with [Volcano](https://github.com/volcano-sh/volcano#quick-start-guide) for gang scheduling. Use an LWS release that serves `disaggregatedset.x-k8s.io/v1` (the current Dynamo operator dependency is LWS `v0.10.0`) when enabling DS; the standard DCD pathway supports LWS `v0.7.0` or newer. Both must be installed before deploying multinode workloads. 1. Install Volcano: @@ -64,10 +64,10 @@ helm repo update helm install volcano volcano-sh/volcano -n volcano-system --create-namespace ``` -2. Install LWS (>= v0.7.0) with Volcano gang scheduling enabled: +2. Install LWS with Volcano gang scheduling enabled. Use an LWS release that serves `disaggregatedset.x-k8s.io/v1` (the current Dynamo operator dependency is LWS `v0.10.0`) when enabling DS; the standard DCD pathway supports LWS `v0.7.0` or newer: ```bash -export LWS_VERSION=0.8.0 +export LWS_VERSION=0.10.0 helm install lws oci://registry.k8s.io/lws/charts/lws \ --version=$LWS_VERSION \ --namespace lws-system \ diff --git a/docs/fern/pages/kubernetes/model-deployment/multinode-deployments.md b/docs/fern/pages/kubernetes/model-deployment/multinode-deployments.md index 3930ff985bb7..b50198c5fcae 100644 --- a/docs/fern/pages/kubernetes/model-deployment/multinode-deployments.md +++ b/docs/fern/pages/kubernetes/model-deployment/multinode-deployments.md @@ -75,14 +75,13 @@ Volcano is a Kubernetes native scheduler optimized for AI workloads at scale. It ### Orchestrator Selection Algorithm -Dynamo automatically selects the best available orchestrator for multinode deployments using the following logic: +Dynamo uses an ordered routing decision for multinode deployments: -#### When Both Grove and LWS are Available: -- **Grove is selected by default** (recommended for advanced AI workloads) -- **LWS is selected** if you explicitly set `nvidia.com/enable-grove: "false"` annotation on your DGD resource +1. **Grove:** selected when Grove is available and `nvidia.com/enable-grove` is not `"false"`. +2. **DisaggregatedSet:** selected when Grove was not selected and the DGD sets `nvidia.com/enable-disaggregatedset: "true"`. Unsupported API or role requirements are reported through `DisaggregatedSetEligible=False` without silently changing pathways. +3. **Standard DCD pathway:** used when neither Grove nor DS is selected. Multinode components on this pathway require LWS and Volcano. -#### When Only One Orchestrator is Available: -- The installed orchestrator (Grove or LWS) is automatically selected +Grove and DS are not mutually exclusive features. Grove has higher priority during initial routing, while DS requires explicit opt-in. Dynamo persists the selected provider in `nvidia.com/workload-provider`; changing enable annotations or installing another provider does not move an existing deployment. If the selected provider is unavailable, the DGD reports `Ready=False` with `selected_workload_provider_unavailable` and does not fall back. Delete and recreate the DGD to select a different pathway. #### Scheduler Integration: - **With Grove**: Dynamo uses Grove for multinode orchestration when the Grove API is available, unless you set `nvidia.com/enable-grove: "false"` on the DGD resource. Scheduler integration is configured separately: