diff --git a/README.md b/README.md index 5ea033a5f..95b5f72ee 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ The [DSA device plugin](cmd/dsa_plugin/README.md) supports acceleration using th ## Device Plugins Operator -Currently the operator has support for the QAT, GPU, FPGA and SGX device plugins: +Currently the operator has support for the QAT, GPU, FPGA, SGX and DSA device plugins: it validates container image references and extends reported statuses. To run an operator instance in the container run diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 5ab1d8f4e..fba80cdf5 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -27,6 +27,7 @@ import ( devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" fpgav2 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/fpga/v2" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/dsa" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/gpu" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/qat" @@ -74,6 +75,15 @@ func main() { os.Exit(1) } + if err = dsa.SetupReconciler(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DsaDevicePlugin") + os.Exit(1) + } + if err = (&devicepluginv1.DsaDevicePlugin{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "DsaDevicePlugin") + os.Exit(1) + } + if err = gpu.SetupReconciler(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GpuDevicePlugin") os.Exit(1) diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml new file mode 100644 index 000000000..c63d55fc8 --- /dev/null +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_dsadeviceplugins.yaml @@ -0,0 +1,144 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: dsadeviceplugins.deviceplugin.intel.com +spec: + additionalPrinterColumns: + - JSONPath: .status.desiredNumberScheduled + name: Desired + type: integer + - JSONPath: .status.numberReady + name: Ready + type: integer + - JSONPath: .spec.nodeSelector + name: Node Selector + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: deviceplugin.intel.com + names: + kind: DsaDevicePlugin + listKind: DsaDevicePluginList + plural: dsadeviceplugins + singular: dsadeviceplugin + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: DsaDevicePlugin is the Schema for the dsadeviceplugins API. It + represents the DSA device plugin responsible for advertising Intel DSA hardware + resources to the kubelet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DsaDevicePluginSpec defines the desired state of DsaDevicePlugin. + properties: + image: + description: Image is a container image with DSA device plugin executable. + type: string + logLevel: + description: LogLevel sets the plugin's log level. + minimum: 0 + type: integer + nodeSelector: + additionalProperties: + type: string + description: NodeSelector provides a simple way to constrain device + plugin pods to nodes with particular labels. + type: object + sharedDevNum: + description: SharedDevNum is a number of containers that can share the + same DSA device. + minimum: 1 + type: integer + type: object + status: + description: DsaDevicePluginStatus defines the observed state of DsaDevicePlugin. + properties: + controlledDaemonSet: + description: ControlledDaemoSet references the DaemonSet controlled + by the operator. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + desiredNumberScheduled: + description: The total number of nodes that should be running the device + plugin pod (including nodes correctly running the device plugin pod). + format: int32 + type: integer + nodeNames: + description: The list of Node names where the device plugin pods are + running. + items: + type: string + type: array + numberReady: + description: The number of nodes that should be running the device plugin + pod and have one or more of the device plugin pod running and ready. + format: int32 + type: integer + required: + - desiredNumberScheduled + - numberReady + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_fpgadeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_fpgadeviceplugins.yaml index 53faeee40..9cbc08c9f 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_fpgadeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_fpgadeviceplugins.yaml @@ -63,7 +63,7 @@ spec: minimum: 0 type: integer mode: - description: Mode is a mode the plugin's operation. + description: Mode is a mode of the plugin's operation. enum: - af - region diff --git a/deployments/operator/crd/kustomization.yaml b/deployments/operator/crd/kustomization.yaml index fa7bd27d6..d6561d0d3 100644 --- a/deployments/operator/crd/kustomization.yaml +++ b/deployments/operator/crd/kustomization.yaml @@ -6,6 +6,7 @@ resources: - bases/deviceplugin.intel.com_qatdeviceplugins.yaml - bases/deviceplugin.intel.com_fpgadeviceplugins.yaml - bases/deviceplugin.intel.com_sgxdeviceplugins.yaml +- bases/deviceplugin.intel.com_dsadeviceplugins.yaml - bases/fpga.intel.com_acceleratorfunctions.yaml - bases/fpga.intel.com_fpgaregions.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/deployments/operator/manifests/bases/intel-device-plugins-operator.clusterserviceversion.yaml b/deployments/operator/manifests/bases/intel-device-plugins-operator.clusterserviceversion.yaml index 7249cb3f5..7134df61c 100644 --- a/deployments/operator/manifests/bases/intel-device-plugins-operator.clusterserviceversion.yaml +++ b/deployments/operator/manifests/bases/intel-device-plugins-operator.clusterserviceversion.yaml @@ -43,10 +43,15 @@ spec: kind: SgxDevicePlugin name: sgxdeviceplugins.deviceplugin.intel.com version: v1 + description: DsaDevicePlugin is the Schema for the dsadeviceplugins API. It represents the DSA device plugin responsible for advertising Intel DSA hardware resources to the kubelet. + displayName: Intel DSA Device Plugin + kind: DsaDevicePlugin + name: dsadeviceplugins.deviceplugin.intel.com + version: v1 description: | [Intel Device Plugins for Kubernetes](https://github.com/intel/intel-device-plugins-for-kubernetes) is a collection of [device plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/) advertising Intel specific hardware resources - to the kubelet. Currently the operator has basic support for the QAT, GPU, FPGA and SGX device plugins: it validates container image references and extends + to the kubelet. Currently the operator has basic support for the QAT, GPU, FPGA, SGX and DSA device plugins: it validates container image references and extends reported statuses. displayName: Intel Device Plugins Operator icon: diff --git a/deployments/operator/rbac/role.yaml b/deployments/operator/rbac/role.yaml index b9ee46f47..4aa12a36d 100644 --- a/deployments/operator/rbac/role.yaml +++ b/deployments/operator/rbac/role.yaml @@ -41,6 +41,26 @@ rules: verbs: - get - update +- apiGroups: + - deviceplugin.intel.com + resources: + - dsadeviceplugins + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - deviceplugin.intel.com + resources: + - dsadeviceplugins/status + verbs: + - get + - patch + - update - apiGroups: - deviceplugin.intel.com resources: diff --git a/deployments/operator/samples/deviceplugin_v1_dsadeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_dsadeviceplugin.yaml new file mode 100644 index 000000000..65b678200 --- /dev/null +++ b/deployments/operator/samples/deviceplugin_v1_dsadeviceplugin.yaml @@ -0,0 +1,8 @@ +apiVersion: deviceplugin.intel.com/v1 +kind: DsaDevicePlugin +metadata: + name: dsadeviceplugin-sample +spec: + image: intel/intel-dsa-plugin:0.18.0 + sharedDevNum: 10 + logLevel: 4 diff --git a/deployments/operator/webhook/manifests.yaml b/deployments/operator/webhook/manifests.yaml index fb4fe2197..46dcdbee8 100644 --- a/deployments/operator/webhook/manifests.yaml +++ b/deployments/operator/webhook/manifests.yaml @@ -6,6 +6,26 @@ metadata: creationTimestamp: null name: mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-deviceplugin-intel-com-v1-dsadeviceplugin + failurePolicy: Fail + name: mdsadeviceplugin.kb.io + rules: + - apiGroups: + - deviceplugin.intel.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - dsadeviceplugins + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -134,6 +154,26 @@ metadata: creationTimestamp: null name: validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-deviceplugin-intel-com-v1-dsadeviceplugin + failurePolicy: Fail + name: vdsadeviceplugin.kb.io + rules: + - apiGroups: + - deviceplugin.intel.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - dsadeviceplugins + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go b/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go new file mode 100644 index 000000000..3f5e58f56 --- /dev/null +++ b/pkg/apis/deviceplugin/v1/dsadeviceplugin_types.go @@ -0,0 +1,95 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// DsaDevicePluginSpec defines the desired state of DsaDevicePlugin. +type DsaDevicePluginSpec struct { + // Important: Run "make generate" to regenerate code after modifying this file + + // Image is a container image with DSA device plugin executable. + Image string `json:"image,omitempty"` + + // SharedDevNum is a number of containers that can share the same DSA device. + // +kubebuilder:validation:Minimum=1 + SharedDevNum int `json:"sharedDevNum,omitempty"` + + // LogLevel sets the plugin's log level. + // +kubebuilder:validation:Minimum=0 + LogLevel int `json:"logLevel,omitempty"` + + // NodeSelector provides a simple way to constrain device plugin pods to nodes with particular labels. + NodeSelector map[string]string `json:"nodeSelector,omitempty"` +} + +// DsaDevicePluginStatus defines the observed state of DsaDevicePlugin. +type DsaDevicePluginStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make generate" to regenerate code after modifying this file + + // ControlledDaemoSet references the DaemonSet controlled by the operator. + // +optional + ControlledDaemonSet v1.ObjectReference `json:"controlledDaemonSet,omitempty"` + + // The total number of nodes that should be running the device plugin + // pod (including nodes correctly running the device plugin pod). + DesiredNumberScheduled int32 `json:"desiredNumberScheduled"` + + // The number of nodes that should be running the device plugin pod and have one + // or more of the device plugin pod running and ready. + NumberReady int32 `json:"numberReady"` + + // The list of Node names where the device plugin pods are running. + // +optional + NodeNames []string `json:"nodeNames,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=`.status.desiredNumberScheduled` +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.numberReady` +// +kubebuilder:printcolumn:name="Node Selector",type=string,JSONPath=`.spec.nodeSelector` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +operator-sdk:csv:customresourcedefinitions:displayName="Intel DSA Device Plugin" + +// DsaDevicePlugin is the Schema for the dsadeviceplugins API. It represents +// the DSA device plugin responsible for advertising Intel DSA hardware resources to +// the kubelet. +type DsaDevicePlugin struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DsaDevicePluginSpec `json:"spec,omitempty"` + Status DsaDevicePluginStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// DsaDevicePluginList contains a list of DsaDevicePlugin. +type DsaDevicePluginList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DsaDevicePlugin `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DsaDevicePlugin{}, &DsaDevicePluginList{}) +} diff --git a/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go new file mode 100644 index 000000000..6a6124602 --- /dev/null +++ b/pkg/apis/deviceplugin/v1/dsadeviceplugin_webhook.go @@ -0,0 +1,90 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/version" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" +) + +const ( + dsaPluginKind = "DsaDevicePlugin" +) + +var ( + // dsadevicepluginlog is for logging in this package. + dsadevicepluginlog = logf.Log.WithName("dsadeviceplugin-resource") + + dsaMinVersion = version.MustParseSemantic("0.18.0") +) + +// SetupWebhookWithManager sets up a webhook for DsaDevicePlugin custom resources. +func (r *DsaDevicePlugin) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-deviceplugin-intel-com-v1-dsadeviceplugin,mutating=true,failurePolicy=fail,groups=deviceplugin.intel.com,resources=dsadeviceplugins,verbs=create;update,versions=v1,name=mdsadeviceplugin.kb.io,sideEffects=None,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &DsaDevicePlugin{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type. +func (r *DsaDevicePlugin) Default() { + dsadevicepluginlog.Info("default", "name", r.Name) + + if len(r.Spec.Image) == 0 { + r.Spec.Image = "intel/intel-dsa-plugin:0.18.0" + } +} + +// +kubebuilder:webhook:verbs=create;update,path=/validate-deviceplugin-intel-com-v1-dsadeviceplugin,mutating=false,failurePolicy=fail,groups=deviceplugin.intel.com,resources=dsadeviceplugins,versions=v1,name=vdsadeviceplugin.kb.io,sideEffects=None,admissionReviewVersions=v1 + +var _ webhook.Validator = &DsaDevicePlugin{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (r *DsaDevicePlugin) ValidateCreate() error { + dsadevicepluginlog.Info("validate create", "name", r.Name) + + if controllers.GetDevicePluginCount(dsaPluginKind) > 0 { + return errors.Errorf("an instance of %q already exists in the cluster", dsaPluginKind) + } + + return r.validatePlugin() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (r *DsaDevicePlugin) ValidateUpdate(old runtime.Object) error { + dsadevicepluginlog.Info("validate update", "name", r.Name) + + return r.validatePlugin() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (r *DsaDevicePlugin) ValidateDelete() error { + dsadevicepluginlog.Info("validate delete", "name", r.Name) + + return nil +} + +func (r *DsaDevicePlugin) validatePlugin() error { + return validatePluginImage(r.Spec.Image, "intel-dsa-plugin", dsaMinVersion) +} diff --git a/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go b/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go index e225c7f22..aeee3ada5 100644 --- a/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go @@ -22,6 +22,108 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DsaDevicePlugin) DeepCopyInto(out *DsaDevicePlugin) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DsaDevicePlugin. +func (in *DsaDevicePlugin) DeepCopy() *DsaDevicePlugin { + if in == nil { + return nil + } + out := new(DsaDevicePlugin) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DsaDevicePlugin) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DsaDevicePluginList) DeepCopyInto(out *DsaDevicePluginList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DsaDevicePlugin, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DsaDevicePluginList. +func (in *DsaDevicePluginList) DeepCopy() *DsaDevicePluginList { + if in == nil { + return nil + } + out := new(DsaDevicePluginList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DsaDevicePluginList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DsaDevicePluginSpec) DeepCopyInto(out *DsaDevicePluginSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DsaDevicePluginSpec. +func (in *DsaDevicePluginSpec) DeepCopy() *DsaDevicePluginSpec { + if in == nil { + return nil + } + out := new(DsaDevicePluginSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DsaDevicePluginStatus) DeepCopyInto(out *DsaDevicePluginStatus) { + *out = *in + out.ControlledDaemonSet = in.ControlledDaemonSet + if in.NodeNames != nil { + in, out := &in.NodeNames, &out.NodeNames + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DsaDevicePluginStatus. +func (in *DsaDevicePluginStatus) DeepCopy() *DsaDevicePluginStatus { + if in == nil { + return nil + } + out := new(DsaDevicePluginStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FpgaDevicePlugin) DeepCopyInto(out *FpgaDevicePlugin) { *out = *in diff --git a/pkg/controllers/dsa/controller.go b/pkg/controllers/dsa/controller.go new file mode 100644 index 000000000..858f11394 --- /dev/null +++ b/pkg/controllers/dsa/controller.go @@ -0,0 +1,258 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dsa contains DSA specific reconciliation logic. +package dsa + +import ( + "context" + "reflect" + "strconv" + "strings" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/reference" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" + "github.com/pkg/errors" +) + +const ( + ownerKey = ".metadata.controller.dsa" + appLabel = "intel-dsa-plugin" +) + +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dsadeviceplugins,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=dsadeviceplugins/status,verbs=get;update;patch + +// SetupReconciler creates a new reconciler for DsaDevicePlugin objects. +func SetupReconciler(mgr ctrl.Manager) error { + c := &controller{scheme: mgr.GetScheme()} + return controllers.SetupWithManager(mgr, c, devicepluginv1.GroupVersion.String(), "DsaDevicePlugin", ownerKey) +} + +type controller struct { + scheme *runtime.Scheme +} + +func (c *controller) CreateEmptyObject() client.Object { + return &devicepluginv1.DsaDevicePlugin{} +} + +func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client) (int, error) { + var list devicepluginv1.DsaDevicePluginList + if err := clnt.List(ctx, &list); err != nil { + return 0, err + } + + return len(list.Items), nil +} + +func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { + devicePlugin := rawObj.(*devicepluginv1.DsaDevicePlugin) + + var nodeSelector map[string]string + dpNodeSelectorSize := len(devicePlugin.Spec.NodeSelector) + if dpNodeSelectorSize > 0 { + nodeSelector = make(map[string]string, dpNodeSelectorSize+1) + for k, v := range devicePlugin.Spec.NodeSelector { + nodeSelector[k] = v + } + nodeSelector["kubernetes.io/arch"] = "amd64" + } else { + nodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + } + + yes := true + daemonSet := apps.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: devicePlugin.Namespace, + GenerateName: devicePlugin.Name + "-", + Labels: map[string]string{ + "app": appLabel, + }, + }, + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": appLabel, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": appLabel, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: appLabel, + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + Args: getPodArgs(devicePlugin), + Image: devicePlugin.Spec.Image, + ImagePullPolicy: "IfNotPresent", + SecurityContext: &v1.SecurityContext{ + ReadOnlyRootFilesystem: &yes, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "devfs", + MountPath: "/dev/dsa", + ReadOnly: true, + }, + { + Name: "chardevs", + MountPath: "/dev/char", + ReadOnly: true, + }, + { + Name: "sysfs", + MountPath: "/sys/bus", + ReadOnly: true, + }, + { + Name: "kubeletsockets", + MountPath: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + NodeSelector: nodeSelector, + Volumes: []v1.Volume{ + { + Name: "devfs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev/dsa", + }, + }, + }, + { + Name: "chardevs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev/char", + }, + }, + }, + { + Name: "sysfs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/sys/bus", + }, + }, + }, + { + Name: "kubeletsockets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + }, + }, + }, + } + return &daemonSet +} + +func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { + dp := rawObj.(*devicepluginv1.DsaDevicePlugin) + + if ds.Spec.Template.Spec.Containers[0].Image != dp.Spec.Image { + ds.Spec.Template.Spec.Containers[0].Image = dp.Spec.Image + updated = true + } + + if dp.Spec.NodeSelector == nil { + dp.Spec.NodeSelector = map[string]string{"kubernetes.io/arch": "amd64"} + } else { + dp.Spec.NodeSelector["kubernetes.io/arch"] = "amd64" + } + if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { + ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector + updated = true + } + + newargs := getPodArgs(dp) + if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") { + ds.Spec.Template.Spec.Containers[0].Args = newargs + updated = true + } + + return updated +} + +func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, nodeNames []string) (updated bool, err error) { + dp := rawObj.(*devicepluginv1.DsaDevicePlugin) + + dsRef, err := reference.GetReference(c.scheme, ds) + if err != nil { + return false, errors.Wrap(err, "unable to make reference to controlled daemon set") + } + + if dp.Status.ControlledDaemonSet.UID != dsRef.UID { + dp.Status.ControlledDaemonSet = *dsRef + updated = true + } + + if dp.Status.DesiredNumberScheduled != ds.Status.DesiredNumberScheduled { + dp.Status.DesiredNumberScheduled = ds.Status.DesiredNumberScheduled + updated = true + } + + if dp.Status.NumberReady != ds.Status.NumberReady { + dp.Status.NumberReady = ds.Status.NumberReady + updated = true + } + + if strings.Join(dp.Status.NodeNames, ",") != strings.Join(nodeNames, ",") { + dp.Status.NodeNames = nodeNames + updated = true + } + + return updated, nil +} + +func getPodArgs(gdp *devicepluginv1.DsaDevicePlugin) []string { + args := make([]string, 0, 4) + args = append(args, "-v", strconv.Itoa(gdp.Spec.LogLevel)) + + if gdp.Spec.SharedDevNum > 0 { + args = append(args, "-shared-dev-num", strconv.Itoa(gdp.Spec.SharedDevNum)) + } else { + args = append(args, "-shared-dev-num", "1") + } + + return args +} diff --git a/test/envtest/dsadeviceplugin_controller_test.go b/test/envtest/dsadeviceplugin_controller_test.go new file mode 100644 index 000000000..3c91dfcd5 --- /dev/null +++ b/test/envtest/dsadeviceplugin_controller_test.go @@ -0,0 +1,87 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envtest + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" +) + +var _ = Describe("DsaDevicePlugin Controller", func() { + + const timeout = time.Second * 30 + const interval = time.Second * 1 + + Context("Basic CRUD operations", func() { + It("should handle DsaDevicePlugin objects correctly", func() { + spec := devicepluginv1.DsaDevicePluginSpec{ + Image: "testimage", + } + + key := types.NamespacedName{ + Name: "dsadeviceplugin-test", + Namespace: "default", + } + + toCreate := &devicepluginv1.DsaDevicePlugin{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Spec: spec, + } + + By("creating DsaDevicePlugin successfully") + Expect(k8sClient.Create(context.Background(), toCreate)).Should(Succeed()) + time.Sleep(time.Second * 5) + + fetched := &devicepluginv1.DsaDevicePlugin{} + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), key, fetched) + return len(fetched.Status.ControlledDaemonSet.UID) > 0 + }, timeout, interval).Should(BeTrue()) + + By("updating image name successfully") + updatedImage := "updated-dsa-testimage" + fetched.Spec.Image = updatedImage + + Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) + fetchedUpdated := &devicepluginv1.DsaDevicePlugin{} + Eventually(func() string { + _ = k8sClient.Get(context.Background(), key, fetchedUpdated) + return fetchedUpdated.Spec.Image + }, timeout, interval).Should(Equal(updatedImage)) + + By("deleting DsaDevicePlugin successfully") + Eventually(func() error { + f := &devicepluginv1.DsaDevicePlugin{} + _ = k8sClient.Get(context.Background(), key, f) + return k8sClient.Delete(context.Background(), f) + }, timeout, interval).Should(Succeed()) + + Eventually(func() error { + f := &devicepluginv1.DsaDevicePlugin{} + return k8sClient.Get(context.Background(), key, f) + }, timeout, interval).ShouldNot(Succeed()) + }) + }) +})