Skip to content

driver: deploy csi-addons sidecar in its own DaemonSet #269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Madhu-1 merged 2 commits into from
Jul 21, 2025
Merged

driver: deploy csi-addons sidecar in its own DaemonSet #269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7783af9
driver: deploy csi-addons sidecar in its own DaemonSet
black-dragon74 Jul 1, 2025
49efcbb
tests: Check that the CSIAddons daemonset is deployed
black-dragon74 Jul 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .github/actions/deploy-charts/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ runs:
shell: bash
run: |
DRIVER_NAME="${{ inputs.driverType }}.csi.ceph.com" yq eval '.drivers |= map(.enabled = (.name == strenv(DRIVER_NAME)))' -i ./deploy/charts/ceph-csi-drivers/values.yaml
yq eval '.drivers |= map(.deployCsiAddons = (.name == "rbd.csi.ceph.com"))' -i ./deploy/charts/ceph-csi-drivers/values.yaml

- name: Install csi driver helm chart
shell: bash
Expand All @@ -94,9 +95,15 @@ runs:

- name: Check csi pods are running
shell: bash
env:
DRIVER_TYPE: ${{ inputs.driverType }}
run: |
podCount=2
if [ "$DRIVER_TYPE" = "rbd" ]; then
podCount=3
fi
for i in {1..180}; do
if [ $(kubectl get pods -n csi-driver --field-selector=status.phase=Running --no-headers | wc -l) -eq 2 ]; then
if [ $(kubectl get pods -n csi-driver --field-selector=status.phase=Running --no-headers | wc -l) -eq $podCount ]; then
echo "All CSI pods are running"
kubectl get pods,deployment,daemonset,replicaset -n csi-driver
exit 0
Expand Down
260 changes: 187 additions & 73 deletions internal/controller/driver_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,16 +229,22 @@ func (r *driverReconcile) reconcile() error {
return err
}

// Concurrently reconcile different aspects of the clusters actual state to meet
// the desired state defined on the driver object
errChan := utils.RunConcurrently(
reconcilers := []func() error{
r.reconcileCsiConfigMap,
r.reconcileLogRotateConfigMap,
r.reconcileK8sCsiDriver,
r.reconcileControllerPluginDeployment,
r.reconcileNodePluginDeamonSet,
r.reconcileLivenessService,
)
}

if r.isRdbDriver() {
reconcilers = append(reconcilers, r.reconcileNodePluginDeamonSetForCsiAddons)
}

// Concurrently reconcile different aspects of the clusters actual state to meet
// the desired state defined on the driver object
errChan := utils.RunConcurrently(reconcilers...)

// Check if any reconcilatin error where raised during the concurrent execution
// of the reconciliation steps.
Expand Down Expand Up @@ -958,17 +964,177 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
}

func (r *driverReconcile) controllerPluginCsiAddonsContainerPort() corev1.ContainerPort {

// the cephFS and rbd drivers need to use different ports
// to avoid port collisions with host network.
port := utils.ControllerPluginCsiAddonsContainerRbdPort
if r.isCephFsDriver() {
port = utils.ControllerPluginCsiAddonsContainerCephFsPort

}

return port
}

func (r *driverReconcile) reconcileNodePluginDeamonSetForCsiAddons() error {
daemonSet := &appsv1.DaemonSet{}
daemonSet.Name = r.generateName("nodeplugin-csi-addons")
daemonSet.Namespace = r.driver.Namespace

log := r.log.WithValues("csiAddonsDaemonSetName", daemonSet.Name)

if !ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
if err := r.Delete(r.ctx, daemonSet); client.IgnoreNotFound(err) != nil {
log.Error(err, "failed to delete csi addons daemonset")
return err
}
return nil
}

log.Info("Reconciling csi addons nodeplugin daemonset")

opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, daemonSet, func() error {
if err := ctrlutil.SetControllerReference(&r.driver, daemonSet, r.Scheme); err != nil {
log.Error(err, "Failed to set owner reference on csi addons nodeplugin daemonset")

return err
}

appName := daemonSet.Name
pluginSpec := cmp.Or(r.driver.Spec.NodePlugin, &csiv1.NodePluginSpec{})
serviceAccountName := cmp.Or(
ptr.Deref(pluginSpec.ServiceAccountName, ""),
fmt.Sprintf("%s%s-nodeplugin-sa", serviceAccountPrefix, r.driverType),
)
imagePullPolicy := cmp.Or(pluginSpec.ImagePullPolicy, corev1.PullIfNotPresent)
logVerbosity := ptr.Deref(r.driver.Spec.Log, csiv1.LogSpec{}).Verbosity
kubeletDirPath := cmp.Or(pluginSpec.KubeletDirPath, defaultKubeletDirPath)
port := utils.NodePluginCsiAddonsContainerPort

logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1.LogSpec{}).Rotation
logRotationEnabled := logRotationSpec != nil

daemonSet.Spec = appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": appName},
},
UpdateStrategy: ptr.Deref(pluginSpec.UpdateStrategy, defaultDaemonSetUpdateStrategy),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: utils.Call(func() map[string]string {
podLabels := map[string]string{}
maps.Copy(podLabels, pluginSpec.Labels)
podLabels["app"] = appName
return podLabels
}),
Annotations: maps.Clone(pluginSpec.Annotations),
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
PriorityClassName: ptr.Deref(pluginSpec.PrioritylClassName, ""),
// to use e.g. Rook orchestrated cluster, and mons' FQDN is
// resolved through k8s service, set dns policy to cluster first
DNSPolicy: corev1.DNSClusterFirstWithHostNet,
Tolerations: pluginSpec.Tolerations,
Containers: utils.Call(func() []corev1.Container {
containers := []corev1.Container{
{
Name: "csi-addons",
Image: r.images["addons"],
ImagePullPolicy: imagePullPolicy,
// We need this in order for this container to be able to access
// the sockets created by the privileged nodeplugin container
// on systems with enforcing selinux.
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"All"},
},
},
Args: utils.DeleteZeroValues(
[]string{
utils.CsiAddonsNodeIdContainerArg,
utils.LogVerbosityContainerArg(logVerbosity),
utils.CsiAddonsAddressContainerArg,
utils.ContainerPortArg(port),
utils.PodContainerArg,
utils.NamespaceContainerArg,
utils.PodUidContainerArg,
utils.StagingPathContainerArg(kubeletDirPath),
utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
},
),
Ports: []corev1.ContainerPort{
port,
},
Env: []corev1.EnvVar{
utils.NodeIdEnvVar,
utils.PodNameEnvVar,
utils.PodNamespaceEnvVar,
utils.PodUidEnvVar,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.PluginDirVolumeMount,
}
if logRotationEnabled {
mounts = append(mounts, utils.LogsDirVolumeMount)
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
corev1.ResourceRequirements{},
),
},
}
// CSI LogRotate Container
if logRotationEnabled {
resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{})
containers = append(containers, corev1.Container{
Name: "log-rotator",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Resources: resources,
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"All"},
},
},
Command: []string{"/bin/bash", "-c", logRotateCmd},
VolumeMounts: []corev1.VolumeMount{
utils.LogsDirVolumeMount,
utils.LogRotateDirVolumeMount,
},
})
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
volumes := []corev1.Volume{
utils.PluginDirVolume(kubeletDirPath, r.driver.Name),
}

if logRotationEnabled {
logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath)
volumes = append(
volumes,
utils.LogsDirVolume(logHostPath, daemonSet.Name),
utils.LogRotateDirVolumeName(r.driver.Name),
)
}
return volumes
}),
},
},
}

return nil
})

logCreateOrUpdateResult(log, "csi addons node plugin daemonset", daemonSet, opResult, err)
return err
}

func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
Expand Down Expand Up @@ -1157,12 +1323,11 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
),
},
}
// CSI Addons Sidecar Container
if r.isRdbDriver() && ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
port := utils.NodePluginCsiAddonsContainerPort
// Liveness Sidecar Container
if r.driver.Spec.Liveness != nil {
containers = append(containers, corev1.Container{
Name: "csi-addons",
Image: r.images["addons"],
Name: "liveness-prometheus",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
Expand All @@ -1172,76 +1337,25 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
},
Args: utils.DeleteZeroValues(
[]string{
utils.CsiAddonsNodeIdContainerArg,
utils.LogVerbosityContainerArg(logVerbosity),
utils.CsiAddonsAddressContainerArg,
utils.ContainerPortArg(port),
utils.PodContainerArg,
utils.NamespaceContainerArg,
utils.PodUidContainerArg,
utils.StagingPathContainerArg(kubeletDirPath),
utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
utils.TypeContainerArg("liveness"),
utils.EndpointContainerArg,
utils.MetricsPortContainerArg(r.driver.Spec.Liveness.MetricsPort),
utils.MetricsPathContainerArg,
utils.PoolTimeContainerArg,
utils.TimeoutContainerArg(3),
},
),
Ports: []corev1.ContainerPort{
port,
},
Env: []corev1.EnvVar{
utils.NodeIdEnvVar,
utils.PodNameEnvVar,
utils.PodNamespaceEnvVar,
utils.PodUidEnvVar,
utils.PodIpEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.PluginDirVolumeMount,
},
VolumeMounts: utils.Call(func() []corev1.VolumeMount {
mounts := []corev1.VolumeMount{
utils.PluginDirVolumeMount,
}
if logRotationEnabled {
mounts = append(mounts, utils.LogsDirVolumeMount)
}
return mounts
}),
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
pluginSpec.Resources.Liveness,
corev1.ResourceRequirements{},
),
})
// Liveness Sidecar Container
Comment thread
Madhu-1 marked this conversation as resolved.
if r.driver.Spec.Liveness != nil {
containers = append(containers, corev1.Container{
Name: "liveness-prometheus",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"All"},
},
},
Args: utils.DeleteZeroValues(
[]string{
utils.TypeContainerArg("liveness"),
utils.EndpointContainerArg,
utils.MetricsPortContainerArg(r.driver.Spec.Liveness.MetricsPort),
utils.MetricsPathContainerArg,
utils.PoolTimeContainerArg,
utils.TimeoutContainerArg(3),
},
),
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
},
VolumeMounts: []corev1.VolumeMount{
utils.PluginDirVolumeMount,
},
Resources: ptr.Deref(
pluginSpec.Resources.Liveness,
corev1.ResourceRequirements{},
),
})
}
}
// CSI LogRotate Container
if logRotationEnabled {
Expand Down
Loading