Add distributed snapshotting support to kubernetes-distributed deployment

deploy/kubernetes-distributed/README.md

@@ -6,3 +6,19 @@ node, using distributed provisioning, and configures it so that it has
 The "kind" storage class parameter can selected between the two. If
 not set, an arbitrary kind with enough capacity is picked.
 
+## Prerequisites
+
+Snapshot support in this deployment uses per-node `csi-snapshotter`
+sidecars (`--node-deployment=true`). For that to work, the cluster
+must run an external `snapshot-controller` started with
+`--enable-distributed-snapshotting=true`, plus the matching
+`VolumeSnapshot*` CRDs. The `deploy.sh` script in this directory does
+not install snapshot-controller; install it separately before
+deploying the driver. See the
+[external-snapshotter documentation](https://github.com/kubernetes-csi/external-snapshotter#distributed-snapshotting)
+for the install steps.
+
+Cross-node snapshot restore (provisioning a PVC from a VolumeSnapshot
+whose source data lives on a different node) is not yet covered by
+this deployment alone. The companion `csi-topology-coordinator`
+component handles that case.

deploy/kubernetes-distributed/deploy.sh

@@ -122,6 +122,10 @@ function version_gt() {
 CSI_PROVISIONER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-plugin.yaml" csi-provisioner false)/deploy/kubernetes/rbac.yaml"
 : ${CSI_PROVISIONER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-plugin.yaml" csi-provisioner "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml}
 
+SNAPSHOTTER_RBAC_RELATIVE_PATH="csi-snapshotter/rbac-csi-snapshotter.yaml"
+CSI_SNAPSHOTTER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-plugin.yaml" csi-snapshotter false)/deploy/kubernetes/${SNAPSHOTTER_RBAC_RELATIVE_PATH}"
+: ${CSI_SNAPSHOTTER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-plugin.yaml" csi-snapshotter "${UPDATE_RBAC_RULES}")/deploy/kubernetes/${SNAPSHOTTER_RBAC_RELATIVE_PATH}}
+
 # Some images are not affected by *_REGISTRY/*_TAG and IMAGE_* variables.
 # The default is to update unless explicitly excluded.
 update_image () {
@@ -135,7 +139,7 @@ run () {
 
 # rbac rules
 echo "applying RBAC rules"
-for component in CSI_PROVISIONER; do
+for component in CSI_PROVISIONER CSI_SNAPSHOTTER; do
     eval current="\${${component}_RBAC}"
     eval original="\${${component}_RBAC_YAML}"
     if [ "$current" != "$original" ]; then

deploy/kubernetes-distributed/destroy.sh

@@ -31,4 +31,4 @@ set -o pipefail
 # kubectl maintains a few standard resources under "all" category which can be deleted by using just "kubectl delete all"
 # and other resources such as roles, clusterrole, serivceaccount etc needs to be deleted explicitly
 kubectl delete all --all-namespaces -l app.kubernetes.io/instance=hostpath.csi.k8s.io,app.kubernetes.io/part-of=csi-driver-host-path --wait=true
-kubectl delete role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver --all-namespaces -l app.kubernetes.io/instance=hostpath.csi.k8s.io,app.kubernetes.io/part-of=csi-driver-host-path --wait=true
+kubectl delete role,clusterrole,rolebinding,clusterrolebinding,serviceaccount,storageclass,csidriver,volumesnapshotclass --all-namespaces -l app.kubernetes.io/instance=hostpath.csi.k8s.io,app.kubernetes.io/part-of=csi-driver-host-path --wait=true

deploy/kubernetes-distributed/hostpath/csi-hostpath-plugin.yaml

@@ -1,3 +1,86 @@
+# All of the individual sidecar RBAC roles get bound
+# to this account.
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: csi-hostpathplugin-sa
+  namespace: default
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpathplugin
+    app.kubernetes.io/component: serviceaccount
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpathplugin
+    app.kubernetes.io/component: provisioner-cluster-role
+  name: csi-hostpathplugin-provisioner-cluster-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: external-provisioner-runner
+subjects:
+- kind: ServiceAccount
+  name: csi-hostpathplugin-sa
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpathplugin
+    app.kubernetes.io/component: snapshotter-cluster-role
+  name: csi-hostpathplugin-snapshotter-cluster-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: external-snapshotter-runner
+subjects:
+- kind: ServiceAccount
+  name: csi-hostpathplugin-sa
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpathplugin
+    app.kubernetes.io/component: provisioner-role
+  name: csi-hostpathplugin-provisioner-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: external-provisioner-cfg
+subjects:
+- kind: ServiceAccount
+  name: csi-hostpathplugin-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpathplugin
+    app.kubernetes.io/component: snapshotter-role
+  name: csi-hostpathplugin-snapshotter-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: external-snapshotter-leaderelection
+subjects:
+- kind: ServiceAccount
+  name: csi-hostpathplugin-sa
+---
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:
@@ -22,7 +105,7 @@ spec:
         app.kubernetes.io/name: csi-hostpathplugin
         app.kubernetes.io/component: plugin
     spec:
-      serviceAccountName: csi-provisioner
+      serviceAccountName: csi-hostpathplugin-sa
       containers:
         - name: csi-provisioner
           image: registry.k8s.io/sig-storage/csi-provisioner:v6.2.0
@@ -137,6 +220,27 @@ spec:
           - --csi-address=/csi/csi.sock
           - --health-port=9898
 
+        - name: csi-snapshotter
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.4.0
+          args:
+            - -v=5
+            - --csi-address=/csi/csi.sock
+            - --node-deployment=true
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          securityContext:
+            # This is necessary only for systems with SELinux, where
+            # non-privileged sidecar containers cannot access unix domain socket
+            # created by privileged CSI driver container.
+            privileged: true
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+
       volumes:
         - hostPath:
             path: /var/lib/kubelet/plugins/csi-hostpath

deploy/kubernetes-distributed/hostpath/csi-hostpath-snapshotclass.yaml

@@ -0,0 +1,13 @@
+# Usage of the v1 API implies that the cluster must have
+# external-snapshotter v4.x installed.
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-hostpath-snapclass
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpath-snapclass
+    app.kubernetes.io/component: volumesnapshotclass
+driver: hostpath.csi.k8s.io #csi-hostpath
+deletionPolicy: Delete