diff --git a/deploy/kubernetes-1.15-no-attacher/README.md b/deploy/kubernetes-1.15-no-attacher/README.md new file mode 100644 index 000000000..64ff75360 --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/README.md @@ -0,0 +1,9 @@ +The deployment for Kubernetes 1.15 uses CSI 1.0 and thus is +incompatible with Kubernetes < 1.13. + +The sidecars depend on 1.15 API changes for migration and resizing, +and 1.14 API changes for CSIDriver and CSINode. + +In contrast to the normal deployment, this deployment does not use the +external-attacher sidecar. Instead, it creates a CSIDriver instance +that tells Kubernetes not to use attach (ControllerPublishVolume) at all. diff --git a/deploy/kubernetes-1.15-no-attacher/deploy-hostpath.sh b/deploy/kubernetes-1.15-no-attacher/deploy-hostpath.sh new file mode 120000 index 000000000..2a11fbd3c --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/deploy-hostpath.sh @@ -0,0 +1 @@ +../util/deploy-hostpath-no-attacher.sh \ No newline at end of file diff --git a/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-csidriver.yaml b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-csidriver.yaml new file mode 100644 index 000000000..bd610330d --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-csidriver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: hostpath.csi.k8s.io +spec: + attachRequired: false + podInfoOnMount: false diff --git a/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-plugin.yaml new file mode 100644 index 000000000..29324d793 --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-plugin.yaml @@ -0,0 +1,138 @@ +# Service defined here, plus serviceName below in StatefulSet, +# are needed only because of condition explained in +# https://github.com/kubernetes/kubernetes/issues/69608 + +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpathplugin + labels: + app: csi-hostpathplugin +spec: + selector: + app: csi-hostpathplugin + ports: + - name: dummy + port: 12345 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpathplugin +spec: + serviceName: "csi-hostpathplugin" + # One replica only: + # Host path driver only works when everything runs + # on a single node. We achieve that by starting it once and then + # co-locate all other pods via inter-pod affinity + replicas: 1 + selector: + matchLabels: + app: csi-hostpathplugin + template: + metadata: + labels: + app: csi-hostpathplugin + spec: + hostNetwork: true + containers: + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"] + args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock + securityContext: + privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - mountPath: /csi-data-dir + name: csi-data-dir + + - name: hostpath + image: quay.io/k8scsi/hostpathplugin:v1.1.0 + args: + - "--drivername=hostpath.csi.k8s.io" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=9898 + + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins + type: Directory + name: plugins-dir + - hostPath: + # 'path' is where PV data is persisted on host. + # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot + path: /var/lib/csi-hostpath-data/ + type: DirectoryOrCreate + name: csi-data-dir diff --git a/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-provisioner.yaml b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-provisioner.yaml new file mode 100644 index 000000000..ae1b770c8 --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-provisioner.yaml @@ -0,0 +1,55 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpath-provisioner + labels: + app: csi-hostpath-provisioner +spec: + selector: + app: csi-hostpath-provisioner + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-provisioner +spec: + serviceName: "csi-hostpath-provisioner" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-provisioner + template: + metadata: + labels: + app: csi-hostpath-provisioner + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + serviceAccountName: csi-provisioner + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.3.0 + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --connection-timeout=15s + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-snapshotter.yaml b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-snapshotter.yaml new file mode 100644 index 000000000..6a1e35e00 --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-snapshotter.yaml @@ -0,0 +1,55 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpath-snapshotter + labels: + app: csi-hostpath-snapshotter +spec: + selector: + app: csi-hostpath-snapshotter + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-snapshotter +spec: + serviceName: "csi-hostpath-snapshotter" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-snapshotter + template: + metadata: + labels: + app: csi-hostpath-snapshotter + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + serviceAccount: csi-snapshotter + containers: + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v1.2.0 + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --connection-timeout=15s + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-testing.yaml b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-testing.yaml new file mode 100644 index 000000000..3e6b837d1 --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/hostpath/csi-hostpath-testing.yaml @@ -0,0 +1,59 @@ +# WARNING: this is only for testing purposes. Do not install in a production +# cluster. +# +# This exposes the hostpath's Unix domain csi.sock as a TCP port to the +# outside world. The mapping from Unix domain socket to TCP is done +# by socat. +# +# This is useful for testing with csi-sanity or csc. + +apiVersion: v1 +kind: Service +metadata: + name: hostpath-service +spec: + type: NodePort + selector: + app: csi-hostpath-socat + ports: + - port: 10000 # fixed port inside the pod, dynamically allocated port outside +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-socat +spec: + serviceName: "csi-hostpath-socat" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-socat + template: + metadata: + labels: + app: csi-hostpath-socat + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + containers: + - name: socat + image: alpine/socat:1.0.3 + args: + - tcp-listen:10000,fork,reuseaddr + - unix-connect:/csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.15-no-attacher/snapshotter/csi-hostpath-snapshotclass.yaml b/deploy/kubernetes-1.15-no-attacher/snapshotter/csi-hostpath-snapshotclass.yaml new file mode 100644 index 000000000..e222c1783 --- /dev/null +++ b/deploy/kubernetes-1.15-no-attacher/snapshotter/csi-hostpath-snapshotclass.yaml @@ -0,0 +1,5 @@ +apiVersion: snapshot.storage.k8s.io/v1alpha1 +kind: VolumeSnapshotClass +metadata: + name: csi-hostpath-snapclass +snapshotter: hostpath.csi.k8s.io diff --git a/deploy/util/deploy-hostpath-no-attacher.sh b/deploy/util/deploy-hostpath-no-attacher.sh new file mode 100755 index 000000000..7ce85974f --- /dev/null +++ b/deploy/util/deploy-hostpath-no-attacher.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash + +# This script captures the steps required to successfully +# deploy the hostpath plugin driver. This should be considered +# authoritative and all updates for this process should be +# done here and referenced elsewhere. + +# The script assumes that kubectl is available on the OS path +# where it is executed. + +set -e +set -o pipefail + +BASE_DIR=$(dirname "$0") + +# If set, the following env variables override image registry and/or tag for each of the images. +# They are named after the image name, with hyphen replaced by underscore and in upper case. +# +# - CSI_NODE_DRIVER_REGISTRAR_REGISTRY +# - CSI_NODE_DRIVER_REGISTRAR_TAG +# - CSI_PROVISIONER_REGISTRY +# - CSI_PROVISIONER_TAG +# - CSI_SNAPSHOTTER_REGISTRY +# - CSI_SNAPSHOTTER_TAG +# - HOSTPATHPLUGIN_REGISTRY +# - HOSTPATHPLUGIN_TAG +# +# Alternatively, it is possible to override all registries or tags with: +# - IMAGE_REGISTRY +# - IMAGE_TAG +# These are used as fallback when the more specific variables are unset or empty. +# +# Beware that the .yaml files do not have "imagePullPolicy: Always". That means that +# also the "canary" images will only be pulled once. This is good for testing +# (starting a pod multiple times will always run with the same canary image), but +# implies that refreshing that image has to be done manually. +# +# As a special case, 'none' as registry removes the registry name. + +# The default is to use the RBAC rules that match the image that is +# being used, also in the case that the image gets overridden. This +# way if there are breaking changes in the RBAC rules, the deployment +# will continue to work. +# +# However, such breaking changes should be rare and only occur when updating +# to a new major version of a sidecar. Nonetheless, to allow testing the scenario +# where the image gets overridden but not the RBAC rules, updating the RBAC +# rules can be disabled. +: ${UPDATE_RBAC_RULES:=true} +function rbac_version () { + yaml="$1" + image="$2" + update_rbac="$3" + + # get version from `image: quay.io/k8scsi/csi-attacher:v1.0.1`, ignoring comments + version="$(sed -e 's/ *#.*$//' "$yaml" | grep "image:.*$image" | sed -e 's/ *#.*//' -e 's/.*://')" + if $update_rbac; then + # apply overrides + varname=$(echo $image | tr - _ | tr a-z A-Z) + eval version=\${${varname}_TAG:-\${IMAGE_TAG:-\$version}} + fi + + # When using canary images, we have to assume that the + # canary images were built from the corresponding branch. + case "$version" in canary) version=master;; + *-canary) version="$(echo "$version" | sed -e 's/\(.*\)-canary/release-\1/')";; + esac + + echo "$version" +} + +# In addition, the RBAC rules can be overridden separately. +CSI_PROVISIONER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner false)/deploy/kubernetes/rbac.yaml" +: ${CSI_PROVISIONER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} +CSI_SNAPSHOTTER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter false)/deploy/kubernetes/rbac.yaml" +: ${CSI_SNAPSHOTTER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} + +INSTALL_CRD=${INSTALL_CRD:-"false"} + +# Some images are not affected by *_REGISTRY/*_TAG and IMAGE_* variables. +# The default is to update unless explicitly excluded. +update_image () { + case "$1" in socat) return 1;; esac +} + +run () { + echo "$@" >&2 + "$@" +} + +# rbac rules +echo "applying RBAC rules" +for component in CSI_PROVISIONER CSI_SNAPSHOTTER; do + eval current="\${${component}_RBAC}" + eval original="\${${component}_RBAC_YAML}" + if [ "$current" != "$original" ]; then + echo "Using non-default RBAC rules for $component. Changes from $original to $current are:" + diff -c <(wget --quiet -O - "$original") <(if [[ "$current" =~ ^http ]]; then wget --quiet -O - "$current"; else cat "$current"; fi) || true + fi + run kubectl apply -f "${current}" +done + +# deploy hostpath plugin and registrar sidecar +echo "deploying hostpath components" +for i in $(ls ${BASE_DIR}/hostpath/*.yaml | sort); do + echo " $i" + modified="$(cat "$i" | while IFS= read -r line; do + nocomments="$(echo "$line" | sed -e 's/ *#.*$//')" + if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then + # Split 'image: quay.io/k8scsi/csi-attacher:v1.0.1' + # into image (quay.io/k8scsi/csi-attacher:v1.0.1), + # registry (quay.io/k8scsi), + # name (csi-attacher), + # tag (v1.0.1). + image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;') + registry=$(echo "$image" | sed -e 's;\(.*\)/.*;\1;') + name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;') + tag=$(echo "$image" | sed -e 's;.*:;;') + + # Variables are with underscores and upper case. + varname=$(echo $name | tr - _ | tr a-z A-Z) + + # Now replace registry and/or tag, if set as env variables. + # If not set, the replacement is the same as the original value. + # Only do this for the images which are meant to be configurable. + if update_image "$name"; then + prefix=$(eval echo \${${varname}_REGISTRY:-${IMAGE_REGISTRY:-${registry}}}/ | sed -e 's;none/;;') + suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}}) + line="$(echo "$nocomments" | sed -e "s;$image;${prefix}${name}${suffix};")" + fi + echo " using $line" >&2 + fi + echo "$line" + done)" + if ! echo "$modified" | kubectl apply -f -; then + echo "modified version of $i:" + echo "$modified" + exit 1 + fi +done + +# Wait until all pods are running. We have to make some assumptions +# about the deployment here, otherwise we wouldn't know what to wait +# for: the expectation is that we run provisioner, snapshotter, +# socat and hostpath plugin in the default namespace. +cnt=0 +while [ $(kubectl get pods 2>/dev/null | grep '^csi-hostpath.* Running ' | wc -l) -lt 4 ] || ! kubectl describe volumesnapshotclasses.snapshot.storage.k8s.io 2>/dev/null >/dev/null; do + if [ $cnt -gt 30 ]; then + echo "Running pods:" + kubectl describe pods + + echo >&2 "ERROR: hostpath deployment not ready after over 5min" + exit 1 + fi + echo $(date +%H:%M:%S) "waiting for hostpath deployment to complete, attempt #$cnt" + cnt=$(($cnt + 1)) + sleep 10 +done + +# deploy snapshotclass +echo "deploying snapshotclass" +kubectl apply -f ${BASE_DIR}/snapshotter/csi-hostpath-snapshotclass.yaml