Skip to content

Commit 382c527

Browse files
aravindavkaravindavk
authored
csi: Mounted Block Volume support (kadalu#642)
Create a Storage Class with `pv_type: Block` and use that Storage Class to claim PVs. Example Storage Class ``` kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: kadalu.storage-pool-1-block provisioner: kadalu parameters: storage_name: "storage-pool-1" pv_type: Block ``` Signed-off-by: Aravinda Vishwanathapura <[email protected]>
1 parent 910afaf commit 382c527

12 files changed

+270
-31
lines changed

‎csi/controllerserver.py

+29-16
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from kadalulib import CommandException, logf, send_analytics_tracker, execute
1414
from volumeutils import (HOSTVOL_MOUNTDIR, PV_TYPE_SUBVOL, PV_TYPE_VIRTBLOCK,
1515
check_external_volume, create_subdir_volume,
16-
create_virtblock_volume, delete_volume, expand_volume,
16+
create_block_volume, delete_volume, expand_volume,
1717
get_pv_hosting_volumes, is_hosting_volume_free,
1818
mount_and_select_hosting_volume, search_volume,
1919
unmount_glusterfs, update_free_size,
@@ -70,6 +70,12 @@ def execute_gluster_quota_command(privkey, user, host, gvolname, path, size):
7070
return None
7171

7272

73+
def pvc_access_mode(request):
74+
"""Fetch Access modes from Volume capabilities"""
75+
for vol_capability in request.volume_capabilities:
76+
return vol_capability.access_mode.mode
77+
78+
7379
class ControllerServer(csi_pb2_grpc.ControllerServicer):
7480
"""
7581
ControllerServer object is responsible for handling host
@@ -112,17 +118,24 @@ def CreateVolume(self, request, context):
112118
pvsize = request.capacity_range.required_bytes
113119

114120
pvtype = PV_TYPE_SUBVOL
115-
# 'latest' finds a place here, because only till 0.5.0 version
116-
# we had 'latest' as a separate version. After that, 'latest' is
117-
# just a link to latest version.
118-
if KADALU_VERSION in ["0.5.0", "0.4.0", "0.3.0"]:
119-
for vol_capability in request.volume_capabilities:
120-
# using getattr to avoid Pylint error
121-
single_node_writer = getattr(csi_pb2.VolumeCapability.AccessMode,
122-
"SINGLE_NODE_WRITER")
123-
124-
if vol_capability.access_mode.mode == single_node_writer:
125-
pvtype = PV_TYPE_VIRTBLOCK
121+
122+
# Mounted BlockVolume is requested via Storage Class.
123+
# GlusterFS File Volume may not be useful for some workloads
124+
# they can request for the Virtual Block formated and mounted
125+
# as default MountVolume.
126+
if request.parameters.get("pv_type", "").lower() == "block":
127+
pvtype = PV_TYPE_VIRTBLOCK
128+
129+
single_node_writer = getattr(csi_pb2.VolumeCapability.AccessMode,
130+
"SINGLE_NODE_WRITER")
131+
132+
# Multi node writer is not allowed for PV_TYPE_VIRTBLOCK
133+
if pvc_access_mode(request) != single_node_writer:
134+
errmsg = "Only SINGLE_NODE_WRITER is allowed for block Volume"
135+
logging.error(errmsg)
136+
context.set_details(errmsg)
137+
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
138+
return csi_pb2.CreateVolumeResponse()
126139

127140
logging.debug(logf(
128141
"Found PV type",
@@ -235,8 +248,8 @@ def CreateVolume(self, request, context):
235248
return csi_pb2.CreateVolumeResponse()
236249

237250
if pvtype == PV_TYPE_VIRTBLOCK:
238-
vol = create_virtblock_volume(
239-
mntdir, request.name, pvsize)
251+
vol = create_block_volume(
252+
pvtype, mntdir, request.name, pvsize)
240253
else:
241254
use_gluster_quota = False
242255
if (os.path.isfile("/etc/secret-volume/ssh-privatekey") \
@@ -335,8 +348,8 @@ def CreateVolume(self, request, context):
335348

336349
mntdir = os.path.join(HOSTVOL_MOUNTDIR, hostvol)
337350
if pvtype == PV_TYPE_VIRTBLOCK:
338-
vol = create_virtblock_volume(
339-
mntdir, request.name, pvsize)
351+
vol = create_block_volume(
352+
pvtype, mntdir, request.name, pvsize)
340353
else:
341354
use_gluster_quota = False
342355
vol = create_subdir_volume(

‎csi/nodeserver.py

+1
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ def NodePublishVolume(self, request, context):
116116
pvpath=pvpath,
117117
pvtype=pvtype,
118118
hostvol=hostvol,
119+
target_path=request.target_path,
119120
duration_seconds=time.time() - start_time
120121
))
121122
return csi_pb2.NodePublishVolumeResponse()

‎csi/volumeutils.py

+14-11
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
MOUNT_CMD = "/bin/mount"
2121
UNMOUNT_CMD = "/bin/umount"
2222
MKFS_XFS_CMD = "/sbin/mkfs.xfs"
23+
XFS_GROWFS_CMD = "/sbin/xfs_growfs"
2324
RESERVED_SIZE_PERCENTAGE = 10
2425
HOSTVOL_MOUNTDIR = "/mnt"
2526
VOLFILES_DIR = "/kadalu/volfiles"
@@ -245,10 +246,10 @@ def mount_and_select_hosting_volume(pv_hosting_volumes, required_size):
245246
return None
246247

247248

248-
def create_virtblock_volume(hostvol_mnt, volname, size):
249+
def create_block_volume(pvtype, hostvol_mnt, volname, size):
249250
"""Create virtual block volume"""
250251
volhash = get_volname_hash(volname)
251-
volpath = get_volume_path(PV_TYPE_VIRTBLOCK, volhash, volname)
252+
volpath = get_volume_path(pvtype, volhash, volname)
252253
volpath_full = os.path.join(hostvol_mnt, volpath)
253254
logging.debug(logf(
254255
"Volume hash",
@@ -261,7 +262,7 @@ def create_virtblock_volume(hostvol_mnt, volname, size):
261262
# Create a file with required size
262263
makedirs(os.path.dirname(volpath_full))
263264
logging.debug(logf(
264-
"Created virtblock directory",
265+
f"Created {pvtype} directory",
265266
path=os.path.dirname(volpath)
266267
))
267268

@@ -282,17 +283,19 @@ def create_virtblock_volume(hostvol_mnt, volname, size):
282283
size=size
283284
))
284285

285-
# TODO: Multiple FS support based on volume_capability mount option
286-
execute(MKFS_XFS_CMD, volpath_full)
287-
logging.debug(logf(
288-
"Created Filesystem",
289-
path=volpath,
290-
command=MKFS_XFS_CMD
291-
))
286+
if pvtype == PV_TYPE_VIRTBLOCK:
287+
# TODO: Multiple FS support based on volume_capability mount option
288+
execute(MKFS_XFS_CMD, volpath_full)
289+
logging.debug(logf(
290+
"Created Filesystem",
291+
path=volpath,
292+
command=MKFS_XFS_CMD
293+
))
294+
292295
save_pv_metadata(hostvol_mnt, volpath, size)
293296
return Volume(
294297
volname=volname,
295-
voltype=PV_TYPE_VIRTBLOCK,
298+
voltype=pvtype,
296299
volhash=volhash,
297300
hostvol=os.path.basename(hostvol_mnt),
298301
size=size,

‎doc/README.adoc

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ One more blog to check is link:https://thoughtexpo.com/tags/kadalu/[Blogs @ Thou
2424
* link:try-kadalu.adoc[Try Kadalu]
2525
* link:external-gluster-storage.adoc[External Gluster Storage]
2626
* link:storage-config-options.adoc[Storage Config Options]
27+
* link:mounted-block-volume.adoc[Mounted Block Volume]
2728
* link:storage-classes.adoc[Storage Classes]
2829
* link:troubleshooting.adoc[Troubleshooting]
2930

‎doc/index.yml

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
- title: Try Kadalu
99
- title: External Gluster Storage
1010
- title: Storage Config Options
11+
- title: Mounted Block Volume
1112
- title: Storage Classes
1213
- title: Troubleshooting
1314
- section: Use cases

‎doc/mounted-block-volume.adoc

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
= Mounted Block Volumes
2+
3+
For some workloads, using File Volume may not be performant. Kadalu provides an option to use block volume(PV) but as mounted Volume.
4+
5+
Create a new Storage Class with `pv_type: Block` so that Provisioner creates Block PV.
6+
7+
[source,yaml]
8+
----
9+
kind: StorageClass
10+
apiVersion: storage.k8s.io/v1
11+
metadata:
12+
name: kadalu.db
13+
provisioner: kadalu
14+
parameters:
15+
storage_name: "storage-pool-1"
16+
pv_type: Block
17+
----
18+
19+
Above Storage Class adds filter for Storage pool name `storage-pool-1`. All the Persistent Volume claims using this Storage class will be processed as Block Volumes. Refer link:./storage-classes.adoc[Storage Classes] documentation for using more options.
20+
21+
Virtual Block PVs will be formatted using `mkfs.xfs` command.
22+
23+
*NOTE*: Only `ReadWriteOnce` access mode is allowed while using this Storage Class.

‎doc/storage-classes.adoc

+33
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,39 @@ spec:
202202
storage: 1Gi
203203
----
204204

205+
=== pv_type
206+
207+
Use this option if the PV created using this Storage Class should be a Mounted Block Volume.
208+
209+
[source,yaml]
210+
----
211+
kind: StorageClass
212+
apiVersion: storage.k8s.io/v1
213+
metadata:
214+
name: kadalu.db-storage
215+
provisioner: kadalu
216+
parameters:
217+
storage_name: "storage-pool-1"
218+
pv_type: Block
219+
----
220+
221+
And the PVC,
222+
223+
[source,yaml]
224+
----
225+
kind: PersistentVolumeClaim
226+
apiVersion: v1
227+
metadata:
228+
name: pv8
229+
spec:
230+
storageClassName: kadalu.db-storage
231+
accessModes:
232+
- ReadWriteOnce
233+
resources:
234+
requests:
235+
storage: 1Gi
236+
----
237+
205238
The number of customization a Storage Class can provide is
206239
impressive. The only limit is your imagination. Please open a new
207240
https://github.com/kadalu/kadalu/issues[issue] if your use case

‎examples/sample-test-app1.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,44 @@ spec:
6565
persistentVolumeClaim:
6666
claimName: pv1-2
6767
restartPolicy: OnFailure
68+
---
69+
kind: StorageClass
70+
apiVersion: storage.k8s.io/v1
71+
metadata:
72+
name: kadalu.storage-pool-1-block
73+
provisioner: kadalu
74+
parameters:
75+
storage_name: "storage-pool-1"
76+
pv_type: Block
77+
---
78+
kind: PersistentVolumeClaim
79+
apiVersion: v1
80+
metadata:
81+
name: pv1-3
82+
spec:
83+
storageClassName: kadalu.storage-pool-1-block
84+
accessModes:
85+
- ReadWriteOnce
86+
resources:
87+
requests:
88+
storage: 500Mi
89+
---
90+
apiVersion: v1
91+
kind: Pod
92+
metadata:
93+
name: pod1-3
94+
labels:
95+
app: sample-app
96+
spec:
97+
containers:
98+
- name: sample-app
99+
image: docker.io/kadalu/sample-pv-check-app:latest
100+
imagePullPolicy: IfNotPresent
101+
volumeMounts:
102+
- mountPath: "/mnt/pv"
103+
name: csivol2
104+
volumes:
105+
- name: csivol2
106+
persistentVolumeClaim:
107+
claimName: pv1-3
108+
restartPolicy: OnFailure

‎examples/sample-test-app2.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,44 @@ spec:
6666
persistentVolumeClaim:
6767
claimName: pv2-2
6868
restartPolicy: OnFailure
69+
---
70+
kind: StorageClass
71+
apiVersion: storage.k8s.io/v1
72+
metadata:
73+
name: kadalu.storage-pool-2-block
74+
provisioner: kadalu
75+
parameters:
76+
storage_name: "storage-pool-2"
77+
pv_type: Block
78+
---
79+
kind: PersistentVolumeClaim
80+
apiVersion: v1
81+
metadata:
82+
name: pv2-3
83+
spec:
84+
storageClassName: kadalu.storage-pool-2-block
85+
accessModes:
86+
- ReadWriteOnce
87+
resources:
88+
requests:
89+
storage: 500Mi
90+
---
91+
apiVersion: v1
92+
kind: Pod
93+
metadata:
94+
name: pod2-3
95+
labels:
96+
app: sample-app
97+
spec:
98+
containers:
99+
- name: sample-app
100+
image: docker.io/kadalu/sample-pv-check-app:latest
101+
imagePullPolicy: IfNotPresent
102+
volumeMounts:
103+
- mountPath: "/mnt/pv"
104+
name: csivol
105+
volumes:
106+
- name: csivol
107+
persistentVolumeClaim:
108+
claimName: pv2-3
109+
restartPolicy: OnFailure

‎examples/sample-test-app3.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,44 @@ spec:
6666
persistentVolumeClaim:
6767
claimName: pv3-2
6868
restartPolicy: OnFailure
69+
---
70+
kind: StorageClass
71+
apiVersion: storage.k8s.io/v1
72+
metadata:
73+
name: kadalu.storage-pool-3-block
74+
provisioner: kadalu
75+
parameters:
76+
storage_name: "storage-pool-3"
77+
pv_type: Block
78+
---
79+
kind: PersistentVolumeClaim
80+
apiVersion: v1
81+
metadata:
82+
name: pv3-3
83+
spec:
84+
storageClassName: kadalu.storage-pool-3-block
85+
accessModes:
86+
- ReadWriteOnce
87+
resources:
88+
requests:
89+
storage: 500Mi
90+
---
91+
apiVersion: v1
92+
kind: Pod
93+
metadata:
94+
name: pod3-3
95+
labels:
96+
app: sample-app
97+
spec:
98+
containers:
99+
- name: sample-app
100+
image: docker.io/kadalu/sample-pv-check-app:latest
101+
imagePullPolicy: IfNotPresent
102+
volumeMounts:
103+
- mountPath: "/mnt/pv"
104+
name: csivol
105+
volumes:
106+
- name: csivol
107+
persistentVolumeClaim:
108+
claimName: pv3-3
109+
restartPolicy: OnFailure

0 commit comments

Comments
 (0)