test: use block devices for I/O limit tests

contrib/cirrus/setup_environment.sh

@@ -209,6 +209,12 @@ mount -t tmpfs -o size=75%,mode=0700 none /var/lib/containers
 showrun echo "Setting CI_DESIRED_STORAGE [=$CI_DESIRED_STORAGE] for *e2e* tests"
 echo "STORAGE_FS=$CI_DESIRED_STORAGE" >>/etc/ci_environment
 
+if ((CONTAINER==0)); then  # not yet inside a container
+    # Load null_blk to use /dev/nullb0 for testing block
+    # devices limits
+    modprobe null_blk nr_devices=1 || :
+fi
+
 # Required to be defined by caller: The environment where primary testing happens
 # shellcheck disable=SC2154
 showrun echo "about to set up for TEST_ENVIRON [=$TEST_ENVIRON]"

libpod/container_inspect.go

@@ -653,7 +653,7 @@ func (c *Container) GetDevices(priv bool, ctrSpec spec.Spec, deviceNodes map[str
 		for _, dev := range ctrSpec.Linux.Devices {
 			key := fmt.Sprintf("%d:%d", dev.Major, dev.Minor)
 			if deviceNodes == nil {
-				nodes, err := util.FindDeviceNodes()
+				nodes, err := util.FindDeviceNodes(false)
 				if err != nil {
 					return nil, err
 				}
@@ -678,7 +678,7 @@ func blkioDeviceThrottle(deviceNodes map[string]string, devs []spec.LinuxThrottl
 	for _, dev := range devs {
 		key := fmt.Sprintf("%d:%d", dev.Major, dev.Minor)
 		if deviceNodes == nil {
-			nodes, err := util.FindDeviceNodes()
+			nodes, err := util.FindDeviceNodes(true)
 			if err != nil {
 				return nil, err
 			}

libpod/container_inspect_linux.go

@@ -89,7 +89,7 @@ func (c *Container) platformInspectContainerHostConfig(ctrSpec *spec.Spec, hostC
 						continue
 					}
 					if deviceNodes == nil {
-						nodes, err := util.FindDeviceNodes()
+						nodes, err := util.FindDeviceNodes(true)
 						if err != nil {
 							return err
 						}

pkg/util/utils_linux.go

@@ -33,7 +33,8 @@ func GetContainerPidInformationDescriptors() ([]string, error) {
 // [major:minor] is the device's major and minor numbers formatted as, for
 // example, 2:0 and path is the path to the device node.
 // Symlinks to nodes are ignored.
-func FindDeviceNodes() (map[string]string, error) {
+// If onlyBlockDevices is specified, character devices are ignored.
+func FindDeviceNodes(onlyBlockDevices bool) (map[string]string, error) {
 	nodes := make(map[string]string)
 	err := filepath.WalkDir("/dev", func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
@@ -44,7 +45,13 @@ func FindDeviceNodes() (map[string]string, error) {
 		}
 
 		// If we aren't a device node, do nothing.
-		if d.Type()&(os.ModeDevice|os.ModeCharDevice) == 0 {
+		if d.Type()&os.ModeDevice == 0 {
+			return nil
+		}
+
+		// Ignore character devices, because it is not possible to set limits on them.
+		// os.ModeCharDevice is usable only when os.ModeDevice is set.
+		if onlyBlockDevices && d.Type()&os.ModeCharDevice != 0 {
 			return nil
 		}
 

pkg/util/utils_unsupported.go

@@ -5,6 +5,6 @@ package util
 import "errors"
 
 // FindDeviceNodes is not implemented anywhere except Linux.
-func FindDeviceNodes() (map[string]string, error) {
+func FindDeviceNodes(onlyBlockDevices bool) (map[string]string, error) {
 	return nil, errors.New("not supported on non-Linux OSes")
 }

test/apiv2/20-containers.at

@@ -710,15 +710,15 @@ t GET libpod/containers/$cname/json 200 \
   .ImageName=$IMAGE \
   .Name=$cname
 
-if root; then
+if root && test -e /dev/nullb0; then
   podman run -dt --name=updateCtr alpine
   echo '{
     "Memory":{"Limit":500000},
     "CPU":{"Shares":123},
-    "DeviceReadBPs": [{ "Path": "/dev/zero", "Rate": 10485760 }],
-    "DeviceWriteBPs": [{ "Path": "/dev/zero", "Rate": 31457280 }],
-    "DeviceReadIOPs": [{ "Path": "/dev/zero", "Rate": 2000 }],
-    "DeviceWriteIOPs": [{ "Path": "/dev/zero", "Rate": 4000 }]
+    "DeviceReadBPs": [{ "Path": "/dev/nullb0", "Rate": 10485760 }],
+    "DeviceWriteBPs": [{ "Path": "/dev/nullb0", "Rate": 31457280 }],
+    "DeviceReadIOPs": [{ "Path": "/dev/nullb0", "Rate": 2000 }],
+    "DeviceWriteIOPs": [{ "Path": "/dev/nullb0", "Rate": 4000 }]
     }' >${TMPD}/update.json
   t POST libpod/containers/updateCtr/update ${TMPD}/update.json 201
 
@@ -734,25 +734,25 @@ if root; then
 
   BlkioDeviceReadBps_expected='[
   {
-    "Path": "/dev/zero",
+    "Path": "/dev/nullb0",
     "Rate": 10485760
   }
 ]'
   BlkioDeviceWriteBPs_expected='[
   {
-    "Path": "/dev/zero",
+    "Path": "/dev/nullb0",
     "Rate": 31457280
   }
 ]'
   BlkioDeviceReadIOPs_expected='[
   {
-    "Path": "/dev/zero",
+    "Path": "/dev/nullb0",
     "Rate": 2000
   }
 ]'
   BlkioDeviceWriteIOPs_expected='[
   {
-    "Path": "/dev/zero",
+    "Path": "/dev/nullb0",
     "Rate": 4000
   }
 ]'

test/e2e/common_test.go

@@ -942,6 +942,13 @@ func SkipIfNotRootless(reason string) {
 	}
 }
 
+func SkipIfNotExist(reason, path string) {
+	checkReason(reason)
+	if _, err := os.Stat(path); err != nil {
+		Skip("[doesNotExist]: " + path + " does not exist: " + reason)
+	}
+}
+
 func SkipIfSystemdNotRunning(reason string) {
 	checkReason(reason)
 
@@ -1648,3 +1655,7 @@ func makeTempDirInDir(dir string) string {
 	Expect(err).ToNot(HaveOccurred())
 	return path
 }
+
+func skipWithoutDevNullb0() {
+	SkipIfNotExist("use modprobe null_blk nr_devices=1 to create it", "/dev/nullb0")
+}

test/e2e/run_test.go

@@ -848,13 +848,14 @@ USER bin`, BB)
 
 	It("podman run device-read-bps test", func() {
 		SkipIfRootless("Setting device-read-bps not supported for rootless users")
+		skipWithoutDevNullb0()
 
 		var session *PodmanSessionIntegration
 
 		if CGROUPSV2 {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/zero:1mb", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/nullb0:1mb", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
 		} else {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/zero:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/nullb0:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"})
 		}
 
 		session.WaitWithDefaultTimeout()
@@ -866,13 +867,14 @@ USER bin`, BB)
 
 	It("podman run device-write-bps test", func() {
 		SkipIfRootless("Setting device-write-bps not supported for rootless users")
+		skipWithoutDevNullb0()
 
 		var session *PodmanSessionIntegration
 
 		if CGROUPSV2 {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/zero:1mb", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/nullb0:1mb", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
 		} else {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/zero:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/nullb0:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"})
 		}
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
@@ -883,12 +885,14 @@ USER bin`, BB)
 
 	It("podman run device-read-iops test", func() {
 		SkipIfRootless("Setting device-read-iops not supported for rootless users")
+		skipWithoutDevNullb0()
+
 		var session *PodmanSessionIntegration
 
 		if CGROUPSV2 {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/zero:100", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/nullb0:100", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
 		} else {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/zero:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/nullb0:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"})
 		}
 
 		session.WaitWithDefaultTimeout()
@@ -900,12 +904,14 @@ USER bin`, BB)
 
 	It("podman run device-write-iops test", func() {
 		SkipIfRootless("Setting device-write-iops not supported for rootless users")
+		skipWithoutDevNullb0()
+
 		var session *PodmanSessionIntegration
 
 		if CGROUPSV2 {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/zero:100", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/nullb0:100", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
 		} else {
-			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/zero:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"})
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/nullb0:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"})
 		}
 
 		session.WaitWithDefaultTimeout()

test/e2e/update_test.go

@@ -87,6 +87,7 @@ var _ = Describe("Podman update", func() {
 	It("podman update container all options v2", func() {
 		SkipIfCgroupV1("testing flags that only work in cgroup v2")
 		SkipIfRootless("many of these handlers are not enabled while rootless in CI")
+		skipWithoutDevNullb0()
 		session := podmanTest.Podman([]string{"run", "-dt", ALPINE})
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(ExitCleanly())
@@ -103,10 +104,10 @@ var _ = Describe("Podman update", func() {
 			"--memory-swap", "2G",
 			"--memory-reservation", "2G",
 			"--blkio-weight", "123",
-			"--device-read-bps", "/dev/zero:10mb",
-			"--device-write-bps", "/dev/zero:10mb",
-			"--device-read-iops", "/dev/zero:1000",
-			"--device-write-iops", "/dev/zero:1000",
+			"--device-read-bps", "/dev/nullb0:10mb",
+			"--device-write-bps", "/dev/nullb0:10mb",
+			"--device-read-iops", "/dev/nullb0:1000",
+			"--device-write-iops", "/dev/nullb0:1000",
 			"--pids-limit", "123",
 			ctrID}
 

test/system/030-run.bats

@@ -1123,18 +1123,24 @@ EOF
 @test "podman run --device-read-bps" {
     skip_if_rootless "cannot use this flag in rootless mode"
 
+    if test \! -e /dev/nullb0; then
+        skip "/dev/nullb0 not present, use 'modprobe null_blk nr_devices=1' to create it"
+    fi
+
     local cid
+    local dev_maj_min=$(stat -c %Hr:%Lr /dev/nullb0)
+
     # this test is a triple check on blkio flags since they seem to sneak by the tests
     if is_cgroupsv2; then
-        run_podman run -dt --device-read-bps=/dev/zero:1M $IMAGE top
+        run_podman run -dt --device-read-bps=/dev/nullb0:1M $IMAGE top
         cid=$output
         run_podman exec -it $output cat /sys/fs/cgroup/io.max
-        is "$output" ".*1:5 rbps=1048576 wbps=max riops=max wiops=max" "throttle devices passed successfully.*"
+        is "$output" ".*$dev_maj_min rbps=1048576 wbps=max riops=max wiops=max" "throttle devices passed successfully.*"
     else
-        run_podman run -dt --device-read-bps=/dev/zero:1M $IMAGE top
+        run_podman run -dt --device-read-bps=/dev/nullb0:1M $IMAGE top
         cid=$output
         run_podman exec -it $output cat /sys/fs/cgroup/blkio/blkio.throttle.read_bps_device
-        is "$output" ".*1:5 1048576" "throttle devices passed successfully.*"
+        is "$output" ".*$dev_maj_min 1048576" "throttle devices passed successfully.*"
     fi
     run_podman container rm -f -t0 $cid
 }

test/system/280-update.bats

@@ -63,10 +63,10 @@ memory-reservation  =           400M | 2 | $mm.soft_limit_in_bytes  = 419430400
 blkio-weight        =            321 | - | -                                       | io.bfq.weight   = default 321 $lomajmin 98
 blkio-weight-device = $LOOPDEVICE:98 | - | -                                       | io.bfq.weight   = default 321 $lomajmin 98
 
-device-read-bps     = /dev/zero:10mb | - | -                                       | io.max          = $devicemax
-device-read-iops    = /dev/zero:2000 | - | -                                       | io.max          = $devicemax
-device-write-bps    = /dev/zero:30mb | - | -                                       | io.max          = $devicemax
-device-write-iops   = /dev/zero:4000 | - | -                                       | io.max          = $devicemax
+device-read-bps     = $LOOPDEVICE:10mb | - | -                                       | io.max          = $devicemax
+device-read-iops    = $LOOPDEVICE:2000 | - | -                                       | io.max          = $devicemax
+device-write-bps    = $LOOPDEVICE:30mb | - | -                                       | io.max          = $devicemax
+device-write-iops   = $LOOPDEVICE:4000 | - | -                                       | io.max          = $devicemax
 "
 
     # Run a container