From 2155a24e73bca1972fba109b36e9944a3bd1fb5f Mon Sep 17 00:00:00 2001 From: Mikko Ylinen Date: Wed, 16 Dec 2020 17:20:05 +0200 Subject: [PATCH 1/2] qat: add new devices and change defaults The plugin now detects/accepts 4xxx and c4xxx devices too and defaults to those drivers that are part of Linux mainline. Signed-off-by: Mikko Ylinen --- cmd/qat_plugin/README.md | 4 ++-- cmd/qat_plugin/dpdkdrv/dpdkdrv.go | 4 ++-- cmd/qat_plugin/qat_plugin.go | 2 +- .../qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/qat_plugin/README.md b/cmd/qat_plugin/README.md index d963d910e..52933d7da 100644 --- a/cmd/qat_plugin/README.md +++ b/cmd/qat_plugin/README.md @@ -51,7 +51,7 @@ The QAT plugin can take a number of command line arguments, summarised in the fo | Flag | Argument | Meaning | |:---- |:-------- |:------- | | -dpdk-driver | string | DPDK Device driver for configuring the QAT device (default: `vfio-pci`) | -| -kernel-vf-drivers | string | Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC,C62x,C3xxx and D15xx (default: `dh895xccvf,c6xxvf,c3xxxvf,d15xxvf`) | +| -kernel-vf-drivers | string | Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx, C4xxx and D15xx (default: `dh895xccvf,c6xxvf,c3xxxvf`) | | -max-num-devices | int | maximum number of QAT devices to be provided to the QuickAssist device plugin (default: `32`) | | -mode | string | plugin mode which can be either `dpdk` or `kernel` (default: `dpdk`) | @@ -66,7 +66,7 @@ table summarises the defaults: | Argument | Variable | Default setting | Explanation | |:-------- |:-------- |:--------------- |:----------- | | -dpdk-driver | `$DPDK_DRIVER` | vfio-pci | A more robust and secure choice than the `igb_uio` alternative | -| -kernel-vf-drivers | `$KERNEL_VF_DRIVERS` | dh895xccvf,c6xxvf,c3xxxvf,d15xxvf | Modify to suit your hardware setup | +| -kernel-vf-drivers | `$KERNEL_VF_DRIVERS` | dh895xccvf,c6xxvf,c3xxxvf | Modify to suit your hardware setup | | -max-num-devices | `$MAX_NUM_DEVICES` | 32 | Modify to suit your hardware setup if necessary | For more details on the `-dpdk-driver` choice, see diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go index adfbecd02..e87ea1c6e 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go @@ -210,7 +210,7 @@ func (dp *DevicePlugin) bindDevice(vfBdf string) error { func isValidKerneDriver(kernelvfDriver string) bool { switch kernelvfDriver { - case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf": + case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf", "c4xxvf", "4xxxvf": return true } return false @@ -225,7 +225,7 @@ func isValidDpdkDeviceDriver(dpdkDriver string) bool { } func isValidVfDeviceID(vfDevID string) bool { switch vfDevID { - case "0442", "0443", "37c9", "19e3": + case "0442", "0443", "37c9", "19e3", "4941", "18a1": return true } return false diff --git a/cmd/qat_plugin/qat_plugin.go b/cmd/qat_plugin/qat_plugin.go index 9e00f8ddd..19d4b886b 100644 --- a/cmd/qat_plugin/qat_plugin.go +++ b/cmd/qat_plugin/qat_plugin.go @@ -38,7 +38,7 @@ func main() { mode := flag.String("mode", "dpdk", "plugin mode which can be either dpdk (default) or kernel") dpdkDriver := flag.String("dpdk-driver", "vfio-pci", "DPDK Device driver for configuring the QAT device") - kernelVfDrivers := flag.String("kernel-vf-drivers", "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC,C62x,C3xxx and D15xx") + kernelVfDrivers := flag.String("kernel-vf-drivers", "dh895xccvf,c6xxvf,c3xxxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, and D15xx") maxNumDevices := flag.Int("max-num-devices", 32, "maximum number of QAT devices to be provided to the QuickAssist device plugin") flag.Parse() diff --git a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml b/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml index 757f4ff51..2eeb4c99d 100644 --- a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml +++ b/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml @@ -21,7 +21,7 @@ spec: command: ["/bin/sh", "-c"] args: - > - for dev in `for pf in 0434 0435 19a3 37c8 6f54; do lspci | awk -v dev="8086:$pf" '$0 ~ dev {print "0000:" $1}'; done`; do + for dev in `for pf in 0434 0435 19a3 37c8 6f54 4940 18a0; do lspci | awk -v dev="8086:$pf" '$0 ~ dev {print "0000:" $1}'; done`; do DEVPATH="/sys/bus/pci/devices/$dev" NUMVFS="$DEVPATH/sriov_numvfs" if [ -w "$NUMVFS" -a $(cat "$NUMVFS") -eq 0 ]; then From 85fce2dcab3764dc59f28a276d9e808e56b83bba Mon Sep 17 00:00:00 2001 From: Mikko Ylinen Date: Wed, 16 Dec 2020 17:20:16 +0200 Subject: [PATCH 2/2] qat: rework device scanning The updated dp.scan() changes the way how VF devices are detected. The main reason for the change is to take into account cases where the QAT VF driver is not present in the system at all but only the PF driver is loaded (and the SR-IOV devices are are enabled). The rework also takes into account bare metal and VM deployments and adds a test case for checking the virtualized environment. Signed-off-by: Mikko Ylinen --- cmd/qat_plugin/dpdkdrv/dpdkdrv.go | 132 +++++++++---- cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go | 250 +++++++++++-------------- 2 files changed, 203 insertions(+), 179 deletions(-) diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go index e87ea1c6e..c5e167d13 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go @@ -67,7 +67,7 @@ func NewDevicePlugin(maxDevices int, kernelVfDrivers string, dpdkDriver string) kernelDrivers := strings.Split(kernelVfDrivers, ",") for _, driver := range kernelDrivers { - if !isValidKerneDriver(driver) { + if !isValidKernelDriver(driver) { return nil, errors.Errorf("wrong kernel VF driver: %s", driver) } } @@ -119,11 +119,11 @@ func (dp *DevicePlugin) getDpdkDevice(vfBdf string) (string, error) { return "", errors.WithStack(err) } s := filepath.Base(group) - klog.V(1).Infof("The vfio device group detected is %v", s) return s, nil - } - return "", errors.New("Unknown DPDK driver") + default: + return "", errors.New("Unknown DPDK driver") + } } func (dp *DevicePlugin) getDpdkDeviceSpecs(dpdkDeviceName string) []pluginapi.DeviceSpec { @@ -208,7 +208,7 @@ func (dp *DevicePlugin) bindDevice(vfBdf string) error { return nil } -func isValidKerneDriver(kernelvfDriver string) bool { +func isValidKernelDriver(kernelvfDriver string) bool { switch kernelvfDriver { case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf", "c4xxvf", "4xxxvf": return true @@ -245,56 +245,108 @@ func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error return nil } +func getPciDevicesWithPattern(pattern string) (pciDevices []string) { + pciDevices = make([]string, 0) + + devs, err := filepath.Glob(pattern) + if err != nil { + klog.Warningf("bad pattern: %s", pattern) + return + } + + for _, devBdf := range devs { + targetDev, err := filepath.EvalSymlinks(devBdf) + if err != nil { + klog.Warningf("unable to evaluate symlink: %s", devBdf) + continue + } + pciDevices = append(pciDevices, targetDev) + } + + return +} + +func (dp *DevicePlugin) getVfDevices() []string { + qatPfDevices := make([]string, 0) + qatVfDevices := make([]string, 0) + + // Get PF BDFs bound to a PF driver + for _, vfDriver := range dp.kernelVfDrivers { + pfDriver := strings.TrimSuffix(vfDriver, "vf") + pattern := filepath.Join(dp.pciDriverDir, pfDriver, "????:??:??.?") + qatPfDevices = append(qatPfDevices, getPciDevicesWithPattern(pattern)...) + } + + // Get VF devices belonging to a PF device + for _, qatPfDevice := range qatPfDevices { + pattern := filepath.Join(qatPfDevice, "virtfn*") + qatVfDevices = append(qatVfDevices, getPciDevicesWithPattern(pattern)...) + } + + if len(qatVfDevices) > 0 { + return qatVfDevices + } + + // No PF devices found, running in a VM? + for _, vfDriver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { + pattern := filepath.Join(dp.pciDriverDir, vfDriver, "????:??:??.?") + qatVfDevices = append(qatVfDevices, getPciDevicesWithPattern(pattern)...) + } + + return qatVfDevices +} + +func getCurrentDriver(device string) string { + symlink := filepath.Join(device, "driver") + driver, err := filepath.EvalSymlinks(symlink) + if err != nil { + klog.Warningf("unable to evaluate symlink: %s", symlink) + return "" + } + return filepath.Base(driver) +} + func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) { devTree := dpapi.NewDeviceTree() n := 0 - for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { - files, err := ioutil.ReadDir(filepath.Join(dp.pciDriverDir, driver)) + + for _, vfDevice := range dp.getVfDevices() { + vfBdf := filepath.Base(vfDevice) + + vfdevID, err := dp.getDeviceID(vfBdf) if err != nil { - klog.Warningf("Can't read sysfs for driver as Driver %s is not available: Skipping", driver) + return nil, errors.Wrapf(err, "Cannot obtain device ID for %s", vfBdf) + } + if !isValidVfDeviceID(vfdevID) { continue } + n = n + 1 - for _, file := range files { - if !strings.HasPrefix(file.Name(), "0000:") { - continue - } - vfdevID, err := dp.getDeviceID(file.Name()) - if err != nil { - return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name()) - } - if !isValidVfDeviceID(vfdevID) { - continue - } - n = n + 1 // increment after all junk got filtered out - - if n > dp.maxDevices { - break - } - - // initialize newly found devices which aren't bound to DPDK driver yet - if driver != dp.dpdkDriver { - err = dp.bindDevice(file.Name()) - if err != nil { - return nil, err - } - } + if n > dp.maxDevices { + break + } - dpdkDeviceName, err := dp.getDpdkDevice(file.Name()) + if getCurrentDriver(vfDevice) != dp.dpdkDriver { + err = dp.bindDevice(vfBdf) if err != nil { return nil, err } + } - klog.V(1).Infof("%s device: corresponding DPDK device detected is %s", file.Name(), dpdkDeviceName) - - envs := map[string]string{ - fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), - } + dpdkDeviceName, err := dp.getDpdkDevice(vfBdf) + if err != nil { + return nil, err + } - devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs) + klog.V(1).Infof("Device %s found", vfBdf) - devTree.AddDevice("generic", file.Name(), devinfo) + envs := map[string]string{ + fmt.Sprintf("%s%d", envVarPrefix, n): vfBdf, } + + devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs) + + devTree.AddDevice("generic", vfBdf, devinfo) } return devTree, nil diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go index 10b3340a1..002b94616 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go @@ -126,79 +126,38 @@ func TestScanPrivate(t *testing.T) { { name: "No error returned for uninitialized device plugin", }, - { - name: "Only DPDK driver is set and no devs allowed and vfdevID cannot be determined", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"}, - expectedErr: true, - }, - { - name: "Only DPDK driver is set and no dev exists", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/1111:test"}, - }, - { - name: "igb_uio DPDK driver with no valid DPDK device under uio directory", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:02:00.0"}, - maxDevNum: 1, - expectedErr: true, - }, - { - name: "igb_uio DPDKdriver with no DPDK bound devices", - dpdkDriver: "igb_uio", - dirs: []string{ - "sys/bus/pci/drivers/igb_uio/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio", - }, - maxDevNum: 1, - expectedErr: true, - }, { name: "igb_uio DPDKdriver with one DPDK bound device where vfdevID cannot be determined", dpdkDriver: "igb_uio", dirs: []string{ - "sys/bus/pci/drivers/igb_uio/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - }, - maxDevNum: 1, - expectedErr: true, - }, - { - name: "igb_uio DPDKdriver with one DPDK bound device", - dpdkDriver: "igb_uio", - dirs: []string{ - "sys/bus/pci/drivers/igb_uio/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - }, - files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/drivers/igb_uio/0000:02:01.0", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:00.0", }, - maxDevNum: 1, - expectedDevNum: 0, - }, - { - name: "igb_uio DPDKdriver with one kernel bound device, but unbindable", - dpdkDriver: "igb_uio", - kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedErr: true, }, { - name: "igb_uio DPDKdriver with one kernel bound device which gets lost after unbinding", + name: "igb_uio DPDKdriver with one kernel bound device, but unbindable", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:01.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), + }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + "sys/bus/pci/devices/0000:02:01.0/driver": "sys/bus/pci/drivers/c6xxvf", }, maxDevNum: 1, expectedErr: true, @@ -208,67 +167,41 @@ func TestScanPrivate(t *testing.T) { dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), - }, - maxDevNum: 1, - expectedDevNum: 0, - }, - { - name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", - dpdkDriver: "igb_uio", - kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("some junk"), }, - files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), - "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedDevNum: 0, }, - { - name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) ", - dpdkDriver: "igb_uio", - kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", - }, - files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), - "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), - }, - maxDevNum: 1, - expectedDevNum: 1, - }, { name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where reading uioDirPath for obtaining device file fails ", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + }, maxDevNum: 1, expectedErr: true, }, @@ -277,16 +210,21 @@ func TestScanPrivate(t *testing.T) { dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/uio", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + }, maxDevNum: 1, expectedErr: true, }, @@ -295,62 +233,71 @@ func TestScanPrivate(t *testing.T) { dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.1", - "sys/bus/pci/devices/0000:02:00.1/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.1/driver", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:01.1/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.1/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.1/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.1/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.1/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.1/device": []byte("0x37c9"), + }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn1": "sys/bus/pci/devices/0000:02:01.1", }, maxDevNum: 1, expectedDevNum: 1, }, { - name: "vfio-pci DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", + name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/vfio-pci", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/drivers/igb_uio", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.0/driver", "sys/bus/pci/devices/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), - "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("some junk"), + "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedDevNum: 0, }, { - name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)", + name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)", dpdkDriver: "vfio-pci", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/vfio-pci", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/driver", "sys/bus/pci/devices/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), }, symlinks: map[string]string{ - "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedDevNum: 1, @@ -361,19 +308,44 @@ func TestScanPrivate(t *testing.T) { kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ "sys/bus/pci/drivers/vfio-pci", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", - "sys/bus/pci/devices/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:01.0", + "sys/bus/pci/devices/0000:02:01.0/vfio-pci/vfiotestfile", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + }, maxDevNum: 1, expectedErr: true, }, + { + name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9), running in a VM", + dpdkDriver: "vfio-pci", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/c6xxvf", + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/devices/0000:02:01.0/driver", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/drivers/c6xxvf/0000:02:01.0": "sys/bus/pci/devices/0000:02:01.0", + }, + maxDevNum: 1, + expectedDevNum: 1, + }, } for _, tt := range tcases { if err := os.MkdirAll(tmpdir, 0750); err != nil {