diff --git a/cmd/qat_plugin/README.md b/cmd/qat_plugin/README.md index d963d910e..52933d7da 100644 --- a/cmd/qat_plugin/README.md +++ b/cmd/qat_plugin/README.md @@ -51,7 +51,7 @@ The QAT plugin can take a number of command line arguments, summarised in the fo | Flag | Argument | Meaning | |:---- |:-------- |:------- | | -dpdk-driver | string | DPDK Device driver for configuring the QAT device (default: `vfio-pci`) | -| -kernel-vf-drivers | string | Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC,C62x,C3xxx and D15xx (default: `dh895xccvf,c6xxvf,c3xxxvf,d15xxvf`) | +| -kernel-vf-drivers | string | Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx, C4xxx and D15xx (default: `dh895xccvf,c6xxvf,c3xxxvf`) | | -max-num-devices | int | maximum number of QAT devices to be provided to the QuickAssist device plugin (default: `32`) | | -mode | string | plugin mode which can be either `dpdk` or `kernel` (default: `dpdk`) | @@ -66,7 +66,7 @@ table summarises the defaults: | Argument | Variable | Default setting | Explanation | |:-------- |:-------- |:--------------- |:----------- | | -dpdk-driver | `$DPDK_DRIVER` | vfio-pci | A more robust and secure choice than the `igb_uio` alternative | -| -kernel-vf-drivers | `$KERNEL_VF_DRIVERS` | dh895xccvf,c6xxvf,c3xxxvf,d15xxvf | Modify to suit your hardware setup | +| -kernel-vf-drivers | `$KERNEL_VF_DRIVERS` | dh895xccvf,c6xxvf,c3xxxvf | Modify to suit your hardware setup | | -max-num-devices | `$MAX_NUM_DEVICES` | 32 | Modify to suit your hardware setup if necessary | For more details on the `-dpdk-driver` choice, see diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go index adfbecd02..c5e167d13 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go @@ -67,7 +67,7 @@ func NewDevicePlugin(maxDevices int, kernelVfDrivers string, dpdkDriver string) kernelDrivers := strings.Split(kernelVfDrivers, ",") for _, driver := range kernelDrivers { - if !isValidKerneDriver(driver) { + if !isValidKernelDriver(driver) { return nil, errors.Errorf("wrong kernel VF driver: %s", driver) } } @@ -119,11 +119,11 @@ func (dp *DevicePlugin) getDpdkDevice(vfBdf string) (string, error) { return "", errors.WithStack(err) } s := filepath.Base(group) - klog.V(1).Infof("The vfio device group detected is %v", s) return s, nil - } - return "", errors.New("Unknown DPDK driver") + default: + return "", errors.New("Unknown DPDK driver") + } } func (dp *DevicePlugin) getDpdkDeviceSpecs(dpdkDeviceName string) []pluginapi.DeviceSpec { @@ -208,9 +208,9 @@ func (dp *DevicePlugin) bindDevice(vfBdf string) error { return nil } -func isValidKerneDriver(kernelvfDriver string) bool { +func isValidKernelDriver(kernelvfDriver string) bool { switch kernelvfDriver { - case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf": + case "dh895xccvf", "c6xxvf", "c3xxxvf", "d15xxvf", "c4xxvf", "4xxxvf": return true } return false @@ -225,7 +225,7 @@ func isValidDpdkDeviceDriver(dpdkDriver string) bool { } func isValidVfDeviceID(vfDevID string) bool { switch vfDevID { - case "0442", "0443", "37c9", "19e3": + case "0442", "0443", "37c9", "19e3", "4941", "18a1": return true } return false @@ -245,56 +245,108 @@ func (dp *DevicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error return nil } +func getPciDevicesWithPattern(pattern string) (pciDevices []string) { + pciDevices = make([]string, 0) + + devs, err := filepath.Glob(pattern) + if err != nil { + klog.Warningf("bad pattern: %s", pattern) + return + } + + for _, devBdf := range devs { + targetDev, err := filepath.EvalSymlinks(devBdf) + if err != nil { + klog.Warningf("unable to evaluate symlink: %s", devBdf) + continue + } + pciDevices = append(pciDevices, targetDev) + } + + return +} + +func (dp *DevicePlugin) getVfDevices() []string { + qatPfDevices := make([]string, 0) + qatVfDevices := make([]string, 0) + + // Get PF BDFs bound to a PF driver + for _, vfDriver := range dp.kernelVfDrivers { + pfDriver := strings.TrimSuffix(vfDriver, "vf") + pattern := filepath.Join(dp.pciDriverDir, pfDriver, "????:??:??.?") + qatPfDevices = append(qatPfDevices, getPciDevicesWithPattern(pattern)...) + } + + // Get VF devices belonging to a PF device + for _, qatPfDevice := range qatPfDevices { + pattern := filepath.Join(qatPfDevice, "virtfn*") + qatVfDevices = append(qatVfDevices, getPciDevicesWithPattern(pattern)...) + } + + if len(qatVfDevices) > 0 { + return qatVfDevices + } + + // No PF devices found, running in a VM? + for _, vfDriver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { + pattern := filepath.Join(dp.pciDriverDir, vfDriver, "????:??:??.?") + qatVfDevices = append(qatVfDevices, getPciDevicesWithPattern(pattern)...) + } + + return qatVfDevices +} + +func getCurrentDriver(device string) string { + symlink := filepath.Join(device, "driver") + driver, err := filepath.EvalSymlinks(symlink) + if err != nil { + klog.Warningf("unable to evaluate symlink: %s", symlink) + return "" + } + return filepath.Base(driver) +} + func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) { devTree := dpapi.NewDeviceTree() n := 0 - for _, driver := range append([]string{dp.dpdkDriver}, dp.kernelVfDrivers...) { - files, err := ioutil.ReadDir(filepath.Join(dp.pciDriverDir, driver)) + + for _, vfDevice := range dp.getVfDevices() { + vfBdf := filepath.Base(vfDevice) + + vfdevID, err := dp.getDeviceID(vfBdf) if err != nil { - klog.Warningf("Can't read sysfs for driver as Driver %s is not available: Skipping", driver) + return nil, errors.Wrapf(err, "Cannot obtain device ID for %s", vfBdf) + } + if !isValidVfDeviceID(vfdevID) { continue } + n = n + 1 - for _, file := range files { - if !strings.HasPrefix(file.Name(), "0000:") { - continue - } - vfdevID, err := dp.getDeviceID(file.Name()) - if err != nil { - return nil, errors.Wrapf(err, "Cannot obtain deviceID for the device with PCI address: %s", file.Name()) - } - if !isValidVfDeviceID(vfdevID) { - continue - } - n = n + 1 // increment after all junk got filtered out - - if n > dp.maxDevices { - break - } - - // initialize newly found devices which aren't bound to DPDK driver yet - if driver != dp.dpdkDriver { - err = dp.bindDevice(file.Name()) - if err != nil { - return nil, err - } - } + if n > dp.maxDevices { + break + } - dpdkDeviceName, err := dp.getDpdkDevice(file.Name()) + if getCurrentDriver(vfDevice) != dp.dpdkDriver { + err = dp.bindDevice(vfBdf) if err != nil { return nil, err } + } - klog.V(1).Infof("%s device: corresponding DPDK device detected is %s", file.Name(), dpdkDeviceName) - - envs := map[string]string{ - fmt.Sprintf("%s%d", envVarPrefix, n): file.Name(), - } + dpdkDeviceName, err := dp.getDpdkDevice(vfBdf) + if err != nil { + return nil, err + } - devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs) + klog.V(1).Infof("Device %s found", vfBdf) - devTree.AddDevice("generic", file.Name(), devinfo) + envs := map[string]string{ + fmt.Sprintf("%s%d", envVarPrefix, n): vfBdf, } + + devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs) + + devTree.AddDevice("generic", vfBdf, devinfo) } return devTree, nil diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go index 10b3340a1..002b94616 100644 --- a/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go +++ b/cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go @@ -126,79 +126,38 @@ func TestScanPrivate(t *testing.T) { { name: "No error returned for uninitialized device plugin", }, - { - name: "Only DPDK driver is set and no devs allowed and vfdevID cannot be determined", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:test"}, - expectedErr: true, - }, - { - name: "Only DPDK driver is set and no dev exists", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/1111:test"}, - }, - { - name: "igb_uio DPDK driver with no valid DPDK device under uio directory", - dpdkDriver: "igb_uio", - dirs: []string{"sys/bus/pci/drivers/igb_uio/0000:02:00.0"}, - maxDevNum: 1, - expectedErr: true, - }, - { - name: "igb_uio DPDKdriver with no DPDK bound devices", - dpdkDriver: "igb_uio", - dirs: []string{ - "sys/bus/pci/drivers/igb_uio/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio", - }, - maxDevNum: 1, - expectedErr: true, - }, { name: "igb_uio DPDKdriver with one DPDK bound device where vfdevID cannot be determined", dpdkDriver: "igb_uio", dirs: []string{ - "sys/bus/pci/drivers/igb_uio/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - }, - maxDevNum: 1, - expectedErr: true, - }, - { - name: "igb_uio DPDKdriver with one DPDK bound device", - dpdkDriver: "igb_uio", - dirs: []string{ - "sys/bus/pci/drivers/igb_uio/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - }, - files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/drivers/igb_uio/0000:02:01.0", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:00.0", }, - maxDevNum: 1, - expectedDevNum: 0, - }, - { - name: "igb_uio DPDKdriver with one kernel bound device, but unbindable", - dpdkDriver: "igb_uio", - kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedErr: true, }, { - name: "igb_uio DPDKdriver with one kernel bound device which gets lost after unbinding", + name: "igb_uio DPDKdriver with one kernel bound device, but unbindable", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:01.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), + }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + "sys/bus/pci/devices/0000:02:01.0/driver": "sys/bus/pci/drivers/c6xxvf", }, maxDevNum: 1, expectedErr: true, @@ -208,67 +167,41 @@ func TestScanPrivate(t *testing.T) { dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), - }, - maxDevNum: 1, - expectedDevNum: 0, - }, - { - name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", - dpdkDriver: "igb_uio", - kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("some junk"), }, - files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), - "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedDevNum: 0, }, - { - name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) ", - dpdkDriver: "igb_uio", - kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", - }, - files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), - "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), - }, - maxDevNum: 1, - expectedDevNum: 1, - }, { name: "igb_uio DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9) where reading uioDirPath for obtaining device file fails ", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + }, maxDevNum: 1, expectedErr: true, }, @@ -277,16 +210,21 @@ func TestScanPrivate(t *testing.T) { dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/uio", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + }, maxDevNum: 1, expectedErr: true, }, @@ -295,62 +233,71 @@ func TestScanPrivate(t *testing.T) { dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/igb_uio", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.0/driver", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.1", - "sys/bus/pci/devices/0000:02:00.1/uio/sometestfile", - "sys/bus/pci/devices/0000:02:00.1/driver", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:01.1/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.1/driver", + "sys/bus/pci/devices/0000:02:00.0", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.1/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.1/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.1/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.1/device": []byte("0x37c9"), + }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn1": "sys/bus/pci/devices/0000:02:01.1", }, maxDevNum: 1, expectedDevNum: 1, }, { - name: "vfio-pci DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", + name: "igb_uio DPDKdriver with one kernel bound device (not QAT device) where vfdevID is not equal to qatDevId (37c9)", dpdkDriver: "igb_uio", kernelVfDrivers: []string{"c6xxvf"}, - dirs: []string{ - "sys/bus/pci/drivers/vfio-pci", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/drivers/igb_uio", + "sys/bus/pci/devices/0000:02:01.0/uio/sometestfile", + "sys/bus/pci/devices/0000:02:01.0/driver", "sys/bus/pci/devices/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("some junk"), - "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("some junk"), + "sys/bus/pci/drivers/igb_uio/new_id": []byte("some junk"), + }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedDevNum: 0, }, { - name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)", + name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9)", dpdkDriver: "vfio-pci", kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ + "sys/bus/pci/drivers/c6xx", "sys/bus/pci/drivers/vfio-pci", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", + "sys/bus/pci/devices/0000:02:01.0/driver", "sys/bus/pci/devices/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), }, symlinks: map[string]string{ - "sys/bus/pci/devices/0000:02:00.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", }, maxDevNum: 1, expectedDevNum: 1, @@ -361,19 +308,44 @@ func TestScanPrivate(t *testing.T) { kernelVfDrivers: []string{"c6xxvf"}, dirs: []string{ "sys/bus/pci/drivers/vfio-pci", - "sys/bus/pci/drivers/c6xxvf/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/driver", - "sys/bus/pci/devices/0000:02:00.0", - "sys/bus/pci/devices/0000:02:00.0/vfio-pci/vfiotestfile", + "sys/bus/pci/drivers/c6xx", + "sys/bus/pci/devices/0000:02:01.0/driver", + "sys/bus/pci/devices/0000:02:01.0", + "sys/bus/pci/devices/0000:02:01.0/vfio-pci/vfiotestfile", }, files: map[string][]byte{ - "sys/bus/pci/devices/0000:02:00.0/driver/unbind": []byte("some junk"), - "sys/bus/pci/devices/0000:02:00.0/device": []byte("0x37c9"), + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), }, + symlinks: map[string]string{ + "sys/bus/pci/drivers/c6xx/0000:02:00.0": "sys/bus/pci/devices/0000:02:00.0", + "sys/bus/pci/devices/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0", + }, maxDevNum: 1, expectedErr: true, }, + { + name: "vfio-pci DPDKdriver with one kernel bound device (QAT device) where vfdevID is equal to qatDevId (37c9), running in a VM", + dpdkDriver: "vfio-pci", + kernelVfDrivers: []string{"c6xxvf"}, + dirs: []string{ + "sys/bus/pci/drivers/c6xxvf", + "sys/bus/pci/drivers/vfio-pci", + "sys/bus/pci/devices/0000:02:01.0/driver", + }, + files: map[string][]byte{ + "sys/bus/pci/devices/0000:02:01.0/driver/unbind": []byte("some junk"), + "sys/bus/pci/devices/0000:02:01.0/device": []byte("0x37c9"), + "sys/bus/pci/drivers/vfio-pci/new_id": []byte("some junk"), + }, + symlinks: map[string]string{ + "sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile", + "sys/bus/pci/drivers/c6xxvf/0000:02:01.0": "sys/bus/pci/devices/0000:02:01.0", + }, + maxDevNum: 1, + expectedDevNum: 1, + }, } for _, tt := range tcases { if err := os.MkdirAll(tmpdir, 0750); err != nil { diff --git a/cmd/qat_plugin/qat_plugin.go b/cmd/qat_plugin/qat_plugin.go index 9e00f8ddd..19d4b886b 100644 --- a/cmd/qat_plugin/qat_plugin.go +++ b/cmd/qat_plugin/qat_plugin.go @@ -38,7 +38,7 @@ func main() { mode := flag.String("mode", "dpdk", "plugin mode which can be either dpdk (default) or kernel") dpdkDriver := flag.String("dpdk-driver", "vfio-pci", "DPDK Device driver for configuring the QAT device") - kernelVfDrivers := flag.String("kernel-vf-drivers", "dh895xccvf,c6xxvf,c3xxxvf,d15xxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC,C62x,C3xxx and D15xx") + kernelVfDrivers := flag.String("kernel-vf-drivers", "dh895xccvf,c6xxvf,c3xxxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, and D15xx") maxNumDevices := flag.Int("max-num-devices", 32, "maximum number of QAT devices to be provided to the QuickAssist device plugin") flag.Parse() diff --git a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml b/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml index 757f4ff51..2eeb4c99d 100644 --- a/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml +++ b/deployments/qat_plugin/overlays/sriov_numvfs/sriov_numvfs_init.yaml @@ -21,7 +21,7 @@ spec: command: ["/bin/sh", "-c"] args: - > - for dev in `for pf in 0434 0435 19a3 37c8 6f54; do lspci | awk -v dev="8086:$pf" '$0 ~ dev {print "0000:" $1}'; done`; do + for dev in `for pf in 0434 0435 19a3 37c8 6f54 4940 18a0; do lspci | awk -v dev="8086:$pf" '$0 ~ dev {print "0000:" $1}'; done`; do DEVPATH="/sys/bus/pci/devices/$dev" NUMVFS="$DEVPATH/sriov_numvfs" if [ -w "$NUMVFS" -a $(cat "$NUMVFS") -eq 0 ]; then