While trying to run Xilinx QDMA virtual prototype demo according to https://github.com/Xilinx/systemctlm-cosim-demo/blob/master/pcie/versal/cpm-qdma-demos.md the host kernel panics during data transfer.

[artakarakelyan]

While trying to run Xilinx QDMA virtual prototype demo according to https://github.com/Xilinx/systemctlm-cosim-demo/blob/master/pcie/versal/cpm-qdma-demos.md the host kernel panics during data transfer.

Tried systemC 2.3.3 and 2.3.4, tried QDMA drivers 2022.1, 2022.2 and the actual master branch.

ubuntu@ubuntu:~/dma_ip_drivers/QDMA/linux-kernel$ sudo su
root@ubuntu:/home/ubuntu/dma_ip_drivers/QDMA/linux-kernel# insmod ./bin/qdma-pf.ko
root@ubuntu:/home/ubuntu/dma_ip_drivers/QDMA/linux-kernel# echo 1 > /sys/bus/pci/devices/0000:02:00.0/qdma/qmax
root@ubuntu:/home/ubuntu/dma_ip_drivers/QDMA/linux-kernel# ./bin/dma-ctl qdma02000 q add idx 0 mode mm dir h2c
qdma02000-MM-0 H2C added.
Added 1 Queues.
root@ubuntu:/home/ubuntu/dma_ip_drivers/QDMA/linux-kernel# ./bin/dma-ctl qdma02000 q start idx 0 dir h2c aperture
_sz 4096
dma-ctl: Info: Default ring size set to 2048
1 Queues started, idx 0 ~ 0.
root@ubuntu:/home/ubuntu/dma_ip_drivers/QDMA/linux-kernel# ./bin/dma-to-device -d /dev/qdma02000-MM-0 -f /home/ub
untu/test.pdi -s 1024 -a 0x102100000
[ 113.670121] qdma_cpm4_init_ctxt_memory: clearing the context for all qs
[ 134.043645] invalid opcode: 0000 [#1] SMP PTI
[ 134.046907] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G OE 5.4.0-64-generic #72-Ubuntu
[ 134.049231] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.16.0-0-gd239552ce722-prebuilt.qemu.
org 04/01/2014
[ 134.052543] RIP: 0010:rdtsc_gettime+0x9/0x20 [qdma_pf]
[ 134.054086] Code: c0 48 c7 c6 38 b9 5f c0 41 54 48 c7 c7 d0 37 63 c0 6a 08 50 e8 38 81 fa f7 48 83 c4 28 eb b6
66 90 0f 1f 44 00 00 55 48 89 e5 <0f> 01 f9 48 c1 e2 20 89 c0 5d 48 09 d0 c3 66 0f 1f 84 00 00 00 00
[ 134.059523] RSP: 0018:ffffb59b00003e90 EFLAGS: 00010086
[ 134.061434] RAX: ffffffffc05a6670 RBX: ffff8a01b45f9580 RCX: 0000000000000007
[ 134.063749] RDX: ffff8a01b7b82800 RSI: 0000000000000022 RDI: 0000000000000002
[ 134.065899] RBP: ffffb59b00003e90 R08: 0000000000000022 R09: 0000000004315776
[ 134.068259] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000022
[ 134.070587] R13: ffff8a01b7b82800 R14: 0000000000000002 R15: 0000000000000000
[ 134.073159] FS: 0000000000000000(0000) GS:ffff8a01bba00000(0000) knlGS:0000000000000000
[ 134.075709] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 134.077704] CR2: 00007ffd806d3000 CR3: 000000016431c000 CR4: 00000000000006f0
[ 134.081106] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 134.084290] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 134.087165] Call Trace:
[ 134.088269]
[ 134.089260] data_intr_handler+0x29/0x4d0 [qdma_pf]
[ 134.091228] irq_bottom+0x54/0x70 [qdma_pf]
[ 134.093020] __handle_irq_event_percpu+0x42/0x180
[ 134.095068] handle_irq_event_percpu+0x33/0x80
[ 134.096873] handle_irq_event+0x3b/0x5a
[ 134.098605] handle_edge_irq+0x93/0x1c0
[ 134.100300] do_IRQ+0x55/0xf0
[ 134.101536] common_interrupt+0xf/0xf
[ 134.103076]
[ 134.104404] RIP: 0010:native_safe_halt+0xe/0x10
[ 134.106399] Code: 7b ff ff ff eb bd 90 90 90 90 90 90 e9 07 00 00 00 0f 00 2d b6 ff 52 00 f4 c3 66 90 e9 07 00
00 00 0f 00 2d a6 ff 52 00 fb f4 90 0f 1f 44 00 00 55 48 89 e5 41 55 41 54 53 e8 dd d1 64 ff 65
[ 134.114440] RSP: 0018:ffffffffb9603e18 EFLAGS: 00000246 ORIG_RAX: ffffffffffffffd8
[ 134.117780] RAX: ffffffffb8adb380 RBX: 0000000000000000 RCX: 0000000000000001
[ 134.120914] RDX: 000000000000a21e RSI: 0000000000000087 RDI: 0000000000000000
[ 134.124160] RBP: ffffffffb9603e38 R08: 0000001f468157b9 R09: 0000000000000200
[ 134.126435] R10: 0000000000000000 R11: 0000000000000001 R12: 0000000000000000
[ 134.128732] R13: ffffffffb9613780 R14: 0000000000000000 R15: 0000000000000000
[ 134.130879] ? _cpuidle_text_start+0x8/0x8
[ 134.132229] ? tick_nohz_idle_stop_tick+0x13c/0x290
[ 134.133982] ? default_idle+0x20/0x140
[ 134.135345] arch_cpu_idle+0x15/0x20
[ 134.136651] default_idle_call+0x23/0x30
[ 134.138105] do_idle+0x1fb/0x270
[ 134.139306] cpu_startup_entry+0x20/0x30
[ 134.140656] rest_init+0xae/0xb0
[ 134.141871] arch_call_rest_init+0xe/0x1b
[ 134.143298] start_kernel+0x549/0x56a
[ 134.144877] x86_64_start_reservations+0x24/0x26
[ 134.146594] x86_64_start_kernel+0x75/0x79
[ 134.148860] secondary_startup_64+0xa4/0xb0
[ 134.150388] Modules linked in: qdma_pf(OE) isofs nls_iso8859_1 dm_multipath scsi_dh_rdac scsi_dh_emc scsi_dh_a
lua binfmt_misc ppdev joydev input_leds serio_raw parport_pc parport sch_fq_codel msr efi_pstore drm ip_tables x
tables autofs4 btrfs zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor
raid6_pq libcrc32c raid1 raid0 multipath linear virtio_net net_failover psmouse ahci libahci failover
[ 134.162276] ---[ end trace 06e1b0a595a9f195 ]---
[ 134.164163] RIP: 0010:rdtsc_gettime+0x9/0x20 [qdma_pf]
[ 134.166309] Code: c0 48 c7 c6 38 b9 5f c0 41 54 48 c7 c7 d0 37 63 c0 6a 08 50 e8 38 81 fa f7 48 83 c4 28 eb b6
66 90 0f 1f 44 00 00 55 48 89 e5 <0f> 01 f9 48 c1 e2 20 89 c0 5d 48 09 d0 c3 66 0f 1f 84 00 00 00 00
[ 134.174094] RSP: 0018:ffffb59b00003e90 EFLAGS: 00010086
[ 134.175627] RAX: ffffffffc05a6670 RBX: ffff8a01b45f9580 RCX: 0000000000000007
[ 134.177708] RDX: ffff8a01b7b82800 RSI: 0000000000000022 RDI: 0000000000000002
[ 134.179721] RBP: ffffb59b00003e90 R08: 0000000000000022 R09: 0000000004315776
[ 134.181986] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000022
[ 134.184719] R13: ffff8a01b7b82800 R14: 0000000000000002 R15: 0000000000000000
[ 134.187176] FS: 0000000000000000(0000) GS:ffff8a01bba00000(0000) knlGS:0000000000000000
[ 134.190059] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 134.192044] CR2: 00007ffd806d3000 CR3: 000000016431c000 CR4: 00000000000006f0
[ 134.194864] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 134.197044] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 134.199043] Kernel panic - not syncing: Fatal exception in interrupt
[ 134.200997] Kernel Offset: 0x37000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffff
bfffffff)
[ 134.204291] ---[ end Kernel panic - not syncing: Fatal exception in interrupt ]---
KVM: injection failed, MSI lost (Operation not permitted)
KVM: injection failed, MSI lost (Operation not permitted)

While running dma-to-device with strace the same crash occurs right after write system call to driver

root@ubuntu:/home/ubuntu/dma_ip_drivers/QDMA/linux-kernel# strace ./bin/dma-to-device -d /dev/qdma02000-MM-0 -f /home/ubuntu/test.pdi -s 1024 -a 0x102100000
execve("./bin/dma-to-device", ["./bin/dma-to-device", "-d", "/dev/qdma02000-MM-0", "-f", "/home/ubuntu/test.pdi", "-s", "1024", "-a", "0x102100000"], 0x7ffe5a344860 /* 19 vars /) = 0
brk(NULL) = 0x5558ce021000
arch_prctl(0x3001 / ARCH_??? */, 0x7ffee71cbe80) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=28385, ...}) = 0
mmap(NULL, 28385, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f5ec24f1000
close(3) = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300A\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\30x\346\264ur\f|Q\226\236i\253-'o"..., 68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029592, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f5ec24ef000
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\30x\346\264ur\f|Q\226\236i\253-'o"..., 68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f5ec22fd000
mmap(0x7f5ec231f000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f5ec231f000
mmap(0x7f5ec2497000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f5ec2497000
mmap(0x7f5ec24e5000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f5ec24e5000
mmap(0x7f5ec24eb000, 13920, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f5ec24eb000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7f5ec24f0580) = 0
mprotect(0x7f5ec24e5000, 16384, PROT_READ) = 0
mprotect(0x5558cc4cd000, 4096, PROT_READ) = 0
mprotect(0x7f5ec2525000, 4096, PROT_READ) = 0
munmap(0x7f5ec24f1000, 28385) = 0
brk(NULL) = 0x5558ce021000
brk(0x5558ce042000) = 0x5558ce042000
openat(AT_FDCWD, "/dev/qdma02000-MM-0", O_RDWR) = 3
openat(AT_FDCWD, "/home/ubuntu/test.pdi", O_RDONLY) = 4
read(4, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024) = 1024
lseek(3, 4329570304, SEEK_SET) = 4329570304
write(3, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 1024[ 313.919727] invalid opcode: 0000 [#1] SMP PTI
[ 313.921001] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G OE 5.4.0-64-generic #72-Ubuntu
[ 313.923404] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.16.0-0-gd239552ce722-prebuilt.qemu.org 04/01/2014
[ 313.926590] RIP: 0010:rdtsc_gettime+0x9/0x20 [qdma_pf]
[ 313.928253] Code: c0 48 c7 c6 38 a9 77 c0 41 54 48 c7 c7 d0 27 7b c0 6a 08 50 e8 38 91 02 c9 48 83 c4 28 eb b6 66 90 0f 1f 44 00 00 55 48 89 e5 <0f> 01 f9 48 c1 e2 20 89 c0 5d 48 09 d0 c3 66 0f 1f 84 00 00 00 00