From fcdfaeb357a0c670c2e3878a1795dbc20b70448b Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 13 Sep 2024 14:00:46 +0100 Subject: [PATCH 1/4] feat: enable ftrace for guest kernels This will allows us to debug issues with a guest kernel more easily. Signed-off-by: Egor Lazarchuk --- .../patches/0003-enable-ftrace.patch | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 resources/guest_configs/patches/0003-enable-ftrace.patch diff --git a/resources/guest_configs/patches/0003-enable-ftrace.patch b/resources/guest_configs/patches/0003-enable-ftrace.patch new file mode 100644 index 00000000000..ddc44888dce --- /dev/null +++ b/resources/guest_configs/patches/0003-enable-ftrace.patch @@ -0,0 +1,161 @@ +diff --git a/resources/guest_configs/microvm-kernel-ci-aarch64-4.14.config b/resources/guest_configs/microvm-kernel-ci-aarch64-4.14.config +index 694058ca9..bf06466ab 100644 +--- a/resources/guest_configs/microvm-kernel-ci-aarch64-4.14.config ++++ b/resources/guest_configs/microvm-kernel-ci-aarch64-4.14.config +@@ -2482,7 +2482,17 @@ CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y + CONFIG_HAVE_SYSCALL_TRACEPOINTS=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_DMA_API_DEBUG is not set + # CONFIG_KUNIT is not set + +diff --git a/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config b/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config +index ac44904c1..2a6f2310f 100644 +--- a/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config ++++ b/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config +@@ -3089,7 +3089,17 @@ CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y + CONFIG_HAVE_SYSCALL_TRACEPOINTS=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_SAMPLES is not set + CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y + # CONFIG_STRICT_DEVMEM is not set +diff --git a/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config b/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config +index 26b87a658..f0f765298 100644 +--- a/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config ++++ b/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config +@@ -3309,7 +3309,17 @@ CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y + CONFIG_HAVE_SYSCALL_TRACEPOINTS=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_SAMPLES is not set + # CONFIG_STRICT_DEVMEM is not set + +diff --git a/resources/guest_configs/microvm-kernel-ci-x86_64-4.14.config b/resources/guest_configs/microvm-kernel-ci-x86_64-4.14.config +index ee6df5ffc..3ab74b395 100644 +--- a/resources/guest_configs/microvm-kernel-ci-x86_64-4.14.config ++++ b/resources/guest_configs/microvm-kernel-ci-x86_64-4.14.config +@@ -2694,7 +2694,17 @@ CONFIG_HAVE_SYSCALL_TRACEPOINTS=y + CONFIG_HAVE_FENTRY=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_PROVIDE_OHCI1394_DMA_INIT is not set + # CONFIG_DMA_API_DEBUG is not set + # CONFIG_KUNIT is not set +diff --git a/resources/guest_configs/microvm-kernel-ci-x86_64-5.10-no-acpi.config b/resources/guest_configs/microvm-kernel-ci-x86_64-5.10-no-acpi.config +index b87fb3e44..fc45dda19 100644 +--- a/resources/guest_configs/microvm-kernel-ci-x86_64-5.10-no-acpi.config ++++ b/resources/guest_configs/microvm-kernel-ci-x86_64-5.10-no-acpi.config +@@ -2905,7 +2905,17 @@ CONFIG_HAVE_SYSCALL_TRACEPOINTS=y + CONFIG_HAVE_FENTRY=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_SAMPLES is not set + CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y + CONFIG_STRICT_DEVMEM=y +diff --git a/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config b/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config +index 09461c178..6d85bce2c 100644 +--- a/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config ++++ b/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config +@@ -2987,7 +2987,17 @@ CONFIG_HAVE_SYSCALL_TRACEPOINTS=y + CONFIG_HAVE_FENTRY=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_SAMPLES is not set + CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y + CONFIG_STRICT_DEVMEM=y +diff --git a/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config b/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config +index 967e32031..d11ef968a 100644 +--- a/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config ++++ b/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config +@@ -3185,7 +3185,17 @@ CONFIG_HAVE_OBJTOOL_MCOUNT=y + CONFIG_HAVE_C_RECORDMCOUNT=y + CONFIG_HAVE_BUILDTIME_MCOUNT_SORT=y + CONFIG_TRACING_SUPPORT=y +-# CONFIG_FTRACE is not set ++CONFIG_FTRACE=y ++CONFIG_FUNCTION_TRACER=y ++CONFIG_FUNCTION_GRAPH_TRACER=y ++CONFIG_IRQSOFF_TRACER=y ++CONFIG_PREEMPT_TRACER=y ++CONFIG_SCHED_TRACER=y ++CONFIG_STACK_TRACER=y ++CONFIG_BLK_DEV_IO_TRACE=y ++CONFIG_FUNCTION_PROFILER=y ++CONFIG_FTRACE_MCOUNT_RECORD=y ++CONFIG_FTRACE_SYSCALLS=y + # CONFIG_SAMPLES is not set + CONFIG_HAVE_SAMPLE_FTRACE_DIRECT=y + CONFIG_HAVE_SAMPLE_FTRACE_DIRECT_MULTI=y From 648d4f04663ad13b868a4082907061125fecb6a5 Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 13 Sep 2024 14:03:07 +0100 Subject: [PATCH 2/4] fix(aarch64): disable CONFIG_ARM64_ERRATUM_3194386 option Disable CONFIG_ARM64_ERRATUM_3194386 option in the guest kernels for aarch64. It is enabled by default and makes `ssbs` feature to not be user space visible. This breaks out integration tests. This option is only indented for `Cortex-X4` and `Neoverse-V3` cpus which we are not using, so disabling it is safe. Signed-off-by: Egor Lazarchuk --- ...IG_ARM64_ERRATUM_3194386-for-aarch64.patch | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 resources/guest_configs/patches/0004-disable-CONFIG_ARM64_ERRATUM_3194386-for-aarch64.patch diff --git a/resources/guest_configs/patches/0004-disable-CONFIG_ARM64_ERRATUM_3194386-for-aarch64.patch b/resources/guest_configs/patches/0004-disable-CONFIG_ARM64_ERRATUM_3194386-for-aarch64.patch new file mode 100644 index 00000000000..f3f16925e41 --- /dev/null +++ b/resources/guest_configs/patches/0004-disable-CONFIG_ARM64_ERRATUM_3194386-for-aarch64.patch @@ -0,0 +1,24 @@ +diff --git a/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config b/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config +index ac44904c1..9f912a70b 100644 +--- a/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config ++++ b/resources/guest_configs/microvm-kernel-ci-aarch64-5.10.config +@@ -341,6 +341,7 @@ CONFIG_ARM64_ERRATUM_1463225=y + CONFIG_ARM64_ERRATUM_1542419=y + CONFIG_ARM64_ERRATUM_1508412=y + CONFIG_ARM64_ERRATUM_2457168=y ++# CONFIG_ARM64_ERRATUM_3194386 is not set + CONFIG_CAVIUM_ERRATUM_22375=y + CONFIG_CAVIUM_ERRATUM_23144=y + CONFIG_CAVIUM_ERRATUM_23154=y +diff --git a/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config b/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config +index 26b87a658..29fe130f2 100644 +--- a/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config ++++ b/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config +@@ -362,6 +362,7 @@ CONFIG_ARM64_ERRATUM_2441009=y + CONFIG_ARM64_ERRATUM_2457168=y + CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD=y + CONFIG_ARM64_ERRATUM_2966298=y ++# CONFIG_ARM64_ERRATUM_3194386 is not set + CONFIG_CAVIUM_ERRATUM_22375=y + CONFIG_CAVIUM_ERRATUM_23144=y + CONFIG_CAVIUM_ERRATUM_23154=y From 728080b582f23bbe0a637a6b09fa97362f13ab1e Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Fri, 13 Sep 2024 15:31:50 +0100 Subject: [PATCH 3/4] fix: add net to vm for test_vhost_user_metrics We need to add net interface so the `vm.start()` will wait for a vm to boot. Otherwise the `time.sleep(0.3)` is not enough for a vm to boot and connect to the vhost-user-backend. Signed-off-by: Egor Lazarchuk --- tests/integration_tests/performance/test_vhost_user_metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration_tests/performance/test_vhost_user_metrics.py b/tests/integration_tests/performance/test_vhost_user_metrics.py index 59b4cf1ae08..45fb7a94114 100644 --- a/tests/integration_tests/performance/test_vhost_user_metrics.py +++ b/tests/integration_tests/performance/test_vhost_user_metrics.py @@ -33,6 +33,7 @@ def test_vhost_user_block_metrics( ) vm.spawn(log_level="Info") vm.basic_config(vcpu_count=vcpu_count) + vm.add_net_iface() # Add a block device to test resizing. fs = drive_tools.FilesystemFile(size=orig_size) From 2e878b7124388ca59d1d6f5fbc21b4e6535fe5dd Mon Sep 17 00:00:00 2001 From: Egor Lazarchuk Date: Mon, 16 Sep 2024 11:24:01 +0100 Subject: [PATCH 4/4] chore: slightly increase max boottime for ftrace Because now we have ftrace enabled in guest kernels, the boot time increases slightly and no longer satisfies 150000 us time. Signed-off-by: Egor Lazarchuk --- tests/integration_tests/performance/test_boottime.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests/performance/test_boottime.py b/tests/integration_tests/performance/test_boottime.py index e4f62b388e4..66418f391cf 100644 --- a/tests/integration_tests/performance/test_boottime.py +++ b/tests/integration_tests/performance/test_boottime.py @@ -11,7 +11,9 @@ from framework.properties import global_props # The maximum acceptable boot time in us. -MAX_BOOT_TIME_US = 150000 +# It is a bit bigger than the default 150_000 because +# we have ftrace enabled in the guest kernels. +MAX_BOOT_TIME_US = 170000 # Regex for obtaining boot time from some string. TIMESTAMP_LOG_REGEX = r"Guest-boot-time\s+\=\s+(\d+)\s+us"