diff --git a/.buildkite/pipeline_pr.py b/.buildkite/pipeline_pr.py index 36a904b2e5a4..a90738c99d92 100755 --- a/.buildkite/pipeline_pr.py +++ b/.buildkite/pipeline_pr.py @@ -4,15 +4,8 @@ """Generate Buildkite pipelines dynamically""" -from common import ( - COMMON_PARSER, - devtool_test, - get_changed_files, - group, - overlay_dict, - pipeline_to_json, - run_all_tests, -) +from common import (COMMON_PARSER, devtool_test, get_changed_files, group, + overlay_dict, pipeline_to_json, run_all_tests) # Buildkite default job priority is 0. Setting this to 1 prioritizes PRs over # scheduled jobs and other batch jobs. @@ -32,7 +25,7 @@ "platforms": args.platforms, # buildkite step parameters "priority": DEFAULT_PRIORITY, - "timeout_in_minutes": 45, + "timeout_in_minutes": 1000, "artifacts": ["./test_results/**/*"], } defaults = overlay_dict(defaults, args.step_param) @@ -83,7 +76,7 @@ "⏱ Performance", devtool_test( devtool_opts="--performance -c 1-10 -m 0", - pytest_opts="../tests/integration_tests/performance/", + pytest_opts="../tests/integration_tests/performance/test_vcpu_hotplug.py", binary_dir=args.binary_dir, ), **defaults_for_performance, diff --git a/tests/host_tools/hotplug.sh b/tests/host_tools/hotplug.sh index 276e5b8a93b9..c7e4c824ba48 100644 --- a/tests/host_tools/hotplug.sh +++ b/tests/host_tools/hotplug.sh @@ -6,6 +6,12 @@ while :; do [[ -d /sys/devices/system/cpu/cpu$1 ]] && break done -echo 1 | tee /sys/devices/system/cpu/cpu*/online +for i in $(seq 1 $1); do + echo 1 >/sys/devices/system/cpu/cpu$i/online +done + +while :; do + [[ $(nproc) == $((1 + $1)) ]] && break +done /home/hotplug_time.o diff --git a/tests/host_tools/hotplug_udev.sh b/tests/host_tools/hotplug_udev.sh index 27c5701b3d32..34a19d19943a 100644 --- a/tests/host_tools/hotplug_udev.sh +++ b/tests/host_tools/hotplug_udev.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 while :; do - [[ $(nproc) == $1 ]] && break + [[ $(nproc) == $((1 + $1)) ]] && break done /home/hotplug_time.o diff --git a/tests/integration_tests/performance/test_vcpu_hotplug.py b/tests/integration_tests/performance/test_vcpu_hotplug.py index fdeb9321e434..144aace3e4c2 100644 --- a/tests/integration_tests/performance/test_vcpu_hotplug.py +++ b/tests/integration_tests/performance/test_vcpu_hotplug.py @@ -11,9 +11,12 @@ import pandas import pytest -from framework.utils_iperf import IPerf3Test, emit_iperf3_metrics +from framework.utils_cpuid import check_guest_cpuid_output + +# from framework.utils_iperf import IPerf3Test, emit_iperf3_metrics from host_tools.cargo_build import gcc_compile -from host_tools.fcmetrics import FCMetricsMonitor + +# from host_tools.fcmetrics import FCMetricsMonitor @pytest.mark.skipif( @@ -28,7 +31,7 @@ def test_custom_udev_rule_latency( """Test the latency for hotplugging and booting CPUs in the guest""" gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o")) data = [] - for i in range(50): + for _ in range(20): uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw) uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None}) uvm_hotplug.help.enable_console() @@ -51,7 +54,7 @@ def test_custom_udev_rule_latency( ) uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count}) - time.sleep(2) + time.sleep(5) # Extract API call duration api_duration = ( @@ -73,10 +76,24 @@ def test_custom_udev_rule_latency( / 1000 ) except IndexError: - timestamp = None + uvm_hotplug.kill() + data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None}) + continue data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp}) + check_guest_cpuid_output( + uvm_hotplug, + "lscpu", + None, + ":", + { + "CPU(s)": str(1 + vcpu_count), + "On-line CPU(s) list": f"0-{vcpu_count}", + }, + ) + uvm_hotplug.kill() + output_file = results_dir / f"hotplug-{vcpu_count}.csv" csv_data = pandas.DataFrame.from_dict(data).to_csv( @@ -99,7 +116,7 @@ def test_manual_latency( """Test the latency for hotplugging and booting CPUs in the guest""" gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o")) data = [] - for _ in range(50): + for _ in range(20): uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw) uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None}) uvm_hotplug.help.enable_console() @@ -113,11 +130,13 @@ def test_manual_latency( uvm_hotplug.ssh.scp_put( Path("./host_tools/hotplug_time.o"), Path("/home/hotplug_time.o") ) - uvm_hotplug.ssh.run("tmux new-session -d /bin/bash /home/hotplug.sh") + uvm_hotplug.ssh.run( + f"tmux new-session -d /bin/bash /home/hotplug.sh {vcpu_count}" + ) uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count}) - time.sleep(1.5) + time.sleep(5) # Extract API call duration api_duration = ( float( @@ -139,10 +158,24 @@ def test_manual_latency( ) except IndexError: data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None}) + uvm_hotplug.kill() continue data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp}) + check_guest_cpuid_output( + uvm_hotplug, + "lscpu", + None, + ":", + { + "CPU(s)": str(1 + vcpu_count), + "On-line CPU(s) list": f"0-{vcpu_count}", + }, + ) + + uvm_hotplug.kill() + output_file = results_dir / f"hotplug-{vcpu_count}.csv" csv_data = pandas.DataFrame.from_dict(data).to_csv(