diff --git a/CHANGELOG.md b/CHANGELOG.md index 82647dd4440..f079bf9e4cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,12 @@ and this project adheres to UFFD support not being forward-compatible with new ioctl options introduced in Linux 6.6. See also https://github.com/bytecodealliance/userfaultfd-rs/issues/61. +- [#4630](https://github.com/firecracker-microvm/firecracker/pull/4630): On + x86_64, when taking a snapshot, if a vCPU has MSR_IA32_TSC_DEADLINE set to 0, + Firecracker will replace it with the MSR_IA32_TSC value from the same vCPU. + This is to guarantee that the vCPU will continue receiving TSC interrupts + after restoring from the snapshot even if an interrupt is lost when taking a + snapshot. ## \[1.7.0\] diff --git a/docs/snapshotting/snapshot-support.md b/docs/snapshotting/snapshot-support.md index 76f12666d6b..5f804b86207 100644 --- a/docs/snapshotting/snapshot-support.md +++ b/docs/snapshotting/snapshot-support.md @@ -171,6 +171,11 @@ The snapshot functionality is still in developer preview due to the following: the data store is not persisted across snapshots. - Configuration information for metrics and logs are not saved to the snapshot. These need to be reconfigured on the restored microVM. +- On x86_64, if a vCPU has MSR_IA32_TSC_DEADLINE set to 0 when a snapshot is + taken, Firecracker replaces it with the MSR_IA32_TSC value from the same vCPU. + This is to guarantee that the vCPU will continue receiving TSC interrupts + after restoring from the snapshot even if an interrupt is lost when taking a + snapshot. ## Snapshot versioning diff --git a/src/cpu-template-helper/src/template/dump/x86_64.rs b/src/cpu-template-helper/src/template/dump/x86_64.rs index e859a709131..61be42d6aa3 100644 --- a/src/cpu-template-helper/src/template/dump/x86_64.rs +++ b/src/cpu-template-helper/src/template/dump/x86_64.rs @@ -68,9 +68,11 @@ fn msrs_to_modifier(msrs: &HashMap) -> Vec { // Fireracker diables some features (e.g., PMU) and doesn't support some features (e.g., Hyper-V), // MSRs related to such features are not useful as CPU configuration dump. Excluding such MSRs // reduces maintenance cost when KVM makes change their default values. -const MSR_EXCLUSION_LIST: [MsrRange; 9] = [ +const MSR_EXCLUSION_LIST: [MsrRange; 10] = [ // - MSR_IA32_TSC (0x10): vary depending on the elapsed time. MSR_RANGE!(MSR_IA32_TSC), + // - MSR_IA32_TSC_DEADLINE (0x6e0): varies depending on the elapsed time. + MSR_RANGE!(MSR_IA32_TSC_DEADLINE), // Firecracker doesn't support MCE. // - MSR_IA32_MCG_STATUS (0x17a) // - MSR_IA32_MCG_EXT_CTL (0x4d0) diff --git a/src/vmm/src/vstate/vcpu/x86_64.rs b/src/vmm/src/vstate/vcpu/x86_64.rs index b4542c1c598..67dc8681f25 100644 --- a/src/vmm/src/vstate/vcpu/x86_64.rs +++ b/src/vmm/src/vstate/vcpu/x86_64.rs @@ -19,6 +19,7 @@ use serde::{Deserialize, Serialize}; use crate::arch::x86_64::interrupts; use crate::arch::x86_64::msr::{create_boot_msr_entries, MsrError}; use crate::arch::x86_64::regs::{SetupFpuError, SetupRegistersError, SetupSpecialRegistersError}; +use crate::arch_gen::x86::msr_index::{MSR_IA32_TSC, MSR_IA32_TSC_DEADLINE}; use crate::cpu_config::x86_64::{cpuid, CpuConfiguration}; use crate::logger::{IncMetric, METRICS}; use crate::vstate::memory::{Address, GuestAddress, GuestMemoryMmap}; @@ -282,6 +283,39 @@ impl KvmVcpu { Ok(cpuid) } + /// If the IA32_TSC_DEADLINE MSR value is zero, update it + /// with the IA32_TSC value to guarantee that + /// the vCPU will continue receiving interrupts after restoring from a snapshot. + /// + /// Rationale: we observed that sometimes when taking a snapshot, + /// the IA32_TSC_DEADLINE MSR is cleared, but the interrupt is not + /// delivered to the guest, leading to a situation where one + /// of the vCPUs never receives TSC interrupts after restoring, + /// until the MSR is updated externally, eg by setting the system time. + fn fix_zero_tsc_deadline_msr(msr_chunks: &mut [Msrs]) { + // We do not expect more than 1 TSC MSR entry, but if there are multiple, pick the maximum. + let max_tsc_value = msr_chunks + .iter() + .flat_map(|msrs| msrs.as_slice()) + .filter(|msr| msr.index == MSR_IA32_TSC) + .map(|msr| msr.data) + .max(); + + if let Some(tsc_value) = max_tsc_value { + msr_chunks + .iter_mut() + .flat_map(|msrs| msrs.as_mut_slice()) + .filter(|msr| msr.index == MSR_IA32_TSC_DEADLINE && msr.data == 0) + .for_each(|msr| { + warn!( + "MSR_IA32_TSC_DEADLINE is 0, replacing with {:x}.", + tsc_value + ); + msr.data = tsc_value; + }); + } + } + /// Get MSR chunks for the given MSR index list. /// /// KVM only supports getting `KVM_MAX_MSR_ENTRIES` at a time, so we divide @@ -321,6 +355,8 @@ impl KvmVcpu { msr_chunks.push(msrs); } + Self::fix_zero_tsc_deadline_msr(&mut msr_chunks); + Ok(msr_chunks) } @@ -594,6 +630,7 @@ mod tests { use std::os::unix::io::AsRawFd; + use kvm_bindings::kvm_msr_entry; use kvm_ioctls::Cap; use super::*; @@ -949,4 +986,77 @@ mod tests { } } } + + fn msrs_from_entries(msr_entries: &[(u32, u64)]) -> Msrs { + Msrs::from_entries( + &msr_entries + .iter() + .map(|&(index, data)| kvm_msr_entry { + index, + data, + ..Default::default() + }) + .collect::>(), + ) + .unwrap() + } + + fn assert_msrs(msr_chunks: &[Msrs], expected_msr_entries: &[(u32, u64)]) { + let flattened_msrs = msr_chunks.iter().flat_map(|msrs| msrs.as_slice()); + for (a, b) in flattened_msrs.zip(expected_msr_entries.iter()) { + assert_eq!(a.index, b.0); + assert_eq!(a.data, b.1); + } + } + + #[test] + fn test_fix_zero_tsc_deadline_msr_zero_same_chunk() { + // Place both TSC and TSC_DEADLINE MSRs in the same chunk. + let mut msr_chunks = [msrs_from_entries(&[ + (MSR_IA32_TSC_DEADLINE, 0), + (MSR_IA32_TSC, 42), + ])]; + + KvmVcpu::fix_zero_tsc_deadline_msr(&mut msr_chunks); + + // We expect for the MSR_IA32_TSC_DEADLINE to get updated with the MSR_IA32_TSC value. + assert_msrs( + &msr_chunks, + &[(MSR_IA32_TSC_DEADLINE, 42), (MSR_IA32_TSC, 42)], + ); + } + + #[test] + fn test_fix_zero_tsc_deadline_msr_zero_separate_chunks() { + // Place both TSC and TSC_DEADLINE MSRs in separate chunks. + let mut msr_chunks = [ + msrs_from_entries(&[(MSR_IA32_TSC_DEADLINE, 0)]), + msrs_from_entries(&[(MSR_IA32_TSC, 42)]), + ]; + + KvmVcpu::fix_zero_tsc_deadline_msr(&mut msr_chunks); + + // We expect for the MSR_IA32_TSC_DEADLINE to get updated with the MSR_IA32_TSC value. + assert_msrs( + &msr_chunks, + &[(MSR_IA32_TSC_DEADLINE, 42), (MSR_IA32_TSC, 42)], + ); + } + + #[test] + fn test_fix_zero_tsc_deadline_msr_non_zero() { + let mut msr_chunks = [msrs_from_entries(&[ + (MSR_IA32_TSC_DEADLINE, 1), + (MSR_IA32_TSC, 2), + ])]; + + KvmVcpu::fix_zero_tsc_deadline_msr(&mut msr_chunks); + + // We expect that MSR_IA32_TSC_DEADLINE should remain unchanged, because it is non-zero + // already. + assert_msrs( + &msr_chunks, + &[(MSR_IA32_TSC_DEADLINE, 1), (MSR_IA32_TSC, 2)], + ); + } } diff --git a/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_4.14host.json b/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_4.14host.json index 5c0805211de..e1252ee67a7 100644 --- a/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_4.14host.json +++ b/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_4.14host.json @@ -1276,10 +1276,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0x4b564d00", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_5.10host.json b/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_5.10host.json index feafc6061ba..5501ca88ce4 100644 --- a/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_5.10host.json +++ b/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_5.10host.json @@ -1426,10 +1426,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0x4b564d00", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_6.1host.json b/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_6.1host.json index 17520de539a..e0330895de7 100644 --- a/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_6.1host.json +++ b/tests/data/cpu_template_helper/fingerprint_AMD_MILAN_6.1host.json @@ -1472,10 +1472,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0x4b564d00", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json index 65faa69f424..bb9ce77cb16 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json @@ -962,10 +962,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0xd90", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json index 31f8dad2924..8568992dcef 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json @@ -1158,10 +1158,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0xd90", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_6.1host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_6.1host.json index d19fc9254bf..fc9ee8dddf8 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_6.1host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_6.1host.json @@ -1158,10 +1158,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0xd90", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json index 3c0c51dd927..61c717defd6 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json @@ -916,10 +916,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0x4b564d00", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json index 863ffd2851d..3a6c8d5123e 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json @@ -1250,10 +1250,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0x4b564d00", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_6.1host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_6.1host.json index c5fd825edd0..d307c491d2a 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_6.1host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_ICELAKE_6.1host.json @@ -1273,10 +1273,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0x4b564d00", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json index 1714ac56edd..35c73d0e8af 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json @@ -962,10 +962,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0xd90", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json index 1681b9f7d24..c80924adefa 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json @@ -1158,10 +1158,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0xd90", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" diff --git a/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_6.1host.json b/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_6.1host.json index 6fc76c794e1..324619e3268 100644 --- a/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_6.1host.json +++ b/tests/data/cpu_template_helper/fingerprint_INTEL_SKYLAKE_6.1host.json @@ -1158,10 +1158,6 @@ "addr": "0x277", "bitmap": "0b0000000000000111000001000000011000000000000001110000010000000110" }, - { - "addr": "0x6e0", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000" - }, { "addr": "0xd90", "bitmap": "0b0000000000000000000000000000000000000000000000000000000000000000"