From d98faada6977c7554cdc085282d03ba22062110a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 25 Jul 2024 16:07:09 +0800 Subject: [PATCH 1/8] Implement grain mod --- src/util/metadata/side_metadata/grain.rs | 159 +++++++++++++++++++++++ src/util/metadata/side_metadata/mod.rs | 1 + 2 files changed, 160 insertions(+) create mode 100644 src/util/metadata/side_metadata/grain.rs diff --git a/src/util/metadata/side_metadata/grain.rs b/src/util/metadata/side_metadata/grain.rs new file mode 100644 index 0000000000..3ec3f2085e --- /dev/null +++ b/src/util/metadata/side_metadata/grain.rs @@ -0,0 +1,159 @@ +//! Mechanisms for breaking a range in a bitmap into whole-grain and sub-grain ranges to maximize +//! bulk bitmap access. +//! +//! In this module, the term "grain" refers to the unit at which we access a bitmap. It can be a +//! byte (u8), a word (usize) or other power-of-two byte sizes. +//! +//! The `std::simd` module is still a nightly feature as of Rust 1.79. When it stablizes, we can +//! allow the granularity to be a SIMD vector, too. + +use num_traits::Num; + +use crate::util::{constants::BITS_IN_BYTE, Address}; + +/// Offset of bits in a grain. +type BitOffset = usize; + +/// Bit address within a grain. +#[derive(Debug, Clone, Copy)] +pub struct BitAddress { + pub addr: Address, + pub bit: BitOffset, +} + +#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)] +pub struct Granularity { + log_bytes: usize, +} + +impl Granularity { + pub const fn of_log_bytes(log_bytes: usize) -> Self { + Self { log_bytes } + } + + pub const fn of_log_bits(log_bits: usize) -> Self { + Self { + log_bytes: log_bits >> BITS_IN_BYTE, + } + } + + pub fn of_type() -> Self { + let bytes = std::mem::size_of::(); + let log_bytes = bytes.ilog2() as usize; + + debug_assert_eq!(1 << log_bytes, bytes, "Not power-of-two size: {bytes}"); + + Self::of_log_bytes(log_bytes) + } + + pub const fn log_bytes(&self) -> usize { + self.log_bytes + } + + pub const fn log_bits(&self) -> usize { + self.log_bytes() << BITS_IN_BYTE + } + + pub const fn bytes(&self) -> usize { + 1 << self.log_bytes() + } + + pub const fn bits(&self) -> usize { + 1 << self.log_bits() + } +} + +impl BitAddress { + pub fn is_grain_aligned(&self, granularity: Granularity) -> bool { + self.bit == 0 && self.addr.is_aligned_to(granularity.bytes()) + } + + pub fn is_normalized(&self, granularity: Granularity) -> bool { + self.addr.is_aligned_to(granularity.bytes()) && self.bit < granularity.bytes() + } + + pub fn normalize(&self, granularity: Granularity) -> Self { + let addr = self.addr.align_down(granularity.bytes()); + let rem_bytes = addr.as_usize() % granularity.bytes(); + let bit = self.bit + rem_bytes * BITS_IN_BYTE; + Self { addr, bit } + } + + pub fn align_down_to_grain(&self, granularity: Granularity) -> Address { + debug_assert!(self.is_normalized(granularity)); + self.addr.align_down(granularity.bytes()) + } + + pub fn align_up_to_grain(&self, granularity: Granularity) -> Address { + debug_assert!(self.is_normalized(granularity)); + if self.bit == 0 { + (self.addr + 1usize).align_up(granularity.bytes()) + } else { + self.addr.align_up(granularity.bytes()) + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum VisitRange { + WholeGrain { + start: Address, + end: Address, + }, + SubGrain { + addr: Address, + bit_start: BitOffset, + bit_end: BitOffset, + }, +} + +pub fn break_range( + granularity: Granularity, + start: BitAddress, + end: BitAddress, +) -> Vec { + debug_assert!( + start.is_normalized(granularity), + "{start:?} is not normalized for {granularity:?}" + ); + debug_assert!( + end.is_normalized(granularity), + "{end:?} is not normalized for {granularity:?}" + ); + + if start.addr == end.addr { + // The start and the end are in the same grain. Yield only one SubGrain range. + return vec![VisitRange::SubGrain { + addr: start.addr, + bit_start: start.bit, + bit_end: end.bit, + }]; + } + + // Start with a sub-grain range if the start is not aligned. + let start_subgrain = (!start.is_grain_aligned(granularity)).then(|| VisitRange::SubGrain { + addr: start.addr, + bit_start: start.bit, + bit_end: granularity.bits(), + }); + + // Yield a whole-grain in the middle if long enough. + let start_whole = start.align_up_to_grain(granularity); + let end_whole = end.align_down_to_grain(granularity); + let whole = (start_whole < end_whole).then(|| VisitRange::WholeGrain { + start: start_whole, + end: end_whole, + }); + + // Finally yield a sub-grain range in the end if not aligned. + let end_subgrain = (!end.is_grain_aligned(granularity)).then(|| VisitRange::SubGrain { + addr: end.addr, + bit_start: 0, + bit_end: end.bit, + }); + + [start_subgrain, whole, end_subgrain] + .into_iter() + .flatten() + .collect() +} diff --git a/src/util/metadata/side_metadata/mod.rs b/src/util/metadata/side_metadata/mod.rs index 406f91167d..c3dcead277 100644 --- a/src/util/metadata/side_metadata/mod.rs +++ b/src/util/metadata/side_metadata/mod.rs @@ -1,6 +1,7 @@ //! This module provides an implementation of side table metadata. // For convenience, this module is public and the bindings may create and use side metadata for their purpose. +mod grain; mod constants; mod helpers; #[cfg(target_pointer_width = "32")] From 2f4e38062cac479a06b4a95030cda509ddb1542d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 25 Jul 2024 22:11:26 +0800 Subject: [PATCH 2/8] Fix stupid mistakes and add tests --- src/util/metadata/side_metadata/grain.rs | 298 +++++++++++++++++++++-- 1 file changed, 274 insertions(+), 24 deletions(-) diff --git a/src/util/metadata/side_metadata/grain.rs b/src/util/metadata/side_metadata/grain.rs index 3ec3f2085e..df9089b775 100644 --- a/src/util/metadata/side_metadata/grain.rs +++ b/src/util/metadata/side_metadata/grain.rs @@ -7,20 +7,16 @@ //! The `std::simd` module is still a nightly feature as of Rust 1.79. When it stablizes, we can //! allow the granularity to be a SIMD vector, too. -use num_traits::Num; +use num_traits::{Num, PrimInt}; -use crate::util::{constants::BITS_IN_BYTE, Address}; +use crate::util::{ + constants::{BITS_IN_BYTE, LOG_BITS_IN_BYTE}, + Address, +}; /// Offset of bits in a grain. type BitOffset = usize; -/// Bit address within a grain. -#[derive(Debug, Clone, Copy)] -pub struct BitAddress { - pub addr: Address, - pub bit: BitOffset, -} - #[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)] pub struct Granularity { log_bytes: usize, @@ -33,16 +29,13 @@ impl Granularity { pub const fn of_log_bits(log_bits: usize) -> Self { Self { - log_bytes: log_bits >> BITS_IN_BYTE, + log_bytes: log_bits >> LOG_BITS_IN_BYTE, } } - pub fn of_type() -> Self { + pub const fn of_type() -> Self { let bytes = std::mem::size_of::(); let log_bytes = bytes.ilog2() as usize; - - debug_assert_eq!(1 << log_bytes, bytes, "Not power-of-two size: {bytes}"); - Self::of_log_bytes(log_bytes) } @@ -51,7 +44,7 @@ impl Granularity { } pub const fn log_bits(&self) -> usize { - self.log_bytes() << BITS_IN_BYTE + self.log_bytes() + LOG_BITS_IN_BYTE as usize } pub const fn bytes(&self) -> usize { @@ -63,13 +56,20 @@ impl Granularity { } } +/// Bit address within a grain. +#[derive(Debug, Clone, Copy)] +pub struct BitAddress { + pub addr: Address, + pub bit: BitOffset, +} + impl BitAddress { pub fn is_grain_aligned(&self, granularity: Granularity) -> bool { self.bit == 0 && self.addr.is_aligned_to(granularity.bytes()) } pub fn is_normalized(&self, granularity: Granularity) -> bool { - self.addr.is_aligned_to(granularity.bytes()) && self.bit < granularity.bytes() + self.addr.is_aligned_to(granularity.bytes()) && self.bit < granularity.bits() } pub fn normalize(&self, granularity: Granularity) -> Self { @@ -86,7 +86,7 @@ impl BitAddress { pub fn align_up_to_grain(&self, granularity: Granularity) -> Address { debug_assert!(self.is_normalized(granularity)); - if self.bit == 0 { + if self.bit > 0 { (self.addr + 1usize).align_up(granularity.bytes()) } else { self.addr.align_up(granularity.bytes()) @@ -94,7 +94,20 @@ impl BitAddress { } } -#[derive(Debug, Clone, Copy)] +pub trait AddressToBitAddress { + fn with_bit_offset(&self, offset: BitOffset) -> BitAddress; +} + +impl AddressToBitAddress for Address { + fn with_bit_offset(&self, offset: BitOffset) -> BitAddress { + BitAddress { + addr: *self, + bit: offset, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VisitRange { WholeGrain { start: Address, @@ -107,6 +120,25 @@ pub enum VisitRange { }, } +pub fn bit_mask(granularity: Granularity, bit_start: usize, bit_end: usize) -> T { + debug_assert!( + bit_start < bit_end, + "bit_start ({bit_start}) must be less than bit_end ({bit_end})" + ); + debug_assert!( + bit_end <= granularity.bits(), + "Bit offset too high. Granularity is {} bits, bit_end is {}", + granularity.bits(), + bit_end + ); + + if bit_end == granularity.bits() { + !T::zero() << bit_start + } else { + (!T::zero() << bit_start) & !(!T::zero() << bit_end) + } +} + pub fn break_range( granularity: Granularity, start: BitAddress, @@ -121,13 +153,21 @@ pub fn break_range( "{end:?} is not normalized for {granularity:?}" ); + warn!("break_range: {granularity:?} {start:?} {end:?}"); + if start.addr == end.addr { - // The start and the end are in the same grain. Yield only one SubGrain range. - return vec![VisitRange::SubGrain { - addr: start.addr, - bit_start: start.bit, - bit_end: end.bit, - }]; + // The start and the end are in the same grain. + if start.bit == end.bit { + // Empty. + return vec![]; + } else { + // Yield only one SubGrain range. + return vec![VisitRange::SubGrain { + addr: start.addr, + bit_start: start.bit, + bit_end: end.bit, + }]; + } } // Start with a sub-grain range if the start is not aligned. @@ -157,3 +197,213 @@ pub fn break_range( .flatten() .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + fn mk_addr(addr: usize) -> Address { + unsafe { Address::from_usize(addr) } + } + + fn mk_ba(addr: usize, bit: usize) -> BitAddress { + BitAddress { + addr: mk_addr(addr), + bit, + } + } + + const G64BIT: Granularity = Granularity::of_log_bytes(3); + + #[test] + fn test_empty_range() { + let g = G64BIT; + let base = 0x1000; + for bit in 0..g.bits() { + let ba = mk_ba(base, bit); + let result = break_range(g, ba, ba); + assert!( + result.is_empty(), + "Not empty. bit: {bit}, result: {result:?}" + ); + } + } + + #[test] + fn test_subgrain_range() { + let g = G64BIT; + let base = 0x1000; + for bit0 in 0..g.bits() { + let ba0 = mk_ba(base, bit0); + for bit1 in (bit0 + 1)..g.bits() { + let ba1 = mk_ba(base, bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: bit1 + }], + "Not equal. bit0: {bit0}, bit1: {bit1}", + ); + } + } + } + + #[test] + fn test_end_grain_range() { + let g = G64BIT; + let base = 0x1000; + let ba1 = mk_ba(base + g.bytes(), 0); + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base, bit0); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: g.bits() + }], + "Not equal. bit0: {bit0}", + ); + } + } + + #[test] + fn test_adjacent_grain_range() { + let g = G64BIT; + let base = 0x1000; + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base, bit0); + for bit1 in 1..g.bits() { + let ba1 = mk_ba(base + g.bytes(), bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: g.bits(), + }, + VisitRange::SubGrain { + addr: mk_addr(base + g.bytes()), + bit_start: 0, + bit_end: bit1, + }, + ], + "Not equal. bit0: {bit0}, bit1: {bit1}", + ); + } + } + } + + #[test] + fn test_left_and_whole_range() { + let g = G64BIT; + let base = 0x1000; + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base, bit0); + for word1 in 2..8 { + let ba1 = mk_ba(base + word1 * g.bytes(), 0); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: g.bits(), + }, + VisitRange::WholeGrain { + start: mk_addr(base + g.bytes()), + end: mk_addr(base + word1 * g.bytes()), + }, + ], + "Not equal. bit0: {bit0}, word1: {word1}", + ); + } + } + } + + #[test] + fn test_whole_and_right_range() { + let g = G64BIT; + let base = 0x1000; + for word0 in 1..8 { + let ba0 = mk_ba(base - word0 * g.bytes(), 0); + for bit1 in 1..g.bits() { + let ba1 = mk_ba(base, bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::WholeGrain { + start: mk_addr(base - word0 * g.bytes()), + end: mk_addr(base), + }, + VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: 0, + bit_end: bit1, + }, + ], + "Not equal. word0: {word0}, bit1: {bit1}", + ); + } + } + } + + #[test] + fn test_whole_range() { + let g = G64BIT; + let base = 0x1000; + let ba0 = mk_ba(base, 0); + let ba1 = mk_ba(base + 42 * g.bytes(), 0); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![VisitRange::WholeGrain { + start: mk_addr(base), + end: mk_addr(base + 42 * g.bytes()), + },], + ); + } + + #[test] + fn test_left_whole_right_range() { + let g = G64BIT; + let base0 = 0x1000; + let base1 = 0x2000; + + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base0 - g.bytes(), bit0); + for bit1 in 1..g.bits() { + let ba1 = mk_ba(base1, bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::SubGrain { + addr: mk_addr(base0 - g.bytes()), + bit_start: bit0, + bit_end: g.bits(), + }, + VisitRange::WholeGrain { + start: mk_addr(base0), + end: mk_addr(base1), + }, + VisitRange::SubGrain { + addr: mk_addr(base1), + bit_start: 0, + bit_end: bit1, + }, + ], + "Not equal. bit0: {bit0}, bit1: {bit1}", + ); + } + } + } +} From 2b3cbd08be89782543036ef52b16d8de9904fe61 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 25 Jul 2024 22:54:57 +0800 Subject: [PATCH 3/8] More fixes and tests --- src/util/metadata/side_metadata/grain.rs | 364 +++++++++++++---------- src/util/metadata/side_metadata/mod.rs | 2 +- 2 files changed, 210 insertions(+), 156 deletions(-) diff --git a/src/util/metadata/side_metadata/grain.rs b/src/util/metadata/side_metadata/grain.rs index df9089b775..bbc78a9d11 100644 --- a/src/util/metadata/side_metadata/grain.rs +++ b/src/util/metadata/side_metadata/grain.rs @@ -7,12 +7,9 @@ //! The `std::simd` module is still a nightly feature as of Rust 1.79. When it stablizes, we can //! allow the granularity to be a SIMD vector, too. -use num_traits::{Num, PrimInt}; +use num_traits::PrimInt; -use crate::util::{ - constants::{BITS_IN_BYTE, LOG_BITS_IN_BYTE}, - Address, -}; +use crate::util::{constants::LOG_BITS_IN_BYTE, Address}; /// Offset of bits in a grain. type BitOffset = usize; @@ -33,7 +30,7 @@ impl Granularity { } } - pub const fn of_type() -> Self { + pub const fn of_type() -> Self { let bytes = std::mem::size_of::(); let log_bytes = bytes.ilog2() as usize; Self::of_log_bytes(log_bytes) @@ -57,7 +54,7 @@ impl Granularity { } /// Bit address within a grain. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct BitAddress { pub addr: Address, pub bit: BitOffset, @@ -73,9 +70,16 @@ impl BitAddress { } pub fn normalize(&self, granularity: Granularity) -> Self { - let addr = self.addr.align_down(granularity.bytes()); - let rem_bytes = addr.as_usize() % granularity.bytes(); - let bit = self.bit + rem_bytes * BITS_IN_BYTE; + // Transfer unaligned bytes from addr to bits + let aligned_addr = self.addr.align_down(granularity.bytes()); + let rem_bytes = self.addr - aligned_addr; + let rem_bits = self.bit + (rem_bytes << LOG_BITS_IN_BYTE); + + // Transfer bits outside granularity back to addr + let bit = rem_bits % granularity.bits(); + let carry_bits = rem_bits - bit; + let addr = aligned_addr + (carry_bits >> LOG_BITS_IN_BYTE); + Self { addr, bit } } @@ -213,196 +217,246 @@ mod tests { } } + const G8BIT: Granularity = Granularity::of_log_bytes(0); const G64BIT: Granularity = Granularity::of_log_bytes(3); + const GS: [Granularity; 2] = [G8BIT, G64BIT]; + + #[test] + fn test_grain_alignment() { + assert!(mk_ba(0x1000, 0).is_grain_aligned(G64BIT)); + assert!(!mk_ba(0x1000, 1).is_grain_aligned(G64BIT)); + assert!(!mk_ba(0x1001, 0).is_grain_aligned(G64BIT)); + assert!(!mk_ba(0x1001, 3).is_grain_aligned(G64BIT)); + + assert!(mk_ba(0x1001, 0).is_grain_aligned(G8BIT)); + assert!(!mk_ba(0x1001, 3).is_grain_aligned(G8BIT)); + } + + #[test] + fn test_is_normalized() { + assert!(mk_ba(0x1000, 0).is_normalized(G64BIT)); + assert!(mk_ba(0x1000, 63).is_normalized(G64BIT)); + assert!(!mk_ba(0x1000, 64).is_normalized(G64BIT)); + assert!(!mk_ba(0x1001, 0).is_normalized(G64BIT)); + assert!(!mk_ba(0x1007, 0).is_normalized(G64BIT)); + assert!(!mk_ba(0x1007, 63).is_normalized(G64BIT)); + assert!(mk_ba(0x1008, 0).is_normalized(G64BIT)); + + assert!(mk_ba(0x1000, 0).is_normalized(G8BIT)); + assert!(mk_ba(0x1000, 7).is_normalized(G8BIT)); + assert!(!mk_ba(0x1000, 8).is_normalized(G8BIT)); + assert!(mk_ba(0x1001, 0).is_normalized(G8BIT)); + } + + #[test] + fn test_normalize() { + assert_eq!(mk_ba(0x1000, 0).normalize(G64BIT), mk_ba(0x1000, 0)); + assert_eq!(mk_ba(0x1000, 63).normalize(G64BIT), mk_ba(0x1000, 63)); + assert_eq!(mk_ba(0x1000, 64).normalize(G64BIT), mk_ba(0x1008, 0)); + assert_eq!(mk_ba(0x1000, 65).normalize(G64BIT), mk_ba(0x1008, 1)); + assert_eq!(mk_ba(0x1001, 0).normalize(G64BIT), mk_ba(0x1000, 8)); + assert_eq!(mk_ba(0x1007, 0).normalize(G64BIT), mk_ba(0x1000, 56)); + assert_eq!(mk_ba(0x1007, 7).normalize(G64BIT), mk_ba(0x1000, 63)); + assert_eq!(mk_ba(0x1007, 8).normalize(G64BIT), mk_ba(0x1008, 0)); + assert_eq!(mk_ba(0x1007, 9).normalize(G64BIT), mk_ba(0x1008, 1)); + } #[test] fn test_empty_range() { - let g = G64BIT; - let base = 0x1000; - for bit in 0..g.bits() { - let ba = mk_ba(base, bit); - let result = break_range(g, ba, ba); - assert!( - result.is_empty(), - "Not empty. bit: {bit}, result: {result:?}" - ); + for g in GS { + let base = 0x1000; + for bit in 0..g.bits() { + let ba = mk_ba(base, bit); + let result = break_range(g, ba, ba); + assert!( + result.is_empty(), + "Not empty. bit: {bit}, result: {result:?}" + ); + } } } #[test] fn test_subgrain_range() { - let g = G64BIT; - let base = 0x1000; - for bit0 in 0..g.bits() { - let ba0 = mk_ba(base, bit0); - for bit1 in (bit0 + 1)..g.bits() { - let ba1 = mk_ba(base, bit1); + for g in GS { + let base = 0x1000; + for bit0 in 0..g.bits() { + let ba0 = mk_ba(base, bit0); + for bit1 in (bit0 + 1)..g.bits() { + let ba1 = mk_ba(base, bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: bit1 + }], + "Not equal. bit0: {bit0}, bit1: {bit1}", + ); + } + } + } + } + + #[test] + fn test_end_grain_range() { + for g in GS { + let base = 0x1000; + let ba1 = mk_ba(base + g.bytes(), 0); + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base, bit0); let result = break_range(g, ba0, ba1); assert_eq!( result, vec![VisitRange::SubGrain { addr: mk_addr(base), bit_start: bit0, - bit_end: bit1 + bit_end: g.bits() }], - "Not equal. bit0: {bit0}, bit1: {bit1}", + "Not equal. bit0: {bit0}", ); } } } - #[test] - fn test_end_grain_range() { - let g = G64BIT; - let base = 0x1000; - let ba1 = mk_ba(base + g.bytes(), 0); - for bit0 in 1..g.bits() { - let ba0 = mk_ba(base, bit0); - let result = break_range(g, ba0, ba1); - assert_eq!( - result, - vec![VisitRange::SubGrain { - addr: mk_addr(base), - bit_start: bit0, - bit_end: g.bits() - }], - "Not equal. bit0: {bit0}", - ); - } - } - #[test] fn test_adjacent_grain_range() { - let g = G64BIT; - let base = 0x1000; - for bit0 in 1..g.bits() { - let ba0 = mk_ba(base, bit0); - for bit1 in 1..g.bits() { - let ba1 = mk_ba(base + g.bytes(), bit1); - let result = break_range(g, ba0, ba1); - assert_eq!( - result, - vec![ - VisitRange::SubGrain { - addr: mk_addr(base), - bit_start: bit0, - bit_end: g.bits(), - }, - VisitRange::SubGrain { - addr: mk_addr(base + g.bytes()), - bit_start: 0, - bit_end: bit1, - }, - ], - "Not equal. bit0: {bit0}, bit1: {bit1}", - ); + for g in GS { + let base = 0x1000; + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base, bit0); + for bit1 in 1..g.bits() { + let ba1 = mk_ba(base + g.bytes(), bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: g.bits(), + }, + VisitRange::SubGrain { + addr: mk_addr(base + g.bytes()), + bit_start: 0, + bit_end: bit1, + }, + ], + "Not equal. bit0: {bit0}, bit1: {bit1}", + ); + } } } } #[test] fn test_left_and_whole_range() { - let g = G64BIT; - let base = 0x1000; - for bit0 in 1..g.bits() { - let ba0 = mk_ba(base, bit0); - for word1 in 2..8 { - let ba1 = mk_ba(base + word1 * g.bytes(), 0); - let result = break_range(g, ba0, ba1); - assert_eq!( - result, - vec![ - VisitRange::SubGrain { - addr: mk_addr(base), - bit_start: bit0, - bit_end: g.bits(), - }, - VisitRange::WholeGrain { - start: mk_addr(base + g.bytes()), - end: mk_addr(base + word1 * g.bytes()), - }, - ], - "Not equal. bit0: {bit0}, word1: {word1}", - ); + for g in GS { + let base = 0x1000; + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base, bit0); + for word1 in 2..8 { + let ba1 = mk_ba(base + word1 * g.bytes(), 0); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: bit0, + bit_end: g.bits(), + }, + VisitRange::WholeGrain { + start: mk_addr(base + g.bytes()), + end: mk_addr(base + word1 * g.bytes()), + }, + ], + "Not equal. bit0: {bit0}, word1: {word1}", + ); + } } } } #[test] fn test_whole_and_right_range() { - let g = G64BIT; - let base = 0x1000; - for word0 in 1..8 { - let ba0 = mk_ba(base - word0 * g.bytes(), 0); - for bit1 in 1..g.bits() { - let ba1 = mk_ba(base, bit1); - let result = break_range(g, ba0, ba1); - assert_eq!( - result, - vec![ - VisitRange::WholeGrain { - start: mk_addr(base - word0 * g.bytes()), - end: mk_addr(base), - }, - VisitRange::SubGrain { - addr: mk_addr(base), - bit_start: 0, - bit_end: bit1, - }, - ], - "Not equal. word0: {word0}, bit1: {bit1}", - ); + for g in GS { + let base = 0x1000; + for word0 in 1..8 { + let ba0 = mk_ba(base - word0 * g.bytes(), 0); + for bit1 in 1..g.bits() { + let ba1 = mk_ba(base, bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::WholeGrain { + start: mk_addr(base - word0 * g.bytes()), + end: mk_addr(base), + }, + VisitRange::SubGrain { + addr: mk_addr(base), + bit_start: 0, + bit_end: bit1, + }, + ], + "Not equal. word0: {word0}, bit1: {bit1}", + ); + } } } } #[test] fn test_whole_range() { - let g = G64BIT; - let base = 0x1000; - let ba0 = mk_ba(base, 0); - let ba1 = mk_ba(base + 42 * g.bytes(), 0); - let result = break_range(g, ba0, ba1); - assert_eq!( - result, - vec![VisitRange::WholeGrain { - start: mk_addr(base), - end: mk_addr(base + 42 * g.bytes()), - },], - ); + for g in GS { + let base = 0x1000; + let ba0 = mk_ba(base, 0); + let ba1 = mk_ba(base + 42 * g.bytes(), 0); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![VisitRange::WholeGrain { + start: mk_addr(base), + end: mk_addr(base + 42 * g.bytes()), + },], + ); + } } #[test] fn test_left_whole_right_range() { - let g = G64BIT; - let base0 = 0x1000; - let base1 = 0x2000; - - for bit0 in 1..g.bits() { - let ba0 = mk_ba(base0 - g.bytes(), bit0); - for bit1 in 1..g.bits() { - let ba1 = mk_ba(base1, bit1); - let result = break_range(g, ba0, ba1); - assert_eq!( - result, - vec![ - VisitRange::SubGrain { - addr: mk_addr(base0 - g.bytes()), - bit_start: bit0, - bit_end: g.bits(), - }, - VisitRange::WholeGrain { - start: mk_addr(base0), - end: mk_addr(base1), - }, - VisitRange::SubGrain { - addr: mk_addr(base1), - bit_start: 0, - bit_end: bit1, - }, - ], - "Not equal. bit0: {bit0}, bit1: {bit1}", - ); + for g in GS { + let base0 = 0x1000; + let base1 = 0x2000; + + for bit0 in 1..g.bits() { + let ba0 = mk_ba(base0 - g.bytes(), bit0); + for bit1 in 1..g.bits() { + let ba1 = mk_ba(base1, bit1); + let result = break_range(g, ba0, ba1); + assert_eq!( + result, + vec![ + VisitRange::SubGrain { + addr: mk_addr(base0 - g.bytes()), + bit_start: bit0, + bit_end: g.bits(), + }, + VisitRange::WholeGrain { + start: mk_addr(base0), + end: mk_addr(base1), + }, + VisitRange::SubGrain { + addr: mk_addr(base1), + bit_start: 0, + bit_end: bit1, + }, + ], + "Not equal. bit0: {bit0}, bit1: {bit1}", + ); + } } } } diff --git a/src/util/metadata/side_metadata/mod.rs b/src/util/metadata/side_metadata/mod.rs index c3dcead277..8ff38899f5 100644 --- a/src/util/metadata/side_metadata/mod.rs +++ b/src/util/metadata/side_metadata/mod.rs @@ -1,8 +1,8 @@ //! This module provides an implementation of side table metadata. // For convenience, this module is public and the bindings may create and use side metadata for their purpose. -mod grain; mod constants; +mod grain; mod helpers; #[cfg(target_pointer_width = "32")] mod helpers_32; From a4f38ec76e7dfbc9d98159999d13a7395e76faf1 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 26 Jul 2024 18:50:33 +0800 Subject: [PATCH 4/8] Implement bset and add bench --- benches/bulk_meta/bzero.rs | 30 +++++++ benches/bulk_meta/mod.rs | 1 + benches/main.rs | 14 +-- src/util/metadata/side_metadata/global.rs | 90 ++++++++++++++++++- .../side_metadata/side_metadata_tests.rs | 37 ++++++++ 5 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 benches/bulk_meta/bzero.rs create mode 100644 benches/bulk_meta/mod.rs diff --git a/benches/bulk_meta/bzero.rs b/benches/bulk_meta/bzero.rs new file mode 100644 index 0000000000..465fc7532a --- /dev/null +++ b/benches/bulk_meta/bzero.rs @@ -0,0 +1,30 @@ +use criterion::Criterion; +use mmtk::util::{metadata::side_metadata::SideMetadataSpec, Address}; + +pub fn bench(c: &mut Criterion) { + let size = 256usize; // Match an Immix line size. + + let allocate_u32 = || -> Address { + let ptr = unsafe { + std::alloc::alloc_zeroed(std::alloc::Layout::from_size_align(size, 8).unwrap()) + }; + Address::from_mut_ptr(ptr) + }; + + let start = allocate_u32(); + let end = start + size; + + c.bench_function("bzero_bset_modern", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits(start, 0, end, 0); + SideMetadataSpec::zero_meta_bits(start, 0, end, 0); + }) + }); + + c.bench_function("bzero_bset_classic", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_classic(start, 0, end, 0); + SideMetadataSpec::zero_meta_bits_classic(start, 0, end, 0); + }) + }); +} diff --git a/benches/bulk_meta/mod.rs b/benches/bulk_meta/mod.rs new file mode 100644 index 0000000000..fba67a17b3 --- /dev/null +++ b/benches/bulk_meta/mod.rs @@ -0,0 +1 @@ +pub mod bzero; diff --git a/benches/main.rs b/benches/main.rs index 6c735ce4e2..f254acdba1 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -19,6 +19,8 @@ use criterion::Criterion; // However, I will just keep these benchmarks here. If we find it not useful, and we do not plan to improve MockVM, we can delete // them. +mod bulk_meta; + #[cfg(feature = "mock_test")] mod mock_bench; @@ -29,15 +31,17 @@ pub fn bench_main(_c: &mut Criterion) { "alloc" => mock_bench::alloc::bench(_c), "internal_pointer" => mock_bench::internal_pointer::bench(_c), "sft" => mock_bench::sft::bench(_c), - _ => panic!("Unknown benchmark {:?}", bench), + _ => {} // Fall back to non-mock test. }, Err(_) => panic!("Need to name a benchmark by the env var MMTK_BENCH"), } - #[cfg(not(feature = "mock_test"))] - { - eprintln!("ERROR: Currently there are no benchmarks when the \"mock_test\" feature is not enabled."); - std::process::exit(1); + match std::env::var("MMTK_BENCH") { + Ok(bench) => match bench.as_str() { + "bzero" => bulk_meta::bzero::bench(_c), + _ => panic!("Unknown benchmark {:?}", bench), + }, + Err(_) => panic!("Need to name a benchmark by the env var MMTK_BENCH"), } } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 123698aa6e..76cbdeccfa 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -7,6 +7,8 @@ use crate::util::metadata::metadata_val_traits::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC; use crate::util::Address; +use atomic::Atomic; +use grain::{AddressToBitAddress, BitAddress, Granularity}; use num_traits::FromPrimitive; use std::fmt; use std::io::Result; @@ -282,7 +284,7 @@ impl SideMetadataSpec { } /// This method is used for bulk zeroing side metadata for a data address range. - pub(super) fn zero_meta_bits( + pub fn zero_meta_bits_classic( meta_start_addr: Address, meta_start_bit: u8, meta_end_addr: Address, @@ -310,8 +312,50 @@ impl SideMetadataSpec { ); } + pub fn zero_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + ) { + let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); + let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); + Self::zero_meta_bits_inner(meta_start, meta_end); + } + + pub(super) fn zero_meta_bits_inner(meta_start: BitAddress, meta_end: BitAddress) { + // eprintln!("zero_meta_bits({meta_start:?}, {meta_end:?})"); + + type Grain = usize; + let granularity = Granularity::of_type::(); + let meta_start = meta_start.normalize(granularity); + let meta_end = meta_end.normalize(granularity); + + if meta_start == meta_end { + panic!("Empty range?"); + } + + for range in grain::break_range(granularity, meta_start, meta_end) { + // eprintln!(" range: {range:?}"); + match range { + grain::VisitRange::WholeGrain { start, end } => { + memory::zero(start, end - start); + } + grain::VisitRange::SubGrain { + addr, + bit_start, + bit_end, + } => { + let mask = grain::bit_mask::(granularity, bit_start, bit_end); + // eprintln!(" mask: {mask:b}"); + unsafe { addr.as_ref::>() }.fetch_and(!mask, Ordering::SeqCst); + } + } + } + } + /// This method is used for bulk setting side metadata for a data address range. - pub(super) fn set_meta_bits( + pub fn set_meta_bits_classic( meta_start_addr: Address, meta_start_bit: u8, meta_end_addr: Address, @@ -339,6 +383,48 @@ impl SideMetadataSpec { ); } + pub fn set_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + ) { + let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); + let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); + Self::set_meta_bits_inner(meta_start, meta_end); + } + + pub(super) fn set_meta_bits_inner(meta_start: BitAddress, meta_end: BitAddress) { + // eprintln!("set_meta_bits({meta_start:?}, {meta_end:?})"); + + type Grain = usize; + let granularity = Granularity::of_type::(); + let meta_start = meta_start.normalize(granularity); + let meta_end = meta_end.normalize(granularity); + + if meta_start == meta_end { + panic!("Empty range?"); + } + + for range in grain::break_range(granularity, meta_start, meta_end) { + // eprintln!(" range: {range:?}"); + match range { + grain::VisitRange::WholeGrain { start, end } => { + memory::set(start, 0xffu8, end - start); + } + grain::VisitRange::SubGrain { + addr, + bit_start, + bit_end, + } => { + let mask = grain::bit_mask::(granularity, bit_start, bit_end); + // eprintln!(" mask: {mask:b}"); + unsafe { addr.as_ref::>() }.fetch_or(mask, Ordering::SeqCst); + } + } + } + } + /// This method does bulk update for the given data range. It calculates the metadata bits for the given data range, /// and invoke the given method to update the metadata bits. pub(super) fn bulk_update_metadata( diff --git a/src/util/metadata/side_metadata/side_metadata_tests.rs b/src/util/metadata/side_metadata/side_metadata_tests.rs index ab1abe01d3..ea7b952687 100644 --- a/src/util/metadata/side_metadata/side_metadata_tests.rs +++ b/src/util/metadata/side_metadata/side_metadata_tests.rs @@ -714,6 +714,43 @@ mod tests { assert_eq!(unsafe { start.load::() }, 0xC000_0003); // 1100....0011 } + #[test] + fn test_side_metadata_set_meta_bits() { + let size = 4usize; + let allocate_u32 = || -> Address { + let ptr = unsafe { + std::alloc::alloc_zeroed(std::alloc::Layout::from_size_align(size, 4).unwrap()) + }; + Address::from_mut_ptr(ptr) + }; + let fill_0 = |addr: Address| unsafe { + addr.store(0u32); + }; + + let start = allocate_u32(); + let end = start + size; + + fill_0(start); + // set the word + SideMetadataSpec::set_meta_bits(start, 0, end, 0); + assert_eq!(unsafe { start.load::() }, 0b1111_1111_1111_1111_1111_1111_1111_1111); + + fill_0(start); + // set first 2 bits + SideMetadataSpec::set_meta_bits(start, 0, start, 2); + assert_eq!(unsafe { start.load::() }, 0b0000_0000_0000_0000_0000_0000_0000_0011); + + fill_0(start); + // set last 2 bits + SideMetadataSpec::set_meta_bits(end - 1, 6, end, 0); + assert_eq!(unsafe { start.load::() }, 0b1100_0000_0000_0000_0000_0000_0000_0000); + + fill_0(start); + // set everything except first 2 bits and last 2 bits + SideMetadataSpec::set_meta_bits(start, 2, end - 1, 6); + assert_eq!(unsafe { start.load::() }, 0b0011_1111_1111_1111_1111_1111_1111_1100); + } + #[test] fn test_side_metadata_bcopy_metadata_contiguous() { serial_test(|| { From b855625b9afe14273a86d83c2621298dcad958d4 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 26 Jul 2024 19:42:19 +0800 Subject: [PATCH 5/8] Callback-based implementation --- benches/bulk_meta/bzero.rs | 7 ++ src/util/metadata/side_metadata/global.rs | 83 +++++++++++++++++++++++ src/util/metadata/side_metadata/grain.rs | 80 ++++++++++++++++++++++ 3 files changed, 170 insertions(+) diff --git a/benches/bulk_meta/bzero.rs b/benches/bulk_meta/bzero.rs index 465fc7532a..553f6ea747 100644 --- a/benches/bulk_meta/bzero.rs +++ b/benches/bulk_meta/bzero.rs @@ -21,6 +21,13 @@ pub fn bench(c: &mut Criterion) { }) }); + c.bench_function("bzero_bset_modern_callback", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_callback(start, 0, end, 0); + SideMetadataSpec::zero_meta_bits_callback(start, 0, end, 0); + }) + }); + c.bench_function("bzero_bset_classic", |b| { b.iter(|| { SideMetadataSpec::set_meta_bits_classic(start, 0, end, 0); diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 76cbdeccfa..b25e5a60e8 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -323,6 +323,17 @@ impl SideMetadataSpec { Self::zero_meta_bits_inner(meta_start, meta_end); } + pub fn zero_meta_bits_callback( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + ) { + let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); + let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); + Self::zero_meta_bits_inner_callback(meta_start, meta_end); + } + pub(super) fn zero_meta_bits_inner(meta_start: BitAddress, meta_end: BitAddress) { // eprintln!("zero_meta_bits({meta_start:?}, {meta_end:?})"); @@ -354,6 +365,38 @@ impl SideMetadataSpec { } } + pub(super) fn zero_meta_bits_inner_callback(meta_start: BitAddress, meta_end: BitAddress) { + // eprintln!("zero_meta_bits({meta_start:?}, {meta_end:?})"); + + type Grain = usize; + let granularity = Granularity::of_type::(); + let meta_start = meta_start.normalize(granularity); + let meta_end = meta_end.normalize(granularity); + + if meta_start == meta_end { + panic!("Empty range?"); + } + + grain::break_range_callback(granularity, meta_start, meta_end, true, &mut |range| { + // eprintln!(" range: {range:?}"); + match range { + grain::VisitRange::WholeGrain { start, end } => { + memory::zero(start, end - start); + } + grain::VisitRange::SubGrain { + addr, + bit_start, + bit_end, + } => { + let mask = grain::bit_mask::(granularity, bit_start, bit_end); + // eprintln!(" mask: {mask:b}"); + unsafe { addr.as_ref::>() }.fetch_and(!mask, Ordering::SeqCst); + } + } + false + }); + } + /// This method is used for bulk setting side metadata for a data address range. pub fn set_meta_bits_classic( meta_start_addr: Address, @@ -394,6 +437,17 @@ impl SideMetadataSpec { Self::set_meta_bits_inner(meta_start, meta_end); } + pub fn set_meta_bits_callback( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + ) { + let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); + let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); + Self::set_meta_bits_inner_callback(meta_start, meta_end); + } + pub(super) fn set_meta_bits_inner(meta_start: BitAddress, meta_end: BitAddress) { // eprintln!("set_meta_bits({meta_start:?}, {meta_end:?})"); @@ -425,6 +479,35 @@ impl SideMetadataSpec { } } + pub(super) fn set_meta_bits_inner_callback(meta_start: BitAddress, meta_end: BitAddress) { + type Grain = usize; + let granularity = Granularity::of_type::(); + let meta_start = meta_start.normalize(granularity); + let meta_end = meta_end.normalize(granularity); + + if meta_start == meta_end { + panic!("Empty range?"); + } + + grain::break_range_callback(granularity, meta_start, meta_end, true, &mut |range| { + match range { + grain::VisitRange::WholeGrain { start, end } => { + memory::set(start, 0xffu8, end - start); + } + grain::VisitRange::SubGrain { + addr, + bit_start, + bit_end, + } => { + let mask = grain::bit_mask::(granularity, bit_start, bit_end); + // eprintln!(" mask: {mask:b}"); + unsafe { addr.as_ref::>() }.fetch_or(mask, Ordering::SeqCst); + } + } + false + }); + } + /// This method does bulk update for the given data range. It calculates the metadata bits for the given data range, /// and invoke the given method to update the metadata bits. pub(super) fn bulk_update_metadata( diff --git a/src/util/metadata/side_metadata/grain.rs b/src/util/metadata/side_metadata/grain.rs index bbc78a9d11..6cb9df2d0c 100644 --- a/src/util/metadata/side_metadata/grain.rs +++ b/src/util/metadata/side_metadata/grain.rs @@ -202,6 +202,86 @@ pub fn break_range( .collect() } +pub fn break_range_callback( + granularity: Granularity, + start: BitAddress, + end: BitAddress, + forwards: bool, + visitor: &mut F, +) where + F: FnMut(VisitRange) -> bool, +{ + debug_assert!( + start.is_normalized(granularity), + "{start:?} is not normalized for {granularity:?}" + ); + debug_assert!( + end.is_normalized(granularity), + "{end:?} is not normalized for {granularity:?}" + ); + + warn!("break_range: {granularity:?} {start:?} {end:?}"); + + if start.addr == end.addr { + // The start and the end are in the same grain. + if start.bit == end.bit { + return; + } else { + // Yield only one SubGrain range. + visitor(VisitRange::SubGrain { + addr: start.addr, + bit_start: start.bit, + bit_end: end.bit, + }); + return; + } + } + + // Start with a sub-grain range if the start is not aligned. + let start_subgrain = |v: &mut F| { + if !start.is_grain_aligned(granularity) { + v(VisitRange::SubGrain { + addr: start.addr, + bit_start: start.bit, + bit_end: granularity.bits(), + }); + } + }; + + // Yield a whole-grain in the middle if long enough. + let start_whole = start.align_up_to_grain(granularity); + let end_whole = end.align_down_to_grain(granularity); + let whole = |v: &mut F| { + if start_whole < end_whole { + v(VisitRange::WholeGrain { + start: start_whole, + end: end_whole, + }); + } + }; + + // Finally yield a sub-grain range in the end if not aligned. + let end_subgrain = |v: &mut F| { + if !end.is_grain_aligned(granularity) { + v(VisitRange::SubGrain { + addr: end.addr, + bit_start: 0, + bit_end: end.bit, + }); + } + }; + + if forwards { + start_subgrain(visitor); + whole(visitor); + end_subgrain(visitor); + } else { + end_subgrain(visitor); + whole(visitor); + start_subgrain(visitor); + } +} + #[cfg(test)] mod tests { use super::*; From 1ecee45cba33e8d0adaaac3989c49b00756d4272 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 26 Jul 2024 20:23:32 +0800 Subject: [PATCH 6/8] Refactor and bench --- benches/bulk_meta/bzero.rs | 33 +++- src/util/metadata/side_metadata/global.rs | 223 ++++++++-------------- src/util/metadata/side_metadata/grain.rs | 4 + src/util/metadata/side_metadata/mod.rs | 2 +- 4 files changed, 114 insertions(+), 148 deletions(-) diff --git a/benches/bulk_meta/bzero.rs b/benches/bulk_meta/bzero.rs index 553f6ea747..e6b4864e5d 100644 --- a/benches/bulk_meta/bzero.rs +++ b/benches/bulk_meta/bzero.rs @@ -1,5 +1,8 @@ use criterion::Criterion; -use mmtk::util::{metadata::side_metadata::SideMetadataSpec, Address}; +use mmtk::util::{ + metadata::side_metadata::{grain::AddressToBitAddress, SideMetadataSpec}, + Address, +}; pub fn bench(c: &mut Criterion) { let size = 256usize; // Match an Immix line size. @@ -13,18 +16,34 @@ pub fn bench(c: &mut Criterion) { let start = allocate_u32(); let end = start + size; + let start_ba = start.with_bit_offset(0); + let end_ba = end.with_bit_offset(0); - c.bench_function("bzero_bset_modern", |b| { + c.bench_function("bzero_bset_modern_64", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits(start, 0, end, 0); - SideMetadataSpec::zero_meta_bits(start, 0, end, 0); + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); }) }); - c.bench_function("bzero_bset_modern_callback", |b| { + c.bench_function("bzero_bset_modern_64_callback", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits_callback(start, 0, end, 0); - SideMetadataSpec::zero_meta_bits_callback(start, 0, end, 0); + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); + }) + }); + + c.bench_function("bzero_bset_modern_32", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); + }) + }); + + c.bench_function("bzero_bset_modern_32_callback", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); }) }); diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index b25e5a60e8..ca04043e1e 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -7,9 +7,8 @@ use crate::util::metadata::metadata_val_traits::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC; use crate::util::Address; -use atomic::Atomic; -use grain::{AddressToBitAddress, BitAddress, Granularity}; -use num_traits::FromPrimitive; +use grain::{AddressToBitAddress, BitAddress, Granularity, VisitRange}; +use num_traits::{FromPrimitive, PrimInt}; use std::fmt; use std::io::Result; use std::sync::atomic::{AtomicU8, Ordering}; @@ -283,6 +282,28 @@ impl SideMetadataSpec { } } + pub fn iterate_meta_bits_modern( + meta_start: BitAddress, + meta_end: BitAddress, + granularity: Granularity, + visitor: &mut impl FnMut(VisitRange) -> bool, + ) { + //let meta_start = meta_start.normalize(granularity); + //let meta_end = meta_end.normalize(granularity); + + if meta_start == meta_end { + panic!("Empty range?"); + } + + if CALLBACK { + grain::break_range_callback(granularity, meta_start, meta_end, true, visitor); + } else { + for range in grain::break_range(granularity, meta_start, meta_end) { + visitor(range); + } + } + } + /// This method is used for bulk zeroing side metadata for a data address range. pub fn zero_meta_bits_classic( meta_start_addr: Address, @@ -320,81 +341,39 @@ impl SideMetadataSpec { ) { let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); - Self::zero_meta_bits_inner(meta_start, meta_end); + Self::zero_meta_bits_modern_inner::(meta_start, meta_end); } - pub fn zero_meta_bits_callback( - meta_start_addr: Address, - meta_start_bit: u8, - meta_end_addr: Address, - meta_end_bit: u8, + pub fn zero_meta_bits_modern_inner( + meta_start: BitAddress, + meta_end: BitAddress, ) { - let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); - let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); - Self::zero_meta_bits_inner_callback(meta_start, meta_end); - } - - pub(super) fn zero_meta_bits_inner(meta_start: BitAddress, meta_end: BitAddress) { - // eprintln!("zero_meta_bits({meta_start:?}, {meta_end:?})"); - - type Grain = usize; - let granularity = Granularity::of_type::(); - let meta_start = meta_start.normalize(granularity); - let meta_end = meta_end.normalize(granularity); - - if meta_start == meta_end { - panic!("Empty range?"); - } - - for range in grain::break_range(granularity, meta_start, meta_end) { - // eprintln!(" range: {range:?}"); - match range { - grain::VisitRange::WholeGrain { start, end } => { - memory::zero(start, end - start); - } - grain::VisitRange::SubGrain { - addr, - bit_start, - bit_end, - } => { - let mask = grain::bit_mask::(granularity, bit_start, bit_end); - // eprintln!(" mask: {mask:b}"); - unsafe { addr.as_ref::>() }.fetch_and(!mask, Ordering::SeqCst); - } - } - } - } - - pub(super) fn zero_meta_bits_inner_callback(meta_start: BitAddress, meta_end: BitAddress) { - // eprintln!("zero_meta_bits({meta_start:?}, {meta_end:?})"); - - type Grain = usize; let granularity = Granularity::of_type::(); - let meta_start = meta_start.normalize(granularity); - let meta_end = meta_end.normalize(granularity); - - if meta_start == meta_end { - panic!("Empty range?"); - } - - grain::break_range_callback(granularity, meta_start, meta_end, true, &mut |range| { - // eprintln!(" range: {range:?}"); - match range { - grain::VisitRange::WholeGrain { start, end } => { - memory::zero(start, end - start); - } - grain::VisitRange::SubGrain { - addr, - bit_start, - bit_end, - } => { - let mask = grain::bit_mask::(granularity, bit_start, bit_end); - // eprintln!(" mask: {mask:b}"); - unsafe { addr.as_ref::>() }.fetch_and(!mask, Ordering::SeqCst); + Self::iterate_meta_bits_modern::( + meta_start, + meta_end, + granularity, + &mut |range| { + // eprintln!(" range: {range:?}"); + match range { + grain::VisitRange::WholeGrain { start, end } => { + memory::zero(start, end - start); + } + grain::VisitRange::SubGrain { + addr, + bit_start, + bit_end, + } => { + let mask = grain::bit_mask::(granularity, bit_start, bit_end); + // eprintln!(" mask: {mask:b}"); + unsafe { + Grain::fetch_and(addr, !mask, Ordering::SeqCst); + } + } } - } - false - }); + false + }, + ); } /// This method is used for bulk setting side metadata for a data address range. @@ -434,78 +413,42 @@ impl SideMetadataSpec { ) { let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); - Self::set_meta_bits_inner(meta_start, meta_end); + Self::set_meta_bits_modern_inner::(meta_start, meta_end); } - pub fn set_meta_bits_callback( - meta_start_addr: Address, - meta_start_bit: u8, - meta_end_addr: Address, - meta_end_bit: u8, + pub fn set_meta_bits_modern_inner< + Grain: PrimInt + MetadataValue, + const CALLBACK: bool, + >( + meta_start: BitAddress, + meta_end: BitAddress, ) { - let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); - let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); - Self::set_meta_bits_inner_callback(meta_start, meta_end); - } - - pub(super) fn set_meta_bits_inner(meta_start: BitAddress, meta_end: BitAddress) { - // eprintln!("set_meta_bits({meta_start:?}, {meta_end:?})"); - - type Grain = usize; let granularity = Granularity::of_type::(); - let meta_start = meta_start.normalize(granularity); - let meta_end = meta_end.normalize(granularity); - - if meta_start == meta_end { - panic!("Empty range?"); - } - - for range in grain::break_range(granularity, meta_start, meta_end) { - // eprintln!(" range: {range:?}"); - match range { - grain::VisitRange::WholeGrain { start, end } => { - memory::set(start, 0xffu8, end - start); - } - grain::VisitRange::SubGrain { - addr, - bit_start, - bit_end, - } => { - let mask = grain::bit_mask::(granularity, bit_start, bit_end); - // eprintln!(" mask: {mask:b}"); - unsafe { addr.as_ref::>() }.fetch_or(mask, Ordering::SeqCst); - } - } - } - } - - pub(super) fn set_meta_bits_inner_callback(meta_start: BitAddress, meta_end: BitAddress) { - type Grain = usize; - let granularity = Granularity::of_type::(); - let meta_start = meta_start.normalize(granularity); - let meta_end = meta_end.normalize(granularity); - - if meta_start == meta_end { - panic!("Empty range?"); - } - - grain::break_range_callback(granularity, meta_start, meta_end, true, &mut |range| { - match range { - grain::VisitRange::WholeGrain { start, end } => { - memory::set(start, 0xffu8, end - start); - } - grain::VisitRange::SubGrain { - addr, - bit_start, - bit_end, - } => { - let mask = grain::bit_mask::(granularity, bit_start, bit_end); - // eprintln!(" mask: {mask:b}"); - unsafe { addr.as_ref::>() }.fetch_or(mask, Ordering::SeqCst); + Self::iterate_meta_bits_modern::( + meta_start, + meta_end, + granularity, + &mut |range| { + // eprintln!(" range: {range:?}"); + match range { + grain::VisitRange::WholeGrain { start, end } => { + memory::set(start, 0xffu8, end - start); + } + grain::VisitRange::SubGrain { + addr, + bit_start, + bit_end, + } => { + let mask = grain::bit_mask::(granularity, bit_start, bit_end); + // eprintln!(" mask: {mask:b}"); + unsafe { + Grain::fetch_or(addr, mask, Ordering::SeqCst); + } + } } - } - false - }); + false + }, + ); } /// This method does bulk update for the given data range. It calculates the metadata bits for the given data range, diff --git a/src/util/metadata/side_metadata/grain.rs b/src/util/metadata/side_metadata/grain.rs index 6cb9df2d0c..d809d817af 100644 --- a/src/util/metadata/side_metadata/grain.rs +++ b/src/util/metadata/side_metadata/grain.rs @@ -70,6 +70,10 @@ impl BitAddress { } pub fn normalize(&self, granularity: Granularity) -> Self { + if self.is_normalized(granularity) { + return *self; + } + // Transfer unaligned bytes from addr to bits let aligned_addr = self.addr.align_down(granularity.bytes()); let rem_bytes = self.addr - aligned_addr; diff --git a/src/util/metadata/side_metadata/mod.rs b/src/util/metadata/side_metadata/mod.rs index 8ff38899f5..10ff7cfcc7 100644 --- a/src/util/metadata/side_metadata/mod.rs +++ b/src/util/metadata/side_metadata/mod.rs @@ -2,7 +2,7 @@ // For convenience, this module is public and the bindings may create and use side metadata for their purpose. mod constants; -mod grain; +pub mod grain; mod helpers; #[cfg(target_pointer_width = "32")] mod helpers_32; From b2b18b0bb46b4f4b0ce574998640cee3b140521b Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 29 Jul 2024 15:55:13 +0800 Subject: [PATCH 7/8] more tests --- benches/bulk_meta/bzero.rs | 130 +++++++++++++++++++--- src/util/metadata/side_metadata/global.rs | 84 +++++++++++--- 2 files changed, 188 insertions(+), 26 deletions(-) diff --git a/benches/bulk_meta/bzero.rs b/benches/bulk_meta/bzero.rs index e6b4864e5d..1e40349bfe 100644 --- a/benches/bulk_meta/bzero.rs +++ b/benches/bulk_meta/bzero.rs @@ -1,56 +1,160 @@ use criterion::Criterion; use mmtk::util::{ + constants::{BITS_IN_BYTE, BYTES_IN_WORD}, metadata::side_metadata::{grain::AddressToBitAddress, SideMetadataSpec}, Address, }; pub fn bench(c: &mut Criterion) { - let size = 256usize; // Match an Immix line size. + let data_bytes = 256usize; // Match an Immix line size. + let meta_bits = data_bytes / BYTES_IN_WORD; + let meta_bytes = meta_bits / BITS_IN_BYTE; let allocate_u32 = || -> Address { let ptr = unsafe { - std::alloc::alloc_zeroed(std::alloc::Layout::from_size_align(size, 8).unwrap()) + std::alloc::alloc_zeroed(std::alloc::Layout::from_size_align(meta_bytes, 8).unwrap()) }; Address::from_mut_ptr(ptr) }; let start = allocate_u32(); - let end = start + size; + let end = start + meta_bytes; let start_ba = start.with_bit_offset(0); let end_ba = end.with_bit_offset(0); c.bench_function("bzero_bset_modern_64", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); - SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); }) }); c.bench_function("bzero_bset_modern_64_callback", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); - SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_64_callback_normalize", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_64_callback_normalize_fastpath", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); }) }); c.bench_function("bzero_bset_modern_32", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); - SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); }) }); c.bench_function("bzero_bset_modern_32_callback", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); - SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_32_callback_normalize", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_32_callback_normalize_fastpath", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_8", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_8_callback", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::( + start_ba, end_ba, + ); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_8_callback_normalize", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::( + start_ba, end_ba, + ); + }) + }); + + c.bench_function("bzero_bset_modern_8_callback_normalize_fastpath", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_modern_inner::(start_ba, end_ba); + SideMetadataSpec::zero_meta_bits_modern_inner::(start_ba, end_ba); }) }); c.bench_function("bzero_bset_classic", |b| { b.iter(|| { - SideMetadataSpec::set_meta_bits_classic(start, 0, end, 0); - SideMetadataSpec::zero_meta_bits_classic(start, 0, end, 0); + SideMetadataSpec::set_meta_bits_classic::(start, 0, end, 0); + SideMetadataSpec::zero_meta_bits_classic::(start, 0, end, 0); + }) + }); + + c.bench_function("bzero_bset_classic_fast", |b| { + b.iter(|| { + SideMetadataSpec::set_meta_bits_classic::(start, 0, end, 0); + SideMetadataSpec::zero_meta_bits_classic::(start, 0, end, 0); }) }); } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index ca04043e1e..7822c1398f 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -155,6 +155,26 @@ impl SideMetadataSpec { MMAPPER.is_mapped_address(meta_addr) } + pub(super) fn iterate_meta_bits( + meta_start_addr: Address, + meta_start_bit: u8, + meta_end_addr: Address, + meta_end_bit: u8, + forwards: bool, + visit_bytes: &impl Fn(Address, Address) -> bool, + visit_bits: &impl Fn(Address, u8, u8) -> bool, + ) -> bool { + Self::iterate_meta_bits_inner::( + meta_start_addr, + meta_start_bit, + meta_end_addr, + meta_end_bit, + forwards, + visit_bytes, + visit_bits, + ) + } + /// This method is used for iterating side metadata for a data address range. As we cannot guarantee /// that the data address range can be mapped to whole metadata bytes, we have to deal with cases that /// we need to mask and zero certain bits in a metadata byte. The end address and the end bit are exclusive. @@ -166,7 +186,7 @@ impl SideMetadataSpec { /// * `forwards`: If true, we iterate forwards (from start/low address to end/high address). Otherwise, /// we iterate backwards (from end/high address to start/low address). /// * `visit_bytes`/`visit_bits`: The closures returns whether the itertion is early terminated. - pub(super) fn iterate_meta_bits( + pub(super) fn iterate_meta_bits_inner( meta_start_addr: Address, meta_start_bit: u8, meta_end_addr: Address, @@ -182,6 +202,14 @@ impl SideMetadataSpec { meta_end_addr, meta_end_bit ); + + if FAST_PATH { + // zeroing bytes + if meta_start_bit == 0 && meta_end_bit == 0 { + return visit_bytes(meta_start_addr, meta_end_addr); + } + } + // Start/end is the same, we don't need to do anything. if meta_start_addr == meta_end_addr && meta_start_bit == meta_end_bit { return false; @@ -282,14 +310,37 @@ impl SideMetadataSpec { } } - pub fn iterate_meta_bits_modern( + pub fn iterate_meta_bits_modern< + const CALLBACK: bool, + const NORMALIZE: bool, + const FAST_PATH: bool, + >( meta_start: BitAddress, meta_end: BitAddress, granularity: Granularity, visitor: &mut impl FnMut(VisitRange) -> bool, ) { - //let meta_start = meta_start.normalize(granularity); - //let meta_end = meta_end.normalize(granularity); + if FAST_PATH + && meta_start.is_grain_aligned(granularity) + && meta_end.is_grain_aligned(granularity) + { + visitor(VisitRange::WholeGrain { + start: meta_start.addr, + end: meta_end.addr, + }); + return; + } + + let meta_start = if NORMALIZE { + meta_start.normalize(granularity) + } else { + meta_start + }; + let meta_end = if NORMALIZE { + meta_end.normalize(granularity) + } else { + meta_end + }; if meta_start == meta_end { panic!("Empty range?"); @@ -305,7 +356,7 @@ impl SideMetadataSpec { } /// This method is used for bulk zeroing side metadata for a data address range. - pub fn zero_meta_bits_classic( + pub fn zero_meta_bits_classic( meta_start_addr: Address, meta_start_bit: u8, meta_end_addr: Address, @@ -322,7 +373,7 @@ impl SideMetadataSpec { unsafe { addr.as_ref::() }.fetch_and(mask, Ordering::SeqCst); false }; - Self::iterate_meta_bits( + Self::iterate_meta_bits_inner::( meta_start_addr, meta_start_bit, meta_end_addr, @@ -341,15 +392,20 @@ impl SideMetadataSpec { ) { let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); - Self::zero_meta_bits_modern_inner::(meta_start, meta_end); + Self::zero_meta_bits_modern_inner::(meta_start, meta_end); } - pub fn zero_meta_bits_modern_inner( + pub fn zero_meta_bits_modern_inner< + Grain: PrimInt + MetadataValue, + const CALLBACK: bool, + const NORMALIZE: bool, + const FAST_PATH: bool, + >( meta_start: BitAddress, meta_end: BitAddress, ) { let granularity = Granularity::of_type::(); - Self::iterate_meta_bits_modern::( + Self::iterate_meta_bits_modern::( meta_start, meta_end, granularity, @@ -377,7 +433,7 @@ impl SideMetadataSpec { } /// This method is used for bulk setting side metadata for a data address range. - pub fn set_meta_bits_classic( + pub fn set_meta_bits_classic( meta_start_addr: Address, meta_start_bit: u8, meta_end_addr: Address, @@ -394,7 +450,7 @@ impl SideMetadataSpec { unsafe { addr.as_ref::() }.fetch_or(mask, Ordering::SeqCst); false }; - Self::iterate_meta_bits( + Self::iterate_meta_bits_inner::( meta_start_addr, meta_start_bit, meta_end_addr, @@ -413,18 +469,20 @@ impl SideMetadataSpec { ) { let meta_start = meta_start_addr.with_bit_offset(meta_start_bit as usize); let meta_end = meta_end_addr.with_bit_offset(meta_end_bit as usize); - Self::set_meta_bits_modern_inner::(meta_start, meta_end); + Self::set_meta_bits_modern_inner::(meta_start, meta_end); } pub fn set_meta_bits_modern_inner< Grain: PrimInt + MetadataValue, const CALLBACK: bool, + const NORMALIZE: bool, + const FAST_PATH: bool, >( meta_start: BitAddress, meta_end: BitAddress, ) { let granularity = Granularity::of_type::(); - Self::iterate_meta_bits_modern::( + Self::iterate_meta_bits_modern::( meta_start, meta_end, granularity, From f3750ee9a8000310537c8369f53c921fc788ff1e Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 29 Jul 2024 17:37:24 +0800 Subject: [PATCH 8/8] Implement address_to_contiguous_meta_bit_address --- src/util/metadata/side_metadata/helpers.rs | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index b9bf197dfb..80a251f8e1 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -1,3 +1,4 @@ +use super::grain::{BitAddress, Granularity}; use super::SideMetadataSpec; use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::constants::{BITS_IN_WORD, BYTES_IN_PAGE, LOG_BITS_IN_BYTE}; @@ -25,6 +26,43 @@ pub(super) fn address_to_contiguous_meta_address( } } +pub(super) fn address_to_contiguous_meta_bit_address( + metadata_spec: &SideMetadataSpec, + data_addr: Address, + granularity: Granularity, +) -> BitAddress { + let log_bits_num = metadata_spec.log_num_of_bits; + let log_bytes_in_region = metadata_spec.log_bytes_in_region; + + let meta_start = metadata_spec.get_absolute_offset(); + let num_regions_from_zero = data_addr >> log_bytes_in_region; + + if log_bits_num < granularity.log_bits() { + // Multiple data regions per metadata grain. + let log_regions_per_meta_grain = granularity.log_bits() - log_bits_num; + let meta_grains_from_start = num_regions_from_zero >> log_regions_per_meta_grain; + let meta_bytes_from_start = meta_grains_from_start << granularity.log_bytes(); + + let num_regions_in_meta_grain = num_regions_from_zero & (log_regions_per_meta_grain - 1); + let meta_in_grain_offset = num_regions_in_meta_grain << log_bits_num; + + BitAddress { + addr: meta_start + meta_bytes_from_start, + bit: meta_in_grain_offset, + } + } else { + // Multiple metadata grains per data range. + let log_meta_grains_per_region = log_bits_num - granularity.log_bits(); + let meta_grains_from_start = num_regions_from_zero << log_meta_grains_per_region; + let meta_bytes_from_start = meta_grains_from_start << granularity.log_bytes(); + + BitAddress { + addr: meta_start + meta_bytes_from_start, + bit: 0, + } + } +} + /// Performs reverse address translation from contiguous metadata bits to data addresses. /// /// Arguments: @@ -166,6 +204,32 @@ pub(super) fn meta_byte_mask(metadata_spec: &SideMetadataSpec) -> u8 { ((1usize << (1usize << bits_num_log)) - 1) as u8 } +pub(crate) fn address_to_meta_bit_address( + metadata_spec: &SideMetadataSpec, + data_addr: Address, + granularity: Granularity, +) -> BitAddress { + #[cfg(target_pointer_width = "32")] + let res = { + if metadata_spec.is_global { + address_to_contiguous_meta_address(metadata_spec, data_addr) + } else { + address_to_chunked_meta_address(metadata_spec, data_addr) + } + }; + #[cfg(target_pointer_width = "64")] + let res = { address_to_contiguous_meta_bit_address(metadata_spec, data_addr, granularity) }; + + trace!( + "address_to_meta_bit_address({:?}, addr: {}) -> 0x{:?}", + metadata_spec, + data_addr, + res + ); + + res +} + /// The result type for find meta bits functions. pub enum FindMetaBitResult { Found { addr: Address, bit: u8 },