From e35d56b971f4e1dda8cde421d6de73d27f7a5b0f Mon Sep 17 00:00:00 2001 From: Andreas Molzer Date: Wed, 2 Oct 2019 19:43:10 +0200 Subject: [PATCH 1/2] Avoid copying memory representation of undef data During MIR interpretation it may happen that a place containing uninitialized bytes is copied. This would read the current representation of these bytes and write it to the destination even though they must, by definition, not matter to the execution. This elides that representation change when no bytes are defined in such a copy, saving some cpu cycles. In such a case, the memory of the target allocation is not touched at all which also means that sometimes no physical page backing the memory allocation of the representation needs to be provided by the OS at all, reducing memory pressure on the system. --- src/librustc/mir/interpret/allocation.rs | 6 ++++ src/librustc_mir/interpret/memory.rs | 43 +++++++++++------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/librustc/mir/interpret/allocation.rs b/src/librustc/mir/interpret/allocation.rs index f849361e08be9..22dafaf4e0fba 100644 --- a/src/librustc/mir/interpret/allocation.rs +++ b/src/librustc/mir/interpret/allocation.rs @@ -695,6 +695,12 @@ impl Allocation { } } +impl AllocationDefinedness { + pub fn all_bytes_undef(&self) -> bool { + self.initial == false && self.ranges.len() == 1 + } +} + /// Relocations. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, RustcEncodable, RustcDecodable)] pub struct Relocations(SortedMap); diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index ee7fb18fd05a5..f6ab6d0c8537c 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -855,6 +855,22 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { let relocations = self.get_raw(src.alloc_id)? .prepare_relocation_copy(self, src, size, dest, length); + // Prepare a copy of the undef mask. + let compressed = self.get_raw(src.alloc_id)?.compress_undef_range(src, size); + + if compressed.all_bytes_undef() { + // Fast path: If all bytes are `undef` then there is nothing to copy. The target range + // is marked as undef but we otherwise omit changing the byte representation which may + // be arbitrary for undef bytes. + // This also avoids writing to the target bytes so that the backing allocation is never + // touched if the bytes stay undef for the whole interpreter execution. On contemporary + // operating system this can avoid physically allocating the page. + let dest_alloc = self.get_raw_mut(dest.alloc_id)?; + dest_alloc.mark_definedness(dest, size * length, false); + dest_alloc.mark_relocation_range(relocations); + return Ok(()); + } + let tcx = self.tcx.tcx; // This checks relocation edges on the src. @@ -897,8 +913,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } } - // copy definedness to the destination - self.copy_undef_mask(src, dest, size, length)?; + // now copy over the undef data + self.get_raw_mut(dest.alloc_id)? + .mark_compressed_undef_range(&compressed, dest, size, length); + // copy the relocations to the destination self.get_raw_mut(dest.alloc_id)?.mark_relocation_range(relocations); @@ -908,27 +926,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { /// Undefined bytes impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { - // FIXME: Add a fast version for the common, nonoverlapping case - fn copy_undef_mask( - &mut self, - src: Pointer, - dest: Pointer, - size: Size, - repeat: u64, - ) -> InterpResult<'tcx> { - // The bits have to be saved locally before writing to dest in case src and dest overlap. - assert_eq!(size.bytes() as usize as u64, size.bytes()); - - let src_alloc = self.get_raw(src.alloc_id)?; - let compressed = src_alloc.compress_undef_range(src, size); - - // now fill in all the data - let dest_allocation = self.get_raw_mut(dest.alloc_id)?; - dest_allocation.mark_compressed_undef_range(&compressed, dest, size, repeat); - - Ok(()) - } - pub fn force_ptr( &self, scalar: Scalar, From df72632c69326870205e04901a8c74c3ffe59b77 Mon Sep 17 00:00:00 2001 From: Andreas Molzer Date: Mon, 2 Dec 2019 21:08:42 +0100 Subject: [PATCH 2/2] Remove obsolete uninit mention from impl comment --- src/librustc_mir/interpret/memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/librustc_mir/interpret/memory.rs b/src/librustc_mir/interpret/memory.rs index f6ab6d0c8537c..dd9f769ead451 100644 --- a/src/librustc_mir/interpret/memory.rs +++ b/src/librustc_mir/interpret/memory.rs @@ -924,7 +924,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } } -/// Undefined bytes +/// Machine pointer introspection. impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { pub fn force_ptr( &self,