Simplify slice::Iter::next enough that it inlines

Author: scottmcm

library/core/src/slice/iter.rs

@@ -11,7 +11,7 @@ use crate::iter::{
 use crate::marker::PhantomData;
 use crate::mem::{self, SizedTypeProperties};
 use crate::num::NonZero;
-use crate::ptr::{NonNull, without_provenance, without_provenance_mut};
+use crate::ptr::{NonNull, null, without_provenance, without_provenance_mut};
 use crate::{cmp, fmt};
 
 #[stable(feature = "boxed_slice_into_iter", since = "1.80.0")]

library/core/src/slice/iter/macros.rs

@@ -154,16 +154,26 @@ macro_rules! iterator {
 
             #[inline]
             fn next(&mut self) -> Option<$elem> {
-                // could be implemented with slices, but this avoids bounds checks
+                // intentionally not using the helpers because this is
+                // one of the most mono'd things in the library.
 
-                // SAFETY: The call to `next_unchecked` is
-                // safe since we check if the iterator is empty first.
+                let ptr = self.ptr;
+                let end_or_len = self.end_or_len;
+                // SAFETY: Type invariants.
                 unsafe {
-                    if is_empty!(self) {
-                        None
+                    if T::IS_ZST {
+                        let byte_end = end_or_len as *const u8;
+                        if byte_end == null() {
+                            return None;
+                        }
+                        self.end_or_len = byte_end.wrapping_sub(1) as _;
                     } else {
-                        Some(self.next_unchecked())
+                        if ptr == crate::intrinsics::transmute::<*const T, NonNull<T>>(end_or_len) {
+                            return None;
+                        }
+                        self.ptr = ptr.add(1);
                     }
+                    crate::intrinsics::transmute::<NonNull<T>, Option<$elem>>(ptr)
                 }
             }
 

tests/codegen/slice-iter-nonnull.rs

@@ -14,11 +14,11 @@
 // CHECK-LABEL: @slice_iter_next(
 #[no_mangle]
 pub fn slice_iter_next<'a>(it: &mut std::slice::Iter<'a, u32>) -> Option<&'a u32> {
-    // CHECK: %[[ENDP:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %it, {{i32 4|i64 8}}
-    // CHECK: %[[END:.+]] = load ptr, ptr %[[ENDP]]
+    // CHECK: %[[START:.+]] = load ptr, ptr %it,
     // CHECK-SAME: !nonnull
     // CHECK-SAME: !noundef
-    // CHECK: %[[START:.+]] = load ptr, ptr %it,
+    // CHECK: %[[ENDP:.+]] = getelementptr inbounds{{( nuw)?}} i8, ptr %it, {{i32 4|i64 8}}
+    // CHECK: %[[END:.+]] = load ptr, ptr %[[ENDP]]
     // CHECK-SAME: !nonnull
     // CHECK-SAME: !noundef
     // CHECK: icmp eq ptr %[[START]], %[[END]]

tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir

@@ -4,28 +4,30 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
-    let mut _12: std::iter::Enumerate<std::slice::Iter<'_, T>>;
-    let mut _13: std::iter::Enumerate<std::slice::Iter<'_, T>>;
-    let mut _21: std::option::Option<(usize, &T)>;
-    let mut _24: &impl Fn(usize, &T);
-    let mut _25: (usize, &T);
-    let _26: ();
+    let mut _11: std::ptr::NonNull<T>;
+    let mut _12: *const T;
+    let mut _13: usize;
+    let mut _33: std::option::Option<(usize, &T)>;
+    let mut _36: &impl Fn(usize, &T);
+    let mut _37: (usize, &T);
+    let _38: ();
     scope 1 {
-        debug iter => _13;
-        let _22: usize;
-        let _23: &T;
+        debug (((iter: Enumerate<std::slice::Iter<'_, T>>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull<T>) => _11;
+        debug (((iter: Enumerate<std::slice::Iter<'_, T>>).0: std::slice::Iter<'_, T>).1: *const T) => _12;
+        debug (((iter: Enumerate<std::slice::Iter<'_, T>>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
+        debug ((iter: Enumerate<std::slice::Iter<'_, T>>).1: usize) => _13;
+        let _34: usize;
+        let _35: &T;
         scope 2 {
-            debug i => _22;
-            debug x => _23;
+            debug i => _34;
+            debug x => _35;
         }
         scope 19 (inlined <Enumerate<std::slice::Iter<'_, T>> as Iterator>::next) {
-            let mut _14: &mut std::slice::Iter<'_, T>;
-            let mut _15: std::option::Option<&T>;
-            let mut _19: (usize, bool);
-            let mut _20: (usize, &T);
+            let mut _27: std::option::Option<&T>;
+            let mut _31: (usize, bool);
+            let mut _32: (usize, &T);
             scope 20 {
-                let _18: usize;
+                let _30: usize;
                 scope 25 {
                 }
             }
@@ -40,11 +42,58 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
                 }
             }
             scope 26 (inlined <Option<&T> as Try>::branch) {
-                let mut _16: isize;
-                let _17: &T;
+                let mut _28: isize;
+                let _29: &T;
                 scope 27 {
                 }
             }
+            scope 29 (inlined <std::slice::Iter<'_, T> as Iterator>::next) {
+                let _14: std::ptr::NonNull<T>;
+                let _16: std::ptr::NonNull<T>;
+                let mut _19: bool;
+                let mut _22: std::ptr::NonNull<T>;
+                let mut _24: bool;
+                let mut _25: *const u8;
+                let mut _26: *const T;
+                scope 30 {
+                    let _15: *const T;
+                    scope 31 {
+                        let _23: *const u8;
+                        scope 32 {
+                            scope 33 (inlined null::<u8>) {
+                                scope 34 (inlined without_provenance::<()>) {
+                                    scope 35 (inlined without_provenance_mut::<()>) {
+                                    }
+                                }
+                                scope 36 (inlined std::ptr::from_raw_parts::<u8, ()>) {
+                                }
+                            }
+                            scope 37 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_sub) {
+                                scope 38 (inlined core::num::<impl isize>::wrapping_neg) {
+                                    scope 39 (inlined core::num::<impl isize>::wrapping_sub) {
+                                    }
+                                }
+                                scope 40 (inlined std::ptr::const_ptr::<impl *const u8>::wrapping_offset) {
+                                }
+                            }
+                        }
+                        scope 41 (inlined <NonNull<T> as PartialEq>::eq) {
+                            let mut _17: *mut T;
+                            let mut _18: *mut T;
+                            scope 42 (inlined NonNull::<T>::as_ptr) {
+                            }
+                            scope 43 (inlined NonNull::<T>::as_ptr) {
+                            }
+                        }
+                        scope 44 (inlined NonNull::<T>::add) {
+                            let mut _20: *const T;
+                            let mut _21: *const T;
+                            scope 45 (inlined NonNull::<T>::as_ptr) {
+                            }
+                        }
+                    }
+                }
+            }
         }
     }
     scope 3 (inlined core::slice::<impl [T]>::iter) {
@@ -89,9 +138,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb0: {
-        StorageLive(_11);
         StorageLive(_3);
-        StorageLive(_6);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
@@ -120,86 +167,162 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
         _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
         StorageDead(_9);
         StorageDead(_4);
-        StorageDead(_6);
         StorageDead(_3);
-        _12 = Enumerate::<std::slice::Iter<'_, T>> { iter: copy _11, count: const 0_usize };
-        StorageDead(_11);
+        StorageLive(_11);
+        StorageLive(_12);
         StorageLive(_13);
-        _13 = copy _12;
+        _11 = copy _6;
+        _12 = copy _10;
+        _13 = const 0_usize;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_21);
-        StorageLive(_18);
-        StorageLive(_19);
-        StorageLive(_15);
+        StorageLive(_33);
+        StorageLive(_30);
+        StorageLive(_31);
+        StorageLive(_27);
         StorageLive(_14);
-        _14 = &mut (_13.0: std::slice::Iter<'_, T>);
-        _15 = <std::slice::Iter<'_, T> as Iterator>::next(move _14) -> [return: bb5, unwind unreachable];
+        StorageLive(_15);
+        StorageLive(_23);
+        StorageLive(_16);
+        _14 = copy _11;
+        _15 = copy _12;
+        switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        StorageDead(_14);
-        StorageLive(_16);
-        _16 = discriminant(_15);
-        switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb11];
+        StorageLive(_19);
+        _16 = copy _15 as std::ptr::NonNull<T> (Transmute);
+        StorageLive(_17);
+        _17 = copy _14 as *mut T (Transmute);
+        StorageLive(_18);
+        _18 = copy _16 as *mut T (Transmute);
+        _19 = Eq(move _17, move _18);
+        StorageDead(_18);
+        StorageDead(_17);
+        switchInt(move _19) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_16);
-        StorageDead(_15);
         StorageDead(_19);
-        StorageDead(_18);
+        StorageLive(_22);
+        StorageLive(_21);
+        StorageLive(_20);
+        _20 = copy _14 as *const T (Transmute);
+        _21 = Offset(move _20, const 1_usize);
+        StorageDead(_20);
+        _22 = NonNull::<T> { pointer: move _21 };
         StorageDead(_21);
-        StorageDead(_13);
-        drop(_2) -> [return: bb7, unwind unreachable];
+        _11 = move _22;
+        StorageDead(_22);
+        goto -> bb11;
     }
 
     bb7: {
-        return;
+        StorageDead(_19);
+        StorageDead(_16);
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_28);
+        goto -> bb17;
     }
 
     bb8: {
-        _17 = move ((_15 as Some).0: &T);
-        StorageDead(_16);
-        StorageDead(_15);
-        _18 = copy (_13.1: usize);
-        _19 = AddWithOverflow(copy (_13.1: usize), const 1_usize);
-        assert(!move (_19.1: bool), "attempt to compute `{} + {}`, which would overflow", copy (_13.1: usize), const 1_usize) -> [success: bb9, unwind unreachable];
+        _23 = copy _15 as *const u8 (PtrToPtr);
+        StorageLive(_24);
+        _24 = Eq(copy _23, const {0x0 as *const u8});
+        switchInt(move _24) -> [0: bb9, otherwise: bb16];
     }
 
     bb9: {
-        (_13.1: usize) = move (_19.0: usize);
-        StorageLive(_20);
-        _20 = (copy _18, copy _17);
-        _21 = Option::<(usize, &T)>::Some(move _20);
-        StorageDead(_20);
-        StorageDead(_19);
-        StorageDead(_18);
-        _22 = copy (((_21 as Some).0: (usize, &T)).0: usize);
-        _23 = copy (((_21 as Some).0: (usize, &T)).1: &T);
-        StorageLive(_24);
-        _24 = &_2;
+        StorageDead(_24);
         StorageLive(_25);
-        _25 = (copy _22, copy _23);
-        _26 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _24, move _25) -> [return: bb10, unwind unreachable];
+        _25 = arith_offset::<u8>(move _23, const -1_isize) -> [return: bb10, unwind unreachable];
     }
 
     bb10: {
+        _26 = move _25 as *const T (PtrToPtr);
         StorageDead(_25);
-        StorageDead(_24);
-        StorageDead(_21);
-        goto -> bb4;
+        _12 = copy _26;
+        goto -> bb11;
     }
 
     bb11: {
+        _27 = copy _14 as std::option::Option<&T> (Transmute);
+        StorageDead(_16);
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_28);
+        _28 = discriminant(_27);
+        switchInt(move _28) -> [0: bb17, 1: bb12, otherwise: bb15];
+    }
+
+    bb12: {
+        _29 = move ((_27 as Some).0: &T);
+        StorageDead(_28);
+        StorageDead(_27);
+        _30 = copy _13;
+        _31 = AddWithOverflow(copy _13, const 1_usize);
+        assert(!move (_31.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb13, unwind unreachable];
+    }
+
+    bb13: {
+        _13 = move (_31.0: usize);
+        StorageLive(_32);
+        _32 = (copy _30, copy _29);
+        _33 = Option::<(usize, &T)>::Some(move _32);
+        StorageDead(_32);
+        StorageDead(_31);
+        StorageDead(_30);
+        _34 = copy (((_33 as Some).0: (usize, &T)).0: usize);
+        _35 = copy (((_33 as Some).0: (usize, &T)).1: &T);
+        StorageLive(_36);
+        _36 = &_2;
+        StorageLive(_37);
+        _37 = (copy _34, copy _35);
+        _38 = <impl Fn(usize, &T) as Fn<(usize, &T)>>::call(move _36, move _37) -> [return: bb14, unwind unreachable];
+    }
+
+    bb14: {
+        StorageDead(_37);
+        StorageDead(_36);
+        StorageDead(_33);
+        goto -> bb4;
+    }
+
+    bb15: {
         unreachable;
     }
+
+    bb16: {
+        StorageDead(_24);
+        StorageDead(_16);
+        StorageDead(_23);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_28);
+        goto -> bb17;
+    }
+
+    bb17: {
+        StorageDead(_28);
+        StorageDead(_27);
+        StorageDead(_31);
+        StorageDead(_30);
+        StorageDead(_33);
+        StorageDead(_11);
+        StorageDead(_12);
+        StorageDead(_13);
+        drop(_2) -> [return: bb18, unwind unreachable];
+    }
+
+    bb18: {
+        return;
+    }
 }