Enable -Zshare-generics for inline(never) functions

Cargo.lock

@@ -4493,6 +4493,7 @@ dependencies = [
 name = "rustc_monomorphize"
 version = "0.0.0"
 dependencies = [
+ "rustc_attr",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_fluent_macro",

compiler/rustc_codegen_llvm/src/callee.rs

@@ -113,7 +113,9 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) ->
                 // This is a monomorphization. Its expected visibility depends
                 // on whether we are in share-generics mode.
 
-                if cx.tcx.sess.opts.share_generics() {
+                if cx.tcx.sess.opts.share_generics()
+                    || tcx.codegen_fn_attrs(instance_def_id).inline == rustc_attr::InlineAttr::Never
+                {
                     // We are in share_generics mode.
 
                     if let Some(instance_def_id) = instance_def_id.as_local() {

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -308,7 +308,7 @@ fn exported_symbols_provider_local(
         ));
     }
 
-    if tcx.sess.opts.share_generics() && tcx.local_crate_exports_generics() {
+    if tcx.local_crate_exports_generics() {
         use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility};
         use rustc_middle::ty::InstanceDef;
 
@@ -336,6 +336,16 @@ fn exported_symbols_provider_local(
                 continue;
             }
 
+            if !tcx.sess.opts.share_generics() {
+                if tcx.codegen_fn_attrs(mono_item.def_id()).inline == rustc_attr::InlineAttr::Never
+                {
+                    // this is OK, we explicitly allow sharing inline(never) across crates even
+                    // without share-generics.
+                } else {
+                    continue;
+                }
+            }
+
             match *mono_item {
                 MonoItem::Fn(Instance { def: InstanceDef::Item(def), args }) => {
                     if args.non_erasable_generics(tcx, def).next().is_some() {

compiler/rustc_middle/src/mir/mono.rs

@@ -125,6 +125,13 @@ impl<'tcx> MonoItem<'tcx> {
                     return InstantiationMode::GloballyShared { may_conflict: false };
                 }
 
+                if let InlineAttr::Never = tcx.codegen_fn_attrs(instance.def_id()).inline
+                    && self.is_generic_fn(tcx)
+                {
+                    // Upgrade inline(never) to a globally shared instance.
+                    return InstantiationMode::GloballyShared { may_conflict: true };
+                }
+
                 // At this point we don't have explicit linkage and we're an
                 // inlined function. If we're inlining into all CGUs then we'll
                 // be creating a local copy per CGU.

compiler/rustc_middle/src/ty/context.rs

@@ -1503,8 +1503,6 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn local_crate_exports_generics(self) -> bool {
-        debug_assert!(self.sess.opts.share_generics());
-
         self.crate_types().iter().any(|crate_type| {
             match crate_type {
                 CrateType::Executable

compiler/rustc_middle/src/ty/instance.rs

@@ -194,19 +194,23 @@ impl<'tcx> Instance<'tcx> {
     /// This method already takes into account the global `-Zshare-generics`
     /// setting, always returning `None` if `share-generics` is off.
     pub fn upstream_monomorphization(&self, tcx: TyCtxt<'tcx>) -> Option<CrateNum> {
-        // If we are not in share generics mode, we don't link to upstream
-        // monomorphizations but always instantiate our own internal versions
-        // instead.
-        if !tcx.sess.opts.share_generics() {
-            return None;
-        }
-
         // If this is an item that is defined in the local crate, no upstream
         // crate can know about it/provide a monomorphization.
         if self.def_id().is_local() {
             return None;
         }
 
+        // If we are not in share generics mode, we don't link to upstream
+        // monomorphizations but always instantiate our own internal versions
+        // instead.
+        if !tcx.sess.opts.share_generics()
+            // However, if the def_id is marked inline(never), then it's fine to just reuse the
+            // upstream monomorphization.
+            && tcx.codegen_fn_attrs(self.def_id()).inline != rustc_attr::InlineAttr::Never
+        {
+            return None;
+        }
+
         // If this a non-generic instance, it cannot be a shared monomorphization.
         self.args.non_erasable_generics(tcx, self.def_id()).next()?;
 

compiler/rustc_monomorphize/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
+rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }

compiler/rustc_monomorphize/src/partitioning.rs

@@ -206,8 +206,8 @@ where
     // available to downstream crates. This depends on whether we are in
     // share-generics mode and whether the current crate can even have
     // downstream crates.
-    let export_generics =
-        cx.tcx.sess.opts.share_generics() && cx.tcx.local_crate_exports_generics();
+    let can_export_generics = cx.tcx.local_crate_exports_generics();
+    let always_export_generics = can_export_generics && cx.tcx.sess.opts.share_generics();
 
     let cgu_name_builder = &mut CodegenUnitNameBuilder::new(cx.tcx);
     let cgu_name_cache = &mut UnordMap::default();
@@ -247,7 +247,8 @@ where
             cx.tcx,
             &mono_item,
             &mut can_be_internalized,
-            export_generics,
+            can_export_generics,
+            always_export_generics,
         );
         if visibility == Visibility::Hidden && can_be_internalized {
             internalization_candidates.insert(mono_item);
@@ -734,12 +735,19 @@ fn mono_item_linkage_and_visibility<'tcx>(
     tcx: TyCtxt<'tcx>,
     mono_item: &MonoItem<'tcx>,
     can_be_internalized: &mut bool,
-    export_generics: bool,
+    can_export_generics: bool,
+    always_export_generics: bool,
 ) -> (Linkage, Visibility) {
     if let Some(explicit_linkage) = mono_item.explicit_linkage(tcx) {
         return (explicit_linkage, Visibility::Default);
     }
-    let vis = mono_item_visibility(tcx, mono_item, can_be_internalized, export_generics);
+    let vis = mono_item_visibility(
+        tcx,
+        mono_item,
+        can_be_internalized,
+        can_export_generics,
+        always_export_generics,
+    );
     (Linkage::External, vis)
 }
 
@@ -762,7 +770,8 @@ fn mono_item_visibility<'tcx>(
     tcx: TyCtxt<'tcx>,
     mono_item: &MonoItem<'tcx>,
     can_be_internalized: &mut bool,
-    export_generics: bool,
+    can_export_generics: bool,
+    always_export_generics: bool,
 ) -> Visibility {
     let instance = match mono_item {
         // This is pretty complicated; see below.
@@ -822,7 +831,11 @@ fn mono_item_visibility<'tcx>(
 
     // Upstream `DefId` instances get different handling than local ones.
     let Some(def_id) = def_id.as_local() else {
-        return if export_generics && is_generic {
+        return if is_generic
+            && (always_export_generics
+                || (can_export_generics
+                    && tcx.codegen_fn_attrs(def_id).inline == rustc_attr::InlineAttr::Never))
+        {
             // If it is an upstream monomorphization and we export generics, we must make
             // it available to downstream crates.
             *can_be_internalized = false;
@@ -833,7 +846,12 @@ fn mono_item_visibility<'tcx>(
     };
 
     if is_generic {
-        if export_generics {
+        // An inline(never) function won't get inlined in upstream crates anyway, so we might as
+        // well export it.
+        if always_export_generics
+            || (can_export_generics
+                && tcx.codegen_fn_attrs(def_id).inline == rustc_attr::InlineAttr::Never)
+        {
             if tcx.is_unreachable_local_definition(def_id) {
                 // This instance cannot be used from another crate.
                 Visibility::Hidden

library/alloc/src/raw_vec.rs

@@ -548,7 +548,6 @@ impl<T, A: Allocator> RawVec<T, A> {
 // above `RawVec::grow_amortized` for details. (The `A` parameter isn't
 // significant, because the number of different `A` types seen in practice is
 // much smaller than the number of `T` types.)
-#[inline(never)]
 fn finish_grow<A>(
     new_layout: Result<Layout, LayoutError>,
     current_memory: Option<(NonNull<u8>, Layout)>,

library/alloc/src/vec/mod.rs

@@ -1036,7 +1036,13 @@ impl<T, A: Allocator> Vec<T, A> {
     /// ```
     #[stable(feature = "try_reserve", since = "1.57.0")]
     pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> {
-        self.buf.try_reserve(self.len, additional)
+        let len = self.len;
+        let ret = self.buf.try_reserve(len, additional);
+        unsafe {
+            // Inform the optimier that growing did not change the length.
+            core::hint::assert_unchecked(self.len == len);
+        }
+        ret
     }
 
     /// Tries to reserve the minimum capacity for at least `additional`

library/std/src/panicking.rs

@@ -733,6 +733,22 @@ pub const fn begin_panic<M: Any + Send>(msg: M) -> ! {
     }
 }
 
+// This forces codegen of the function called by panic!() inside the std crate, rather than in
+// downstream crates. Primarily this is useful for rustc's codegen tests, which rely on noticing
+// complete removal of panic from generated IR. Since begin_panic is inline(never), it's only
+// codegen'd once per crate-graph so this pushes that to std rather than our codegen test crates.
+//
+// (See https://github.com/rust-lang/rust/pull/123244 for more info on why).
+//
+// If this is causing problems we can also modify those codegen tests to use a crate type like
+// cdylib which doesn't export "Rust" symbols to downstream linkage units.
+#[unstable(feature = "libstd_sys_internals", reason = "used by the panic! macro", issue = "none")]
+#[doc(hidden)]
+#[allow(dead_code)]
+#[used]
+pub static EMPTY_PANIC: fn(&'static str) -> ! =
+    begin_panic::<&'static str> as fn(&'static str) -> !;
+
 /// Central point for dispatching panics.
 ///
 /// Executes the primary logic for a panic, including checking for recursive

tests/codegen-units/partitioning/auxiliary/cgu_generic_function.rs

@@ -11,10 +11,21 @@ pub fn foo<T>(x: T) -> (T, u32, i8) {
 #[inline(never)]
 fn bar<T>(x: T) -> (T, Struct) {
     let _ = not_exported_and_not_generic(0);
+    exported_and_generic::<u32>(0);
     (x, Struct(1))
 }
 
+pub static F: fn(u32) -> u32 = exported_and_generic::<u32>;
+
 // These should not contribute to the codegen items of other crates.
+
+// This is generic, but it's only instantiated with a u32 argument and that instantiation is present
+// in the local crate (see F above).
+#[inline(never)]
+pub fn exported_and_generic<T>(x: T) -> T {
+    x
+}
+
 #[inline(never)]
 pub fn exported_but_not_generic(x: i32) -> i64 {
     x as i64

tests/codegen/avr/avr-func-addrspace.rs

@@ -85,7 +85,7 @@ pub extern "C" fn test() {
 
     // A call through the Fn trait must use address space 1.
     //
-    // CHECK: call{{.+}}addrspace(1) void @call_through_fn_trait()
+    // CHECK: call{{.+}}addrspace(1) void @call_through_fn_trait(ptr {{.*}} poison)
     call_through_fn_trait(&mut update_bar_value);
 
     // A call through a global variable must use address space 1.

tests/codegen/issues/issue-77812.rs

@@ -15,6 +15,7 @@ extern "C" {
     fn exf2();
 }
 
+#[no_mangle]
 pub static mut GLOBAL: Variant = Variant::Zero;
 
 // CHECK-LABEL: @issue_77812

tests/ui/panics/issue-47429-short-backtraces.legacy.run.stderr

@@ -1,6 +1,6 @@
 thread 'main' panicked at $DIR/issue-47429-short-backtraces.rs:23:5:
 explicit panic
 stack backtrace:
-   0: std::panicking::begin_panic
+   0: std::panicking::begin_panic::<&str>
    1: issue_47429_short_backtraces::main
 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

tests/ui/panics/runtime-switch.legacy.run.stderr

@@ -1,6 +1,6 @@
 thread 'main' panicked at $DIR/runtime-switch.rs:26:5:
 explicit panic
 stack backtrace:
-   0: std::panicking::begin_panic
+   0: std::panicking::begin_panic::<&str>
    1: runtime_switch::main
 note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

tests/ui/panics/short-ice-remove-middle-frames-2.run.stderr

@@ -1,7 +1,7 @@
 thread 'main' panicked at $DIR/short-ice-remove-middle-frames-2.rs:56:5:
 debug!!!
 stack backtrace:
-   0: std::panicking::begin_panic
+   0: std::panicking::begin_panic::<&str>
    1: short_ice_remove_middle_frames_2::eight
    2: short_ice_remove_middle_frames_2::seven::{{closure}}
       [... omitted 3 frames ...]

tests/ui/panics/short-ice-remove-middle-frames.run.stderr

@@ -1,7 +1,7 @@
 thread 'main' panicked at $DIR/short-ice-remove-middle-frames.rs:52:5:
 debug!!!
 stack backtrace:
-   0: std::panicking::begin_panic
+   0: std::panicking::begin_panic::<&str>
    1: short_ice_remove_middle_frames::seven
    2: short_ice_remove_middle_frames::sixth
    3: short_ice_remove_middle_frames::fifth::{{closure}}