Auto merge of rust-lang#112910 - lqd:mcp510, r=petrochenkov

Implement most of MCP510

This implements most of what remains to be done for MCP510:
- turns `-C link-self-contained` into a `+`/`-` list of components, like `-C link-self-contained=+linker,+crto,+libc,+unwind,+sanitizers,+mingw`. The scaffolding is present for all these expected components to be implemented and stabilized in the future on their own time. This PR only handles the `-Zgcc-ld=lld` subset of these link-self-contained components as  `-Clink-self-contained=+linker`
- handles  `-C link-self-contained=y|n`  as-is today, for compatibility with `rustc_codegen_ssa::back::link::self_contained`'s [explicit opt-in and opt-out](https://github.com/lqd/rust/blob/9eee230cd0a56bfba3ce65121798d9f9f4341cdd/compiler/rustc_codegen_ssa/src/back/link.rs#L1671-L1676).
- therefore supports our plan to opt out of `rust-lld` (when it's enabled by default) even for current `-Clink-self-contained` users, with e.g. `-Clink-self-contained -Clink-self-contained=-linker`
- turns `add_gcc_ld_path` into its expected final form, by using the `-C link-self-contained=+linker`  CLI flag, and whether the `LinkerFlavor`  has the expected `Cc::Yes` and `Lld::Yes` shape (this is not yet the case in practice for any CLI linker flavor)
- makes the [new clean linker flavors](rust-lang#96827 (comment)) selectable in the CLI in addition to the legacy ones, in order to opt-in to using `cc` and `lld` to emulate `-Zgcc-ld=lld`
- ensure the new `-C link-self-contained` components, and `-C linker-flavor`s are unstable, and require `-Z unstable-options` to be used

The up-to-date set of flags for the future stable CLI version of `-Zgcc-ld=lld` is currently: `-Clink-self-contained=+linker -Clinker-flavor=gnu-lld-cc -Zunstable-options`.

It's possible we'll also need to do something for distros that don't ship `rust-lld`, but maybe there are already no tool search paths to be added to `cc` in this situation anyways.

r? `@petrochenkov`

Cargo.lock

@@ -3995,6 +3995,7 @@ name = "rustc_session"
 version = "0.0.0"
 dependencies = [
  "atty",
+ "bitflags",
  "getopts",
  "libc",
  "rustc_ast",

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -12,7 +12,7 @@ use rustc_metadata::fs::{copy_to_stdout, emit_wrapper_file, METADATA_FILENAME};
 use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
-use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip};
+use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, Strip};
 use rustc_session::config::{OutputFilenames, OutputType, PrintRequest, SplitDwarfKind};
 use rustc_session::cstore::DllImport;
 use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename};
@@ -1688,7 +1688,7 @@ fn detect_self_contained_mingw(sess: &Session) -> bool {
 /// instead of being found somewhere on the host system.
 /// We only provide such support for a very limited number of targets.
 fn self_contained(sess: &Session, crate_type: CrateType) -> bool {
-    if let Some(self_contained) = sess.opts.cg.link_self_contained {
+    if let Some(self_contained) = sess.opts.cg.link_self_contained.explicitly_set {
         if sess.target.link_self_contained == LinkSelfContainedDefault::False {
             sess.emit_err(errors::UnsupportedLinkSelfContained);
         }
@@ -2246,7 +2246,8 @@ fn add_order_independent_options(
     out_filename: &Path,
     tmpdir: &Path,
 ) {
-    add_gcc_ld_path(cmd, sess, flavor);
+    // Take care of the flavors and CLI options requesting the `lld` linker.
+    add_lld_args(cmd, sess, flavor);
 
     add_apple_sdk(cmd, sess, flavor);
 
@@ -2948,55 +2949,66 @@ fn get_apple_sdk_root(sdk_name: &str) -> Result<String, errors::AppleSdkRootErro
     }
 }
 
-fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
-    if let Some(ld_impl) = sess.opts.unstable_opts.gcc_ld {
-        if let LinkerFlavor::Gnu(Cc::Yes, _)
-        | LinkerFlavor::Darwin(Cc::Yes, _)
-        | LinkerFlavor::WasmLld(Cc::Yes) = flavor
-        {
-            match ld_impl {
-                LdImpl::Lld => {
-                    // Implement the "self-contained" part of -Zgcc-ld
-                    // by adding rustc distribution directories to the tool search path.
-                    for path in sess.get_tools_search_paths(false) {
-                        cmd.arg({
-                            let mut arg = OsString::from("-B");
-                            arg.push(path.join("gcc-ld"));
-                            arg
-                        });
-                    }
-                    // Implement the "linker flavor" part of -Zgcc-ld
-                    // by asking cc to use some kind of lld.
-                    cmd.arg("-fuse-ld=lld");
-
-                    if !flavor.is_gnu() {
-                        // Tell clang to use a non-default LLD flavor.
-                        // Gcc doesn't understand the target option, but we currently assume
-                        // that gcc is not used for Apple and Wasm targets (#97402).
-                        //
-                        // Note that we don't want to do that by default on macOS: e.g. passing a
-                        // 10.7 target to LLVM works, but not to recent versions of clang/macOS, as
-                        // shown in issue #101653 and the discussion in PR #101792.
-                        //
-                        // It could be required in some cases of cross-compiling with
-                        // `-Zgcc-ld=lld`, but this is generally unspecified, and we don't know
-                        // which specific versions of clang, macOS SDK, host and target OS
-                        // combinations impact us here.
-                        //
-                        // So we do a simple first-approximation until we know more of what the
-                        // Apple targets require (and which would be handled prior to hitting this
-                        // `-Zgcc-ld=lld` codepath anyway), but the expectation is that until then
-                        // this should be manually passed if needed. We specify the target when
-                        // targeting a different linker flavor on macOS, and that's also always
-                        // the case when targeting WASM.
-                        if sess.target.linker_flavor != sess.host.linker_flavor {
-                            cmd.arg(format!("--target={}", sess.target.llvm_target));
-                        }
-                    }
-                }
-            }
-        } else {
-            sess.emit_fatal(errors::OptionGccOnly);
+/// When using the linker flavors opting in to `lld`, or the unstable `-Zgcc-ld=lld` flag, add the
+/// necessary paths and arguments to invoke it:
+/// - when the self-contained linker flag is active: the build of `lld` distributed with rustc,
+/// - or any `lld` available to `cc`.
+fn add_lld_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
+    let unstable_use_lld = sess.opts.unstable_opts.gcc_ld.is_some();
+    debug!("add_lld_args requested, flavor: '{flavor:?}', `-Zgcc-ld=lld`: {unstable_use_lld}");
+
+    // Sanity check: using the old unstable `-Zgcc-ld=lld` option requires a `cc`-using flavor.
+    let flavor_uses_cc = flavor.uses_cc();
+    if unstable_use_lld && !flavor_uses_cc {
+        sess.emit_fatal(errors::OptionGccOnly);
+    }
+
+    // If the flavor doesn't use a C/C++ compiler to invoke the linker, or doesn't opt in to `lld`,
+    // we don't need to do anything.
+    let use_lld = flavor.uses_lld() || unstable_use_lld;
+    if !flavor_uses_cc || !use_lld {
+        return;
+    }
+
+    // 1. Implement the "self-contained" part of this feature by adding rustc distribution
+    //    directories to the tool's search path.
+    let self_contained_linker = sess.opts.cg.link_self_contained.linker() || unstable_use_lld;
+    if self_contained_linker {
+        for path in sess.get_tools_search_paths(false) {
+            cmd.arg({
+                let mut arg = OsString::from("-B");
+                arg.push(path.join("gcc-ld"));
+                arg
+            });
+        }
+    }
+
+    // 2. Implement the "linker flavor" part of this feature by asking `cc` to use some kind of
+    //    `lld` as the linker.
+    cmd.arg("-fuse-ld=lld");
+
+    if !flavor.is_gnu() {
+        // Tell clang to use a non-default LLD flavor.
+        // Gcc doesn't understand the target option, but we currently assume
+        // that gcc is not used for Apple and Wasm targets (#97402).
+        //
+        // Note that we don't want to do that by default on macOS: e.g. passing a
+        // 10.7 target to LLVM works, but not to recent versions of clang/macOS, as
+        // shown in issue #101653 and the discussion in PR #101792.
+        //
+        // It could be required in some cases of cross-compiling with
+        // `-Zgcc-ld=lld`, but this is generally unspecified, and we don't know
+        // which specific versions of clang, macOS SDK, host and target OS
+        // combinations impact us here.
+        //
+        // So we do a simple first-approximation until we know more of what the
+        // Apple targets require (and which would be handled prior to hitting this
+        // `-Zgcc-ld=lld` codepath anyway), but the expectation is that until then
+        // this should be manually passed if needed. We specify the target when
+        // targeting a different linker flavor on macOS, and that's also always
+        // the case when targeting WASM.
+        if sess.target.linker_flavor != sess.host.linker_flavor {
+            cmd.arg(format!("--target={}", sess.target.llvm_target));
         }
     }
 }

compiler/rustc_interface/src/tests.rs

@@ -8,6 +8,7 @@ use rustc_session::config::rustc_optgroups;
 use rustc_session::config::DebugInfo;
 use rustc_session::config::Input;
 use rustc_session::config::InstrumentXRay;
+use rustc_session::config::LinkSelfContained;
 use rustc_session::config::TraitSolver;
 use rustc_session::config::{build_configuration, build_session_options, to_crate_config};
 use rustc_session::config::{
@@ -579,7 +580,7 @@ fn test_codegen_options_tracking_hash() {
     untracked!(incremental, Some(String::from("abc")));
     // `link_arg` is omitted because it just forwards to `link_args`.
     untracked!(link_args, vec![String::from("abc"), String::from("def")]);
-    untracked!(link_self_contained, Some(true));
+    untracked!(link_self_contained, LinkSelfContained::on());
     untracked!(linker, Some(PathBuf::from("linker")));
     untracked!(linker_flavor, Some(LinkerFlavorCli::Gcc));
     untracked!(no_stack_check, true);

compiler/rustc_session/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 atty = "0.2.13"
+bitflags = "1.2.1"
 getopts = "0.2"
 rustc_macros = { path = "../rustc_macros" }
 tracing = "0.1"

compiler/rustc_session/src/config.rs

@@ -9,10 +9,9 @@ use crate::{lint, HashStableContext};
 use crate::{EarlyErrorHandler, Session};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-
 use rustc_data_structures::stable_hasher::{StableOrd, ToStableHashKey};
 use rustc_target::abi::Align;
-use rustc_target::spec::{LinkerFlavorCli, PanicStrategy, SanitizerSet, SplitDebuginfo};
+use rustc_target::spec::{PanicStrategy, SanitizerSet, SplitDebuginfo};
 use rustc_target::spec::{Target, TargetTriple, TargetWarnings, TARGETS};
 
 use crate::parse::{CrateCheckConfig, CrateConfig};
@@ -201,6 +200,128 @@ pub enum LinkerPluginLto {
     Disabled,
 }
 
+impl LinkerPluginLto {
+    pub fn enabled(&self) -> bool {
+        match *self {
+            LinkerPluginLto::LinkerPlugin(_) | LinkerPluginLto::LinkerPluginAuto => true,
+            LinkerPluginLto::Disabled => false,
+        }
+    }
+}
+
+/// The different values `-C link-self-contained` can take: a list of individually enabled or
+/// disabled components used during linking, coming from the rustc distribution, instead of being
+/// found somewhere on the host system.
+///
+/// They can be set in bulk via `-C link-self-contained=yes|y|on` or `-C
+/// link-self-contained=no|n|off`, and those boolean values are the historical defaults.
+///
+/// But each component is fine-grained, and can be unstably targeted, to use:
+/// - some CRT objects
+/// - the libc static library
+/// - libgcc/libunwind libraries
+/// - a linker we distribute
+/// - some sanitizer runtime libraries
+/// - all other MinGW libraries and Windows import libs
+///
+#[derive(Default, Clone, PartialEq, Debug)]
+pub struct LinkSelfContained {
+    /// Whether the user explicitly set `-C link-self-contained` on or off, the historical values.
+    /// Used for compatibility with the existing opt-in and target inference.
+    pub explicitly_set: Option<bool>,
+
+    /// The components that are enabled.
+    components: LinkSelfContainedComponents,
+}
+
+bitflags::bitflags! {
+    #[derive(Default)]
+    /// The `-C link-self-contained` components that can individually be enabled or disabled.
+    pub struct LinkSelfContainedComponents: u8 {
+        /// CRT objects (e.g. on `windows-gnu`, `musl`, `wasi` targets)
+        const CRT_OBJECTS = 1 << 0;
+        /// libc static library (e.g. on `musl`, `wasi` targets)
+        const LIBC        = 1 << 1;
+        /// libgcc/libunwind (e.g. on `windows-gnu`, `fuchsia`, `fortanix`, `gnullvm` targets)
+        const UNWIND      = 1 << 2;
+        /// Linker, dlltool, and their necessary libraries (e.g. on `windows-gnu` and for `rust-lld`)
+        const LINKER      = 1 << 3;
+        /// Sanitizer runtime libraries
+        const SANITIZERS  = 1 << 4;
+        /// Other MinGW libs and Windows import libs
+        const MINGW       = 1 << 5;
+    }
+}
+
+impl FromStr for LinkSelfContainedComponents {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "crto" => LinkSelfContainedComponents::CRT_OBJECTS,
+            "libc" => LinkSelfContainedComponents::LIBC,
+            "unwind" => LinkSelfContainedComponents::UNWIND,
+            "linker" => LinkSelfContainedComponents::LINKER,
+            "sanitizers" => LinkSelfContainedComponents::SANITIZERS,
+            "mingw" => LinkSelfContainedComponents::MINGW,
+            _ => return Err(()),
+        })
+    }
+}
+
+impl LinkSelfContained {
+    /// Incorporates an enabled or disabled component as specified on the CLI, if possible.
+    /// For example: `+linker`, and `-crto`.
+    pub(crate) fn handle_cli_component(&mut self, component: &str) -> Result<(), ()> {
+        // Note that for example `-Cself-contained=y -Cself-contained=-linker` is not an explicit
+        // set of all values like `y` or `n` used to be. Therefore, if this flag had previously been
+        // set in bulk with its historical values, then manually setting a component clears that
+        // `explicitly_set` state.
+        if let Some(component_to_enable) = component.strip_prefix("+") {
+            self.explicitly_set = None;
+            self.components.insert(component_to_enable.parse()?);
+            Ok(())
+        } else if let Some(component_to_disable) = component.strip_prefix("-") {
+            self.explicitly_set = None;
+            self.components.remove(component_to_disable.parse()?);
+            Ok(())
+        } else {
+            Err(())
+        }
+    }
+
+    /// Turns all components on or off and records that this was done explicitly for compatibility
+    /// purposes.
+    pub(crate) fn set_all_explicitly(&mut self, enabled: bool) {
+        self.explicitly_set = Some(enabled);
+        self.components = if enabled {
+            LinkSelfContainedComponents::all()
+        } else {
+            LinkSelfContainedComponents::empty()
+        };
+    }
+
+    /// Helper creating a fully enabled `LinkSelfContained` instance. Used in tests.
+    pub fn on() -> Self {
+        let mut on = LinkSelfContained::default();
+        on.set_all_explicitly(true);
+        on
+    }
+
+    /// To help checking CLI usage while some of the values are unstable: returns whether one of the
+    /// components was set individually. This would also require the `-Zunstable-options` flag, to
+    /// be allowed.
+    fn are_unstable_variants_set(&self) -> bool {
+        let any_component_set = !self.components.is_empty();
+        self.explicitly_set.is_none() && any_component_set
+    }
+
+    /// Returns whether the self-contained linker component is enabled.
+    pub fn linker(&self) -> bool {
+        self.components.contains(LinkSelfContainedComponents::LINKER)
+    }
+}
+
 /// Used with `-Z assert-incr-state`.
 #[derive(Clone, Copy, PartialEq, Hash, Debug)]
 pub enum IncrementalStateAssertion {
@@ -213,15 +334,6 @@ pub enum IncrementalStateAssertion {
     NotLoaded,
 }
 
-impl LinkerPluginLto {
-    pub fn enabled(&self) -> bool {
-        match *self {
-            LinkerPluginLto::LinkerPlugin(_) | LinkerPluginLto::LinkerPluginAuto => true,
-            LinkerPluginLto::Disabled => false,
-        }
-    }
-}
-
 /// The different settings that can be enabled via the `-Z location-detail` flag.
 #[derive(Copy, Clone, PartialEq, Hash, Debug)]
 pub struct LocationDetail {
@@ -2544,16 +2656,28 @@ pub fn build_session_options(
         }
     }
 
-    if let Some(flavor) = cg.linker_flavor {
-        if matches!(flavor, LinkerFlavorCli::BpfLinker | LinkerFlavorCli::PtxLinker)
-            && !nightly_options::is_unstable_enabled(matches)
-        {
-            let msg = format!(
-                "linker flavor `{}` is unstable, `-Z unstable-options` \
-                 flag must also be passed to explicitly use it",
-                flavor.desc()
+    // For testing purposes, until we have more feedback about these options: ensure `-Z
+    // unstable-options` is required when using the unstable `-C link-self-contained` options, like
+    // `-C link-self-contained=+linker`, and when using the unstable `-C linker-flavor` options, like
+    // `-C linker-flavor=gnu-lld-cc`.
+    if !nightly_options::is_unstable_enabled(matches) {
+        let uses_unstable_self_contained_option =
+            cg.link_self_contained.are_unstable_variants_set();
+        if uses_unstable_self_contained_option {
+            handler.early_error(
+                "only `-C link-self-contained` values `y`/`yes`/`on`/`n`/`no`/`off` are stable, \
+                the `-Z unstable-options` flag must also be passed to use the unstable values",
             );
-            handler.early_error(msg);
+        }
+
+        if let Some(flavor) = cg.linker_flavor {
+            if flavor.is_unstable() {
+                handler.early_error(format!(
+                    "the linker flavor `{}` is unstable, the `-Z unstable-options` \
+                        flag must also be passed to use the unstable values",
+                    flavor.desc()
+                ));
+            }
         }
     }