Always compile rustdoc with debug logging enabled with download-ci-llvm

Previously, logging at DEBUG or below would always be silenced, because
rustc compiles tracing with the `static_max_level_info` feature. That
makes sense for release artifacts, but not for developing rustdoc.

Instead, this compiles two different versions of tracing: one in the
release artifacts, distributed in the sysroot, and a new version
compiled by rustdoc. Since `rustc_driver` is always linked to the
version of sysroot, this copy/pastes `init_env_logging` into rustdoc.

To avoid compiling an unnecessary version of tracing when
`download-stage1` isn't set, this adds a new `using-ci-artifacts`
feature for rustdoc and passes that feature in bootstrap.

Author: jyn514

Cargo.lock

@@ -4361,6 +4361,9 @@ dependencies = [
  "serde_json",
  "smallvec 1.4.2",
  "tempfile",
+ "tracing",
+ "tracing-subscriber",
+ "tracing-tree",
 ]
 
 [[package]]

src/bootstrap/tool.rs

@@ -493,6 +493,16 @@ impl Step for Rustdoc {
         // rustc compiler it's paired with, so it must be built with the previous stage compiler.
         let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
 
+        // Rustdoc uses its own version of `tracing` when downloading CI
+        // artifacts so that debug logging is available.
+        let using_ci_artifacts;
+        let extra_features: &[_] = if builder.build.config.download_stage1 {
+            using_ci_artifacts = ["using-ci-artifacts".into()];
+            &using_ci_artifacts
+        } else {
+            &[]
+        };
+
         // The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
         // compiler libraries, ...) are built. Rustdoc does not require the presence of any
         // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since
@@ -508,7 +518,7 @@ impl Step for Rustdoc {
             "build",
             "src/tools/rustdoc",
             SourceType::InTree,
-            &[],
+            extra_features,
         );
 
         builder.info(&format!(

src/librustdoc/Cargo.toml

@@ -17,6 +17,17 @@ smallvec = "1.0"
 tempfile = "3"
 itertools = "0.9"
 regex = "1"
+tracing = { version = "0.1", optional = true }
+tracing-tree = { version = "0.1.6", optional = true }
+
+[dependencies.tracing-subscriber]
+version = "0.2.13"
+default-features = false
+features = ["fmt", "env-filter", "smallvec", "parking_lot", "ansi"]
+optional = true
+
+[features]
+using-ci-artifacts = ["tracing", "tracing-tree", "tracing-subscriber"]
 
 [dev-dependencies]
 expect-test = "1.0"

src/librustdoc/lib.rs

@@ -95,14 +95,90 @@ mod visit_lib;
 pub fn main() {
     rustc_driver::set_sigpipe_handler();
     rustc_driver::install_ice_hook();
+
+    // When using CI artifacts (with `download_stage1 = true`),
+    // tracing is built with `--features=static_max_level_info`, which disables
+    // almost all rustdoc logging. To avoid this, compile our own version of
+    // `tracing` that logs all levels.
+    #[cfg(feature = "using-ci-artifacts")]
+    init_logging();
+    // When using CI artifacts, rustc logging is always disabled. To avoid spurious
+    // warnings, don't enable both rustdoc and rustc's version of `tracing`.
+    #[cfg(not(feature = "using-ci-artifacts"))]
     rustc_driver::init_env_logger("RUSTDOC_LOG");
+
     let exit_code = rustc_driver::catch_with_exit_code(|| match get_args() {
         Some(args) => main_args(&args),
         _ => Err(ErrorReported),
     });
     process::exit(exit_code);
 }
 
+#[cfg(feature = "using-ci-artifacts")]
+fn init_logging() {
+    extern crate libc;
+    #[cfg(windows)]
+    extern crate winapi;
+
+    use std::io;
+
+    // FIXME remove these and use winapi 0.3 instead
+    // Duplicates: bootstrap/compile.rs, librustc_errors/emitter.rs, rustc_driver/lib.rs
+    #[cfg(unix)]
+    fn stdout_isatty() -> bool {
+        unsafe { libc::isatty(libc::STDOUT_FILENO) != 0 }
+    }
+
+    #[cfg(windows)]
+    fn stdout_isatty() -> bool {
+        use winapi::um::consoleapi::GetConsoleMode;
+        use winapi::um::processenv::GetStdHandle;
+        use winapi::um::winbase::STD_OUTPUT_HANDLE;
+
+        unsafe {
+            let handle = GetStdHandle(STD_OUTPUT_HANDLE);
+            let mut out = 0;
+            GetConsoleMode(handle, &mut out) != 0
+        }
+    }
+
+    let color_logs = match std::env::var("RUSTDOC_LOG_COLOR") {
+        Ok(value) => match value.as_ref() {
+            "always" => true,
+            "never" => false,
+            "auto" => stdout_isatty(),
+            _ => early_error(
+                ErrorOutputType::default(),
+                &format!(
+                    "invalid log color value '{}': expected one of always, never, or auto",
+                    value
+                ),
+            ),
+        },
+        Err(std::env::VarError::NotPresent) => stdout_isatty(),
+        Err(std::env::VarError::NotUnicode(_value)) => early_error(
+            ErrorOutputType::default(),
+            "non-Unicode log color value: expected one of always, never, or auto",
+        ),
+    };
+    let filter = tracing_subscriber::EnvFilter::from_env("RUSTDOC_LOG");
+    let layer = tracing_tree::HierarchicalLayer::default()
+        .with_writer(io::stderr)
+        .with_indent_lines(true)
+        .with_ansi(color_logs)
+        .with_targets(true)
+        .with_wraparound(10)
+        .with_verbose_exit(true)
+        .with_verbose_entry(true)
+        .with_indent_amount(2);
+    #[cfg(parallel_compiler)]
+    let layer = layer.with_thread_ids(true).with_thread_names(true);
+
+    use tracing_subscriber::layer::SubscriberExt;
+    let subscriber = tracing_subscriber::Registry::default().with(filter).with(layer);
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+}
+
 fn get_args() -> Option<Vec<String>> {
     env::args_os()
         .enumerate()

src/tools/rustdoc/Cargo.toml

@@ -13,3 +13,6 @@ path = "main.rs"
 
 [dependencies]
 rustdoc = { path = "../../librustdoc" }
+
+[features]
+using-ci-artifacts = ["rustdoc/using-ci-artifacts"]