Auto merge of #43482 - Mark-Simulacrum:single-rustdoc, r=alexcrichton

Compile rustdoc on-demand

Fixes #43284, fixes #38318, and fixes #39505.

Doesn't directly help with #42686, since we need to rebuild just as much. In fact, this hurts it, since `./x.py doc --stage 0` will now fail. I'm not sure if it did before, but with these changes it runs into the problem where we attempt to use artifacts from bootstrap rustc with a non-bootstrap rustdoc, running into version conflicts. I believe this is solvable, but leaving for a future PR.

This means that rustdoc will no longer be compiled when compiling rustc, by default. However, it is still built from `./x.py build` (for hosts, but not targets, since we don't produce compiler toolchains for them) and will be built for doc tests and crate tests.

After this, the recommended workflow if you want a rustdoc is: `./x.py build --stage 1 src/tools/rustdoc` which will give you a working rustdoc in `build/triple/stage1/bin/rustdoc`. Note that you can add `src/libstd` onto the command to compile libstd as well so that the rustdoc can easily compile crates in the wild. `./x.py doc --stage 1 src/libstd` will document `libstd` with a freshly built rustdoc (if necessary), and will not rebuild rustc on modifications to rustdoc.

r? @alexcrichton

src/Cargo.toml

@@ -16,6 +16,7 @@ members = [
   "tools/remote-test-server",
   "tools/rust-installer",
   "tools/cargo",
+  "tools/rustdoc",
   "tools/rls",
   # FIXME(https://github.com/rust-lang/cargo/issues/4089): move these to exclude
   "tools/rls/test_data/borrow_error",

src/bootstrap/builder.rs

@@ -256,7 +256,7 @@ impl<'a> Builder<'a> {
                 compile::StartupObjects, tool::BuildManifest, tool::Rustbook, tool::ErrorIndex,
                 tool::UnstableBookGen, tool::Tidy, tool::Linkchecker, tool::CargoTest,
                 tool::Compiletest, tool::RemoteTestServer, tool::RemoteTestClient,
-                tool::RustInstaller, tool::Cargo, tool::Rls),
+                tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc),
             Kind::Test => describe!(check::Tidy, check::Bootstrap, check::DefaultCompiletest,
                 check::HostCompiletest, check::Crate, check::CrateLibrustc, check::Linkcheck,
                 check::Cargotest, check::Cargo, check::Rls, check::Docs, check::ErrorIndex,
@@ -412,12 +412,23 @@ impl<'a> Builder<'a> {
         }
     }
 
-    /// Get the `rustdoc` executable next to the specified compiler
     pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
-        let mut rustdoc = self.rustc(compiler);
-        rustdoc.pop();
-        rustdoc.push(exe("rustdoc", &compiler.host));
-        rustdoc
+        self.ensure(tool::Rustdoc { target_compiler: compiler })
+    }
+
+    pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
+        let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
+        cmd
+            .env("RUSTC_STAGE", compiler.stage.to_string())
+            .env("RUSTC_SYSROOT", if compiler.is_snapshot(&self.build) {
+                INTERNER.intern_path(self.build.rustc_snapshot_libdir())
+            } else {
+                self.sysroot(compiler)
+            })
+            .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
+            .env("CFG_RELEASE_CHANNEL", &self.build.config.channel)
+            .env("RUSTDOC_REAL", self.rustdoc(compiler));
+        cmd
     }
 
     /// Prepares an invocation of `cargo` to be run.
@@ -469,7 +480,11 @@ impl<'a> Builder<'a> {
              .env("RUSTC_LIBDIR", self.rustc_libdir(compiler))
              .env("RUSTC_RPATH", self.config.rust_rpath.to_string())
              .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
-             .env("RUSTDOC_REAL", self.rustdoc(compiler))
+             .env("RUSTDOC_REAL", if cmd == "doc" || cmd == "test" {
+                 self.rustdoc(compiler)
+             } else {
+                 PathBuf::from("/path/to/nowhere/rustdoc/not/required")
+             })
              .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
 
         if mode != Mode::Tool {
@@ -560,6 +575,9 @@ impl<'a> Builder<'a> {
         // FIXME: should update code to not require this env var
         cargo.env("CFG_COMPILER_HOST_TRIPLE", target);
 
+        // Set this for all builds to make sure doc builds also get it.
+        cargo.env("CFG_RELEASE_CHANNEL", &self.build.config.channel);
+
         if self.is_verbose() {
             cargo.arg("-v");
         }

src/bootstrap/check.rs

@@ -119,7 +119,7 @@ impl Step for Linkcheck {
     }
 
     fn make_run(run: RunConfig) {
-        run.builder.ensure(Linkcheck { host: run.host });
+        run.builder.ensure(Linkcheck { host: run.target });
     }
 }
 
@@ -140,7 +140,7 @@ impl Step for Cargotest {
     fn make_run(run: RunConfig) {
         run.builder.ensure(Cargotest {
             stage: run.builder.top_stage,
-            host: run.host,
+            host: run.target,
         });
     }
 
@@ -194,7 +194,7 @@ impl Step for Cargo {
         let build = builder.build;
         let compiler = builder.compiler(self.stage, self.host);
 
-        builder.ensure(tool::Cargo { stage: self.stage, target: self.host });
+        builder.ensure(tool::Cargo { compiler, target: self.host });
         let mut cargo = builder.cargo(compiler, Mode::Tool, self.host, "test");
         cargo.arg("--manifest-path").arg(build.src.join("src/tools/cargo/Cargo.toml"));
         if !build.fail_fast {
@@ -240,7 +240,7 @@ impl Step for Rls {
         let host = self.host;
         let compiler = builder.compiler(stage, host);
 
-        builder.ensure(tool::Rls { stage: self.stage, target: self.host });
+        builder.ensure(tool::Rls { compiler, target: self.host });
         let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
         cargo.arg("--manifest-path").arg(build.src.join("src/tools/rls/Cargo.toml"));
 
@@ -562,7 +562,12 @@ impl Step for Compiletest {
         cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler));
         cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target));
         cmd.arg("--rustc-path").arg(builder.rustc(compiler));
-        cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
+
+        // Avoid depending on rustdoc when we don't need it.
+        if mode == "rustdoc" || mode == "run-make" {
+            cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
+        }
+
         cmd.arg("--src-base").arg(build.src.join("src/test").join(suite));
         cmd.arg("--build-base").arg(testdir(build, compiler.host).join(suite));
         cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target));
@@ -809,8 +814,7 @@ fn markdown_test(builder: &Builder, compiler: Compiler, markdown: &Path) {
     }
 
     println!("doc tests for: {}", markdown.display());
-    let mut cmd = Command::new(builder.rustdoc(compiler));
-    builder.add_rustc_lib_path(compiler, &mut cmd);
+    let mut cmd = builder.rustdoc_cmd(compiler);
     build.add_rust_test_threads(&mut cmd);
     cmd.arg("--test");
     cmd.arg(markdown);
@@ -1165,7 +1169,7 @@ impl Step for RemoteCopyLibs {
         println!("REMOTE copy libs to emulator ({})", target);
         t!(fs::create_dir_all(build.out.join("tmp")));
 
-        let server = builder.ensure(tool::RemoteTestServer { stage: compiler.stage, target });
+        let server = builder.ensure(tool::RemoteTestServer { compiler, target });
 
         // Spawn the emulator and wait for it to come online
         let tool = builder.tool_exe(Tool::RemoteTestClient);

src/bootstrap/compile.rs

@@ -719,15 +719,6 @@ impl Step for Assemble {
         let _ = fs::remove_file(&compiler);
         copy(&rustc, &compiler);
 
-        // See if rustdoc exists to link it into place
-        let rustdoc = exe("rustdoc", &*host);
-        let rustdoc_src = out_dir.join(&rustdoc);
-        let rustdoc_dst = bindir.join(&rustdoc);
-        if fs::metadata(&rustdoc_src).is_ok() {
-            let _ = fs::remove_file(&rustdoc_dst);
-            copy(&rustdoc_src, &rustdoc_dst);
-        }
-
         target_compiler
     }
 }

src/bootstrap/dist.rs

@@ -413,6 +413,9 @@ impl Step for Rustc {
             t!(fs::create_dir_all(image.join("bin")));
             cp_r(&src.join("bin"), &image.join("bin"));
 
+            install(&builder.ensure(tool::Rustdoc { target_compiler: compiler }),
+                &image.join("bin"), 0o755);
+
             // Copy runtime DLLs needed by the compiler
             if libdir != "bin" {
                 for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) {
@@ -963,7 +966,10 @@ impl Step for Cargo {
         // Prepare the image directory
         t!(fs::create_dir_all(image.join("share/zsh/site-functions")));
         t!(fs::create_dir_all(image.join("etc/bash_completion.d")));
-        let cargo = builder.ensure(tool::Cargo { stage, target });
+        let cargo = builder.ensure(tool::Cargo {
+            compiler: builder.compiler(stage, build.build),
+            target
+        });
         install(&cargo, &image.join("bin"), 0o755);
         for man in t!(etc.join("man").read_dir()) {
             let man = t!(man);
@@ -1046,7 +1052,10 @@ impl Step for Rls {
         t!(fs::create_dir_all(&image));
 
         // Prepare the image directory
-        let rls = builder.ensure(tool::Rls { stage, target });
+        let rls = builder.ensure(tool::Rls {
+            compiler: builder.compiler(stage, build.build),
+            target
+        });
         install(&rls, &image.join("bin"), 0o755);
         let doc = image.join("share/doc/rls");
         install(&src.join("README.md"), &doc, 0o644);