feat: rebuild(-ish) rustdoc json for different version of same crates

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -524,8 +524,10 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
             }
             CompileMode::Doc => {
                 let path = if bcx.build_config.intent.wants_doc_json_output() {
-                    self.output_dir(unit)
-                        .join(format!("{}.json", unit.target.crate_name()))
+                    // Always use 'new' layout for '--output-format=json'.
+                    let crate_name = unit.target.crate_name();
+                    self.out_dir_new_layout(unit)
+                        .join(format!("{crate_name}.json"))
                 } else {
                     self.output_dir(unit)
                         .join(unit.target.crate_name())

src/cargo/core/compiler/build_runner/mod.rs

@@ -7,11 +7,12 @@ use std::sync::{Arc, Mutex};
 use crate::core::PackageId;
 use crate::core::compiler::compilation::{self, UnitOutput};
 use crate::core::compiler::locking::LockManager;
+use crate::core::compiler::rustdoc::is_rustdoc_json_output;
 use crate::core::compiler::{self, Unit, UserIntent, artifact};
 use crate::util::cache_lock::CacheLockMode;
 use crate::util::errors::CargoResult;
 use anyhow::{Context as _, bail};
-use cargo_util::paths;
+use cargo_util::paths::{self, copy};
 use cargo_util_terminal::report::{Level, Message};
 use filetime::FileTime;
 use itertools::Itertools;
@@ -235,6 +236,11 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         for unit in &self.bcx.roots {
             self.collect_tests_and_executables(unit)?;
 
+            // Uplift rustdoc
+            if is_rustdoc_json_output(&self) {
+                self.uplift_rustdoc(unit)?;
+            }
+
             // Collect information for `rustdoc --test`.
             if unit.mode.is_doc_test() {
                 let mut unstable_opts = false;
@@ -787,4 +793,21 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
                 .insert(unit.clone(), self.files().metadata(metadata_unit));
         }
     }
+
+    /// Uplifts final json to <doc_dir>/<crate_name>.json (for backward compatibility)
+    /// The final output is produced only after root unit is complete.
+    // TODO: It would be better if we don't do uplifting when new build layout is specified,
+    //       but there is no way we can collect new out dir paths from ops.
+    //       Thus we here always do uplifting.
+    fn uplift_rustdoc(&self, unit: &Unit) -> CargoResult<()> {
+        let doc_dir = self.files().output_dir(unit);
+        let doc_json_dir = self.files().out_dir_new_layout(unit);
+        let crate_name = unit.target.crate_name();
+        let filename = format!("{crate_name}.json");
+
+        let src_path = doc_json_dir.join(&filename);
+        copy(src_path, doc_dir.join(&filename))?;
+
+        Ok(())
+    }
 }

src/cargo/core/compiler/mod.rs

@@ -96,6 +96,7 @@ use self::unit_graph::UnitDep;
 
 use crate::core::compiler::future_incompat::FutureIncompatReport;
 use crate::core::compiler::locking::LockKey;
+use crate::core::compiler::rustdoc::is_rustdoc_json_output;
 use crate::core::compiler::timings::SectionTiming;
 pub use crate::core::compiler::unit::Unit;
 pub use crate::core::compiler::unit::UnitIndex;
@@ -868,7 +869,16 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     add_cap_lints(bcx, unit, &mut rustdoc);
 
     unit.kind.add_target_arg(&mut rustdoc);
-    let doc_dir = build_runner.files().output_dir(unit);
+
+    let doc_dir = if is_rustdoc_json_output(build_runner) {
+        // Always use new layout for '--output-format=json'.
+        // In fix for https://github.com/rust-lang/cargo/issues/16291
+
+        build_runner.files().out_dir_new_layout(unit)
+    } else {
+        build_runner.files().output_dir(unit)
+    };
+
     rustdoc.arg("-o").arg(&doc_dir);
     rustdoc.args(&features_args(unit));
     rustdoc.args(&check_cfg_args(unit));
@@ -972,7 +982,9 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
     let mut rustdoc = prepare_rustdoc(build_runner, unit)?;
 
     let crate_name = unit.target.crate_name();
+    let is_json_output = is_rustdoc_json_output(build_runner);
     let doc_dir = build_runner.files().output_dir(unit);
+    let new_doc_dir = build_runner.files().out_dir_new_layout(unit);
     // Create the documentation directory ahead of time as rustdoc currently has
     // a bug where concurrent invocations will race to create this directory if
     // it doesn't already exist.
@@ -1055,12 +1067,17 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             }
         }
 
-        let crate_dir = doc_dir.join(&crate_name);
+        let crate_dir = if is_json_output {
+            new_doc_dir
+        } else {
+            doc_dir.join(&crate_name)
+        };
+
         if crate_dir.exists() {
             // Remove output from a previous build. This ensures that stale
             // files for removed items are removed.
             debug!("removing pre-existing doc directory {:?}", crate_dir);
-            paths::remove_dir_all(crate_dir)?;
+            paths::remove_dir_all(&crate_dir)?;
         }
         state.running(&rustdoc);
         let timestamp = paths::set_invocation_time(&fingerprint_dir)?;

src/cargo/core/compiler/rustdoc.rs

@@ -242,6 +242,12 @@ pub fn add_root_urls(
     Ok(())
 }
 
+/// Checks whether JSON output should be enabled for rusdoc invocation
+pub fn is_rustdoc_json_output(build_runner: &BuildRunner<'_, '_>) -> bool {
+    build_runner.bcx.build_config.intent.wants_doc_json_output()
+        && build_runner.bcx.gctx.cli_unstable().unstable_options
+}
+
 /// Adds unstable flag [`--output-format`][1] to the given `rustdoc`
 /// invocation. This is for unstable feature `-Zunstable-features`.
 ///
@@ -250,15 +256,13 @@ pub fn add_output_format(
     build_runner: &BuildRunner<'_, '_>,
     rustdoc: &mut ProcessBuilder,
 ) -> CargoResult<()> {
-    let gctx = build_runner.bcx.gctx;
-    if !gctx.cli_unstable().unstable_options {
-        tracing::debug!("`unstable-options` is ignored, required -Zunstable-options flag");
-        return Ok(());
-    }
-
-    if build_runner.bcx.build_config.intent.wants_doc_json_output() {
+    if is_rustdoc_json_output(build_runner) {
         rustdoc.arg("-Zunstable-options");
         rustdoc.arg("--output-format=json");
+    } else {
+        if !build_runner.bcx.gctx.cli_unstable().unstable_options {
+            tracing::debug!("`unstable-options` is ignored, required -Zunstable-options flag");
+        }
     }
 
     Ok(())

tests/testsuite/build_dir.rs

@@ -501,6 +501,46 @@ fn cargo_doc_should_output_to_target_dir() {
     assert_exists(&docs_dir.join("foo/index.html"));
 }
 
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn cargo_rustdoc_json_should_output_to_target_dir() {
+    let p = project()
+        .file("src/lib.rs", "")
+        .file(
+            ".cargo/config.toml",
+            r#"
+            [build]
+            target-dir = "target-dir"
+            build-dir = "build-dir"
+            "#,
+        )
+        .build();
+
+    p.cargo("-Zbuild-dir-new-layout rustdoc -Zfine-grain-locking -Zunstable-options --output-format json")
+        .masquerade_as_nightly_cargo(&["new build-dir layout", "rustdoc-output-format"])
+        .enable_mac_dsym()
+        .run();
+
+    let docs_dir = p.root().join("target-dir/doc");
+
+    assert_exists(&docs_dir);
+    assert_exists(&docs_dir.join("foo.json"));
+
+    p.root().join("build-dir").assert_build_dir_layout(str![
+        r#"
+[ROOT]/foo/build-dir/.rustc_info.json
+[ROOT]/foo/build-dir/.rustdoc_fingerprint.json
+[ROOT]/foo/build-dir/CACHEDIR.TAG
+[ROOT]/foo/build-dir/debug/.cargo-build-lock
+[ROOT]/foo/build-dir/debug/build/foo/[HASH]/.lock
+[ROOT]/foo/build-dir/debug/build/foo/[HASH]/fingerprint/doc-lib-foo
+[ROOT]/foo/build-dir/debug/build/foo/[HASH]/fingerprint/doc-lib-foo.json
+[ROOT]/foo/build-dir/debug/build/foo/[HASH]/fingerprint/invoked.timestamp
+[ROOT]/foo/build-dir/debug/build/foo/[HASH]/out/foo.json
+
+"#
+    ]);
+}
+
 #[cargo_test]
 fn cargo_package_should_build_in_build_dir_and_output_to_target_dir() {
     let p = project()

tests/testsuite/build_dir_legacy.rs

@@ -459,6 +459,45 @@ fn cargo_doc_should_output_to_target_dir() {
     assert_exists(&docs_dir.join("foo/index.html"));
 }
 
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn cargo_rustdoc_json_should_output_to_target_dir() {
+    let p = project()
+        .file("src/lib.rs", "")
+        .file(
+            ".cargo/config.toml",
+            r#"
+            [build]
+            target-dir = "target-dir"
+            build-dir = "build-dir"
+            "#,
+        )
+        .build();
+
+    p.cargo("rustdoc -Zunstable-options --output-format json")
+        .masquerade_as_nightly_cargo(&["new build-dir layout", "rustdoc-output-format"])
+        .enable_mac_dsym()
+        .run();
+
+    let docs_dir = p.root().join("target-dir/doc");
+
+    assert_exists(&docs_dir);
+    assert_exists(&docs_dir.join("foo.json"));
+
+    p.root().join("build-dir").assert_build_dir_layout(str![
+        r#"
+[ROOT]/foo/build-dir/.rustc_info.json
+[ROOT]/foo/build-dir/.rustdoc_fingerprint.json
+[ROOT]/foo/build-dir/CACHEDIR.TAG
+[ROOT]/foo/build-dir/debug/.cargo-build-lock
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo.json
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/invoked.timestamp
+[ROOT]/foo/build-dir/debug/build/foo-[HASH]/out/foo.json
+
+"#
+    ]);
+}
+
 #[cargo_test]
 fn cargo_package_should_build_in_build_dir_and_output_to_target_dir() {
     let p = project()

tests/testsuite/rustdoc.rs

@@ -1,5 +1,7 @@
 //! Tests for the `cargo rustdoc` command.
 
+use std::fs;
+
 use crate::prelude::*;
 use cargo_test_support::str;
 use cargo_test_support::{basic_manifest, cross_compile, project};
@@ -44,7 +46,24 @@ fn rustdoc_simple_json() {
         .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
         .with_stderr_data(str![[r#"
 [DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
-[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/doc [..] --output-format=json[..]
+[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/debug/build/foo-[HASH]/out [..] --output-format=json[..]
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/foo.json
+
+"#]])
+        .run();
+    assert!(p.root().join("target/doc/foo.json").is_file());
+}
+
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn rustdoc_json_with_new_layout() {
+    let p = project().file("src/lib.rs", "").build();
+
+    p.cargo("rustdoc -Z unstable-options -Z build-dir-new-layout  --output-format json -v")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
+[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/debug/build/foo/[HASH]/out [..] --output-format=json[..]
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/foo/target/doc/foo.json
 
@@ -322,3 +341,99 @@ fn fail_with_glob() {
 "#]])
         .run();
 }
+
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn rustdoc_json_same_crate_different_version() {
+    let entry = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "entry"
+                version = "0.1.0"
+                edition = "2021"
+
+                [dependencies]
+                dep_v1 = { path = "../dep_v1", package = "dep" }
+                dep_v2 = { path = "../dep_v2", package = "dep" }
+            "#,
+        )
+        .file("src/lib.rs", "pub fn entry() {}")
+        .build();
+
+    let _dep_v1 = project()
+        .at("dep_v1")
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "dep"
+                version = "1.0.0"
+                edition = "2021"
+            "#,
+        )
+        .file("src/lib.rs", "pub fn dep_v1_fn() {}")
+        .build();
+
+    let _dep_v2 = project()
+        .at("dep_v2")
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "dep"
+                version = "2.0.0"
+                edition = "2021"
+            "#,
+        )
+        .file("src/lib.rs", "pub fn dep_v2_fn() {}")
+        .build();
+
+    entry
+        .cargo("rustdoc -v -Z unstable-options --output-format json -p dep@1.0.0")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[LOCKING] 2 packages to latest compatible versions
+[DOCUMENTING] dep v1.0.0 ([ROOT]/dep_v1)
+[RUNNING] `rustdoc [..] --crate-name dep [ROOT]/dep_v1/src/lib.rs [..] --output-format=json[..]`
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/dep.json
+
+"#]])
+        .run();
+
+    let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
+    assert!(dep_json.contains("dep_v1_fn"));
+    assert!(!dep_json.contains("dep_v2_fn"));
+
+    entry
+        .cargo("rustdoc -v -Z unstable-options --output-format json -p dep@2.0.0")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[DOCUMENTING] dep v2.0.0 ([ROOT]/dep_v2)
+[RUNNING] `rustdoc [..] --crate-name dep [ROOT]/dep_v2/src/lib.rs [..] --output-format=json[..]`
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/dep.json
+
+"#]])
+        .run();
+
+    let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
+    assert!(!dep_json.contains("dep_v1_fn"));
+    assert!(dep_json.contains("dep_v2_fn"));
+
+    entry
+        .cargo("rustdoc -v -Z unstable-options --output-format json -p dep@1.0.0")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[FRESH] dep v1.0.0 ([ROOT]/dep_v1)
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/dep.json
+
+"#]])
+        .run();
+
+    let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
+    assert!(dep_json.contains("dep_v1_fn"));
+    assert!(!dep_json.contains("dep_v2_fn"));
+}