From 60ad206c9855d9743ccebf85b6b2571094d56001 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Mon, 27 May 2024 20:10:46 -0700 Subject: [PATCH 1/9] Enable zstd for debug compression. Set LLVM_ENABLE_ZSTD alongside LLVM_ENABLE_ZLIB so that --compress-debug-sections=zstd is an option. --- src/bootstrap/src/core/build_steps/llvm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index bff731e13b0a9..1e995bb660caf 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -376,13 +376,13 @@ impl Step for Llvm { cfg.define("LLVM_PROFDATA_FILE", path); } - // Disable zstd to avoid a dependency on libzstd.so. - cfg.define("LLVM_ENABLE_ZSTD", "OFF"); - + // Libraries for ELF compression. if !target.is_windows() { cfg.define("LLVM_ENABLE_ZLIB", "ON"); + cfg.define("LLVM_ENABLE_ZSTD", "ON"); } else { cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + cfg.define("LLVM_ENABLE_ZSTD", "OFF"); } // Are we compiling for iOS/tvOS/watchOS/visionOS? From ba5e7a8efe63a0e7d42b00cfbdca617383fe7d90 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Mon, 27 May 2024 21:36:59 -0700 Subject: [PATCH 2/9] Bump download-ci-llvm-stamp for LLVM_ENABLE_ZSTD. --- src/bootstrap/download-ci-llvm-stamp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index 258a034745134..11316004412e6 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/126298 +Last change is for: https://github.com/rust-lang/rust/pull/125642 From 6dd068e23fae747e6551ad73a269671419bda15c Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Wed, 29 May 2024 09:15:53 -0700 Subject: [PATCH 3/9] Statically link to zstd. --- src/bootstrap/src/core/build_steps/llvm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 1e995bb660caf..32610e78c85ec 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -380,6 +380,7 @@ impl Step for Llvm { if !target.is_windows() { cfg.define("LLVM_ENABLE_ZLIB", "ON"); cfg.define("LLVM_ENABLE_ZSTD", "ON"); + cfg.define("LLVM_USE_STATIC_ZSTD", "TRUE"); } else { cfg.define("LLVM_ENABLE_ZLIB", "OFF"); cfg.define("LLVM_ENABLE_ZSTD", "OFF"); From 1cc2120ef4d7a12a4b86d78c5bfc3d55af906786 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Wed, 29 May 2024 18:19:28 -0700 Subject: [PATCH 4/9] Move ELF compression related libraries to configure_llvm so those options also apply to lld. --- src/bootstrap/src/core/build_steps/llvm.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 32610e78c85ec..c14881836d447 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -376,16 +376,6 @@ impl Step for Llvm { cfg.define("LLVM_PROFDATA_FILE", path); } - // Libraries for ELF compression. - if !target.is_windows() { - cfg.define("LLVM_ENABLE_ZLIB", "ON"); - cfg.define("LLVM_ENABLE_ZSTD", "ON"); - cfg.define("LLVM_USE_STATIC_ZSTD", "TRUE"); - } else { - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - cfg.define("LLVM_ENABLE_ZSTD", "OFF"); - } - // Are we compiling for iOS/tvOS/watchOS/visionOS? if target.contains("apple-ios") || target.contains("apple-tvos") @@ -833,6 +823,16 @@ fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmak } } + // Libraries for ELF section compression. + if !target.is_windows() { + cfg.define("LLVM_ENABLE_ZLIB", "ON"); + cfg.define("LLVM_ENABLE_ZSTD", "ON"); + cfg.define("LLVM_USE_STATIC_ZSTD", "TRUE"); + } else { + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + cfg.define("LLVM_ENABLE_ZSTD", "OFF"); + } + if let Some(ref linker) = builder.config.llvm_use_linker { cfg.define("LLVM_USE_LINKER", linker); } From c473dac4a8f229ba8d2590f7b26cfb3077eb2654 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Wed, 29 May 2024 18:20:07 -0700 Subject: [PATCH 5/9] Handle llvm-config --system-libs output that contains static libraries. --- compiler/rustc_llvm/build.rs | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs index 3aa852c83045d..fdcad014ad011 100644 --- a/compiler/rustc_llvm/build.rs +++ b/compiler/rustc_llvm/build.rs @@ -260,6 +260,7 @@ fn main() { cmd.args(&components); for lib in output(&mut cmd).split_whitespace() { + let mut is_static = false; let name = if let Some(stripped) = lib.strip_prefix("-l") { stripped } else if let Some(stripped) = lib.strip_prefix('-') { @@ -267,8 +268,24 @@ fn main() { } else if Path::new(lib).exists() { // On MSVC llvm-config will print the full name to libraries, but // we're only interested in the name part - let name = Path::new(lib).file_name().unwrap().to_str().unwrap(); - name.trim_end_matches(".lib") + // On Unix when we get a static library llvm-config will print the + // full name and we *are* interested in the path, but we need to + // handle it separately. For example, when statically linking to + // libzstd llvm-config will output something like + // -lrt -ldl -lm -lz /usr/local/lib/libzstd.a -lxml2 + // and we transform the zstd part into + // cargo:rustc-link-search-native=/usr/local/lib + // cargo:rustc-link-lib=static=zstd + let path = Path::new(lib); + if lib.ends_with(".a") { + is_static = true; + println!("cargo:rustc-link-search=native={}", path.parent().unwrap().display()); + let name = path.file_stem().unwrap().to_str().unwrap(); + name.trim_start_matches("lib") + } else { + let name = path.file_name().unwrap().to_str().unwrap(); + name.trim_end_matches(".lib") + } } else if lib.ends_with(".lib") { // Some MSVC libraries just come up with `.lib` tacked on, so chop // that off @@ -286,7 +303,13 @@ fn main() { continue; } - let kind = if name.starts_with("LLVM") { llvm_kind } else { "dylib" }; + let kind = if name.starts_with("LLVM") { + llvm_kind + } else if is_static { + "static" + } else { + "dylib" + }; println!("cargo:rustc-link-lib={kind}={name}"); } From 5570b0d2f0ffaa23ccec0a1ecb1fd99ac78fc47e Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Fri, 12 Jul 2024 09:00:05 -0700 Subject: [PATCH 6/9] Test --compress-debug-sections with rust-lld. --- .../rust-lld-compress-debug-sections/main.rs | 4 ++ .../rust-lld-compress-debug-sections/rmake.rs | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tests/run-make/rust-lld-compress-debug-sections/main.rs create mode 100644 tests/run-make/rust-lld-compress-debug-sections/rmake.rs diff --git a/tests/run-make/rust-lld-compress-debug-sections/main.rs b/tests/run-make/rust-lld-compress-debug-sections/main.rs new file mode 100644 index 0000000000000..bf159cd941cee --- /dev/null +++ b/tests/run-make/rust-lld-compress-debug-sections/main.rs @@ -0,0 +1,4 @@ +// Test linking using `cc` with `rust-lld`, using the unstable CLI described in MCP 510 +// see https://github.com/rust-lang/compiler-team/issues/510 for more info + +fn main() {} diff --git a/tests/run-make/rust-lld-compress-debug-sections/rmake.rs b/tests/run-make/rust-lld-compress-debug-sections/rmake.rs new file mode 100644 index 0000000000000..cf86713c83b54 --- /dev/null +++ b/tests/run-make/rust-lld-compress-debug-sections/rmake.rs @@ -0,0 +1,38 @@ +// Checks the `compress-debug-sections` option on rust-lld. + +//@ needs-rust-lld +//@ only-linux +//@ ignore-cross-compile + +// FIXME: This test isn't comprehensive and isn't covering all possible combinations. + +use run_make_support::{assert_contains, cmd, llvm_readobj, run_in_tmpdir, rustc}; + +fn check_compression(compression: &str, to_find: &str) { + run_in_tmpdir(|| { + let out = rustc() + .arg("-Zlinker-features=+lld") + .arg("-Clink-self-contained=+linker") + .arg("-Zunstable-options") + .arg("-Cdebuginfo=full") + .link_arg(&format!("-Wl,--compress-debug-sections={compression}")) + .input("main.rs") + .run_unchecked(); + let stderr = out.stderr_utf8(); + if stderr.is_empty() { + llvm_readobj().arg("-t").arg("main").run().assert_stdout_contains(to_find); + } else { + assert_contains( + stderr, + format!( + "LLVM was not built with LLVM_ENABLE_{to_find} or did not find {compression} at build time" + ), + ); + } + }); +} + +fn main() { + check_compression("zlib", "ZLIB"); + check_compression("zstd", "ZSTD"); +} From 36ccc08475730fd3d2183bfefd7f451b27a46b87 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Fri, 12 Jul 2024 09:21:45 -0700 Subject: [PATCH 7/9] try to please tidy --- tests/run-make/rust-lld-compress-debug-sections/rmake.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/run-make/rust-lld-compress-debug-sections/rmake.rs b/tests/run-make/rust-lld-compress-debug-sections/rmake.rs index cf86713c83b54..901503f40e84b 100644 --- a/tests/run-make/rust-lld-compress-debug-sections/rmake.rs +++ b/tests/run-make/rust-lld-compress-debug-sections/rmake.rs @@ -25,7 +25,8 @@ fn check_compression(compression: &str, to_find: &str) { assert_contains( stderr, format!( - "LLVM was not built with LLVM_ENABLE_{to_find} or did not find {compression} at build time" + "\ +LLVM was not built with LLVM_ENABLE_{to_find} or did not find {compression} at build time" ), ); } From 86f954ddc4a6205650e4b5bdf12058862add3ec5 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Wed, 17 Jul 2024 12:59:09 -0700 Subject: [PATCH 8/9] Add zlib dev package for building lld. --- src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index 9025e9bb0a3af..f870b8499dc68 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ xz-utils \ mingw-w64 \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/ From 9d572b4fa672d9cbfc369a6cf9ad78d2238c5f1c Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Wed, 17 Jul 2024 14:02:48 -0700 Subject: [PATCH 9/9] Add zstd dev package for building lld. --- src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index f870b8499dc68..19683317126ab 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ mingw-w64 \ zlib1g-dev \ + libzstd-dev \ && rm -rf /var/lib/apt/lists/* COPY scripts/sccache.sh /scripts/