Rollup merge of #115358 - durin42:compress-debuginfo, r=oli-obk

debuginfo: add compiler option to allow compressed debuginfo sections

LLVM already supports emitting compressed debuginfo. In debuginfo=full builds, the debug section is often a large amount of data, and it typically compresses very well (3x is not unreasonable.) We add a new knob to allow debuginfo to be compressed when the matching LLVM functionality is present. Like clang, if a known-but-disabled compression mechanism is requested, we disable compression and emit uncompressed debuginfo sections.

The API is different enough on older LLVMs we just pretend the support
is missing on LLVM older than 16.

Author: matthiaskrgr

compiler/rustc_codegen_llvm/messages.ftl

@@ -83,6 +83,8 @@ codegen_llvm_unknown_ctarget_feature_prefix =
     unknown feature specified for `-Ctarget-feature`: `{$feature}`
     .note = features must begin with a `+` to enable or `-` to disable it
 
+codegen_llvm_unknown_debuginfo_compression = unknown debuginfo compression algorithm {$algorithm} - will fall back to uncompressed debuginfo
+
 codegen_llvm_write_bytecode = failed to write bytecode to {$path}: {$err}
 
 codegen_llvm_write_ir = failed to write LLVM IR to {$path}

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -5,13 +5,17 @@ use crate::back::profiling::{
 use crate::base;
 use crate::common;
 use crate::errors::{
-    CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, WithLlvmError, WriteBytecode,
+    CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression,
+    WithLlvmError, WriteBytecode,
 };
 use crate::llvm::{self, DiagnosticInfo, PassManager};
 use crate::llvm_util;
 use crate::type_::Type;
 use crate::LlvmCodegenBackend;
 use crate::ModuleLlvm;
+use llvm::{
+    LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols,
+};
 use rustc_codegen_ssa::back::link::ensure_removed;
 use rustc_codegen_ssa::back::write::{
     BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig,
@@ -234,6 +238,22 @@ pub fn target_machine_factory(
         args_cstr_buff
     };
 
+    let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
+    match sess.opts.debuginfo_compression {
+        rustc_session::config::DebugInfoCompression::Zlib => {
+            if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } {
+                sess.emit_warning(UnknownCompression { algorithm: "zlib" });
+            }
+        }
+        rustc_session::config::DebugInfoCompression::Zstd => {
+            if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } {
+                sess.emit_warning(UnknownCompression { algorithm: "zstd" });
+            }
+        }
+        rustc_session::config::DebugInfoCompression::None => {}
+    };
+    let debuginfo_compression = SmallCStr::new(&debuginfo_compression);
+
     Arc::new(move |config: TargetMachineFactoryConfig| {
         let split_dwarf_file =
             path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0;
@@ -259,6 +279,7 @@ pub fn target_machine_factory(
                 relax_elf_relocations,
                 use_init_array,
                 split_dwarf_file.as_ptr(),
+                debuginfo_compression.as_ptr(),
                 force_emulated_tls,
                 args_cstr_buff.as_ptr() as *const c_char,
                 args_cstr_buff.len(),

compiler/rustc_codegen_llvm/src/errors.rs

@@ -226,3 +226,9 @@ pub(crate) struct WriteBytecode<'a> {
 pub(crate) struct CopyBitcode {
     pub err: std::io::Error,
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_unknown_debuginfo_compression)]
+pub struct UnknownCompression {
+    pub algorithm: &'static str,
+}

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2131,6 +2131,7 @@ extern "C" {
         RelaxELFRelocations: bool,
         UseInitArray: bool,
         SplitDwarfFile: *const c_char,
+        DebugInfoCompression: *const c_char,
         ForceEmulatedTls: bool,
         ArgsCstrBuff: *const c_char,
         ArgsCstrBuffLen: usize,
@@ -2366,6 +2367,10 @@ extern "C" {
 
     pub fn LLVMRustIsBitcode(ptr: *const u8, len: usize) -> bool;
 
+    pub fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool;
+
+    pub fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool;
+
     pub fn LLVMRustGetSymbols(
         buf_ptr: *const u8,
         buf_len: usize,

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -410,6 +410,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     bool RelaxELFRelocations,
     bool UseInitArray,
     const char *SplitDwarfFile,
+    const char *DebugInfoCompression,
     bool ForceEmulatedTls,
     const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
 
@@ -441,6 +442,16 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   if (SplitDwarfFile) {
       Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
   }
+#if LLVM_VERSION_GE(16, 0)
+  if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) {
+    Options.CompressDebugSections = DebugCompressionType::Zlib;
+  } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) {
+    Options.CompressDebugSections = DebugCompressionType::Zstd;
+  } else if (!strcmp("none", DebugInfoCompression)) {
+    Options.CompressDebugSections = DebugCompressionType::None;
+  }
+#endif
+
   Options.RelaxELFRelocations = RelaxELFRelocations;
   Options.UseInitArray = UseInitArray;
 

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -2044,3 +2044,19 @@ extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) {
   }
   return false;
 }
+
+extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() {
+#if LLVM_VERSION_GE(16, 0)
+  return llvm::compression::zlib::isAvailable();
+#else
+  return false;
+#endif
+}
+
+extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() {
+#if LLVM_VERSION_GE(16, 0)
+  return llvm::compression::zstd::isAvailable();
+#else
+  return false;
+#endif
+}

compiler/rustc_session/src/config.rs

@@ -381,6 +381,24 @@ pub enum DebugInfo {
     Full,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Hash)]
+pub enum DebugInfoCompression {
+    None,
+    Zlib,
+    Zstd,
+}
+
+impl ToString for DebugInfoCompression {
+    fn to_string(&self) -> String {
+        match self {
+            DebugInfoCompression::None => "none",
+            DebugInfoCompression::Zlib => "zlib",
+            DebugInfoCompression::Zstd => "zstd",
+        }
+        .to_owned()
+    }
+}
+
 /// Split debug-information is enabled by `-C split-debuginfo`, this enum is only used if split
 /// debug-information is enabled (in either `Packed` or `Unpacked` modes), and the platform
 /// uses DWARF for debug-information.
@@ -1015,6 +1033,7 @@ impl Default for Options {
             crate_types: Vec::new(),
             optimize: OptLevel::No,
             debuginfo: DebugInfo::None,
+            debuginfo_compression: DebugInfoCompression::None,
             lint_opts: Vec::new(),
             lint_cap: None,
             describe_lints: false,
@@ -2277,6 +2296,13 @@ fn select_debuginfo(matches: &getopts::Matches, cg: &CodegenOptions) -> DebugInf
     if max_g > max_c { DebugInfo::Full } else { cg.debuginfo }
 }
 
+fn select_debuginfo_compression(
+    _handler: &EarlyErrorHandler,
+    unstable_opts: &UnstableOptions,
+) -> DebugInfoCompression {
+    unstable_opts.debuginfo_compression
+}
+
 pub(crate) fn parse_assert_incr_state(
     handler: &EarlyErrorHandler,
     opt_assertion: &Option<String>,
@@ -2752,6 +2778,8 @@ pub fn build_session_options(
     // for more details.
     let debug_assertions = cg.debug_assertions.unwrap_or(opt_level == OptLevel::No);
     let debuginfo = select_debuginfo(matches, &cg);
+    let debuginfo_compression: DebugInfoCompression =
+        select_debuginfo_compression(handler, &unstable_opts);
 
     let mut search_paths = vec![];
     for s in &matches.opt_strs("L") {
@@ -2828,6 +2856,7 @@ pub fn build_session_options(
         crate_types,
         optimize: opt_level,
         debuginfo,
+        debuginfo_compression,
         lint_opts,
         lint_cap,
         describe_lints,
@@ -3113,11 +3142,11 @@ impl PpMode {
 /// how the hash should be calculated when adding a new command-line argument.
 pub(crate) mod dep_tracking {
     use super::{
-        BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, ErrorOutputType,
-        InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto, LocationDetail, LtoCli,
-        OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Passes, ResolveDocLinks,
-        SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion,
-        TraitSolver, TrimmedDefPaths,
+        BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression,
+        ErrorOutputType, InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto,
+        LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes,
+        Passes, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath,
+        SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
     };
     use crate::lint;
     use crate::options::WasiExecModel;
@@ -3195,6 +3224,7 @@ pub(crate) mod dep_tracking {
         OptLevel,
         LtoCli,
         DebugInfo,
+        DebugInfoCompression,
         UnstableFeatures,
         NativeLib,
         NativeLibKind,

compiler/rustc_session/src/options.rs

@@ -139,6 +139,7 @@ top_level_options!(
         /// can influence whether overflow checks are done or not.
         debug_assertions: bool [TRACKED],
         debuginfo: DebugInfo [TRACKED],
+        debuginfo_compression: DebugInfoCompression [TRACKED],
         lint_opts: Vec<(String, lint::Level)> [TRACKED_NO_CRATE_HASH],
         lint_cap: Option<lint::Level> [TRACKED_NO_CRATE_HASH],
         describe_lints: bool [UNTRACKED],
@@ -376,6 +377,7 @@ mod desc {
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
     pub const parse_cfprotection: &str = "`none`|`no`|`n` (default), `branch`, `return`, or `full`|`yes`|`y` (equivalent to `branch` and `return`)";
     pub const parse_debuginfo: &str = "either an integer (0, 1, 2), `none`, `line-directives-only`, `line-tables-only`, `limited`, or `full`";
+    pub const parse_debuginfo_compression: &str = "one of `none`, `zlib`, or `zstd`";
     pub const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`";
     pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of();
     pub const parse_optimization_fuel: &str = "crate=integer";
@@ -782,6 +784,19 @@ mod parse {
         true
     }
 
+    pub(crate) fn parse_debuginfo_compression(
+        slot: &mut DebugInfoCompression,
+        v: Option<&str>,
+    ) -> bool {
+        match v {
+            Some("none") => *slot = DebugInfoCompression::None,
+            Some("zlib") => *slot = DebugInfoCompression::Zlib,
+            Some("zstd") => *slot = DebugInfoCompression::Zstd,
+            _ => return false,
+        };
+        true
+    }
+
     pub(crate) fn parse_linker_flavor(slot: &mut Option<LinkerFlavorCli>, v: Option<&str>) -> bool {
         match v.and_then(LinkerFlavorCli::from_str) {
             Some(lf) => *slot = Some(lf),
@@ -1424,6 +1439,8 @@ options! {
         "emit discriminators and other data necessary for AutoFDO"),
     debug_macros: bool = (false, parse_bool, [TRACKED],
         "emit line numbers debug info inside macros (default: no)"),
+    debuginfo_compression: DebugInfoCompression = (DebugInfoCompression::None, parse_debuginfo_compression, [TRACKED],
+        "compress debug info sections (none, zlib, zstd, default: none)"),
     deduplicate_diagnostics: bool = (true, parse_bool, [UNTRACKED],
         "deduplicate identical diagnostics (default: yes)"),
     dep_info_omit_d_target: bool = (false, parse_bool, [TRACKED],

tests/run-make/compressed-debuginfo/Makefile

@@ -0,0 +1,15 @@
+# ignore-cross-compile
+include ../tools.mk
+
+# only-linux
+# min-llvm-version: 16.0
+#
+# This tests debuginfo-compression.
+
+all: zlib zstandard
+
+zlib:
+	test "`$(RUSTC) --crate-name=foo --crate-type=lib --emit=obj -C debuginfo=full -Z debuginfo-compression=zlib foo.rs 2>&1 | sed 's/.*unknown.*zlib.*/missing/' | head -n 1`" = missing || readelf -t $(TMPDIR)/foo.o | grep -q ZLIB
+
+zstandard:
+	test "`$(RUSTC) --crate-name=foo --crate-type=lib --emit=obj -C debuginfo=full -Z debuginfo-compression=zstd foo.rs 2>&1 | sed 's/.*unknown.*zstd.*/missing/' | head -n 1`" = missing || readelf -t $(TMPDIR)/foo.o | grep -q ZST

tests/run-make/compressed-debuginfo/foo.rs

@@ -0,0 +1,3 @@
+pub fn foo() -> i32 {
+    42
+}