separateDebugInfo: add symlinks to executable and source for debuginfod support

doc/release-notes/rl-2511.section.md

@@ -19,6 +19,8 @@
 - `gentium` package now provides `Gentium-*.ttf` files, and not `GentiumPlus-*.ttf` files like before. The font identifiers `Gentium Plus*` are available in the `gentium-plus` package, and if you want to use the more recently updated package `gentium` [by sil](https://software.sil.org/gentium/), you should update your configuration files to use the `Gentium` font identifier.
 - `space-orbit` package has been removed due to lack of upstream maintenance. Debian upstream stopped tracking it in 2011.
 
+- Derivations setting both `separateDebugInfo` and one of `allowedReferences`, `allowedRequistes`, `disallowedReferences` or `disallowedRequisites` must now set `__structuredAttrs` to `true`. The effect of reference whitelisting or blacklisting will be disabled on the `debug` output created by `separateDebugInfo`.
+
 - `gnome-keyring` no longer ships with an SSH agent anymore because it has been deprecated upstream. You should use `gcr_4` instead, which provides the same features. More information on why this was done can be found on [the relevant GCR upstream PR](https://gitlab.gnome.org/GNOME/gcr/-/merge_requests/67).
 
 ## Other Notable Changes {#sec-nixpkgs-release-25.11-notable-changes}
@@ -31,6 +33,14 @@
 
 - `vmalert` now supports multiple instances with the option `services.vmalert.instances."".enable`
 
+- The debug outputs produced by `separateDebugInfo = true;` now contain symlinks mapping build-ids to the original source and ELF file.
+  Specifically, if `$out/bin/ninja` has build-id `483bd7f7229bdb06462222e1e353e4f37e15c293`, then
+  * `$debug/lib/debug/.build-id/48/3bd7f7229bdb06462222e1e353e4f37e15c293.executable` is a symlink to `$out/bin/ninja`
+  * `$debug/lib/debug/.build-id/48/3bd7f7229bdb06462222e1e353e4f37e15c293.source` is a symlink to the value of `$src` during build
+  * `$debug/lib/debug/.build-id/48/3bd7f7229bdb06462222e1e353e4f37e15c293.sourceoverlay` is a symlink to a directory with the same structure as the expanded `$sourceRoot` but containing only a copy of files which were patched during the build
+  * `$debug/lib/debug/.build-id/48/3bd7f7229bdb06462222e1e353e4f37e15c293.debug` is the file containing debug symbols (like before).
+
+
 ## Nixpkgs Library {#sec-nixpkgs-release-25.11-lib}
 
 <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

pkgs/applications/version-management/git/default.nix

@@ -94,6 +94,7 @@ stdenv.mkDerivation (finalAttrs: {
 
   outputs = [ "out" ] ++ lib.optional withManual "doc";
   separateDebugInfo = true;
+  __structuredAttrs = true;
 
   hardeningDisable = [ "format" ];
 
@@ -165,7 +166,7 @@ stdenv.mkDerivation (finalAttrs: {
     ];
 
   # required to support pthread_cancel()
-  NIX_LDFLAGS =
+  env.NIX_LDFLAGS =
     lib.optionalString (stdenv.cc.isGNU && stdenv.hostPlatform.libc == "glibc") "-lgcc_s"
     + lib.optionalString (stdenv.hostPlatform.isFreeBSD) "-lthr";
 

pkgs/build-support/setup-hooks/auto-patchelf.sh

@@ -95,6 +95,7 @@ autoPatchelfPostFixup() {
     if [[ -z "${dontAutoPatchelf-}" ]]; then
         autoPatchelf -- $(for output in $(getAllOutputNames); do
             [ -e "${!output}" ] || continue
+            [ "${output}" = debug ] && continue
             echo "${!output}"
         done)
     fi

pkgs/build-support/setup-hooks/separate-debug-info.sh

@@ -3,18 +3,47 @@ export NIX_LDFLAGS+=" --compress-debug-sections=zlib"
 export NIX_CFLAGS_COMPILE+=" -ggdb -Wa,--compress-debug-sections"
 export NIX_RUSTFLAGS+=" -g -C strip=none"
 
+cksumAlgo=sha256
+
 fixupOutputHooks+=(_separateDebugInfo)
+postUnpackHooks+=(_recordPristineSourceHashes)
+
+_recordPristineSourceHashes() {
+    # shellcheck disable=2154
+    [ -e "$sourceRoot" ] || return 0
+
+    local checksumFileName=__nix_source_checksums
+    echo "separate-debug-info: recording checksum of source files for debug support..."
+    find "$sourceRoot" -type f -exec cksum -a "$cksumAlgo" '{}' \+ > "$checksumFileName"
+    recordedSourceChecksumsFileName="$(readlink -f "$checksumFileName")"
+}
 
 _separateDebugInfo() {
+    # shellcheck disable=2154
     [ -e "$prefix" ] || return 0
 
-    local dst="${debug:-$out}"
-    if [ "$prefix" = "$dst" ]; then return 0; fi
+    local debugOutput="${debug:-$out}"
+    if [ "$prefix" = "$debugOutput" ]; then return 0; fi
 
     # in case there is nothing to strip, don't fail the build
-    mkdir -p "$dst"
+    mkdir -p "$debugOutput"
 
-    dst="$dst/lib/debug/.build-id"
+    local dst="$debugOutput/lib/debug/.build-id"
+
+    local source
+    local sourceOverlay
+    # shellcheck disable=2154
+    if [ -e "$src" ]; then
+        source="$src"
+        if [ -n "${recordedSourceChecksumsFileName:-}" ]; then
+            sourceOverlay="$debugOutput/src/overlay"
+        else
+            sourceOverlay=""
+        fi
+    else
+        source=""
+        sourceOverlay=""
+    fi
 
     # Find executables and dynamic libraries.
     local i
@@ -25,30 +54,64 @@ _separateDebugInfo() {
         [ -z "${OBJCOPY:-}" ] && echo "_separateDebugInfo: '\$OBJCOPY' variable is empty, skipping." 1>&2 && break
 
         # Extract the Build ID. FIXME: there's probably a cleaner way.
-        local id="$($READELF -n "$i" | sed 's/.*Build ID: \([0-9a-f]*\).*/\1/; t; d')"
+        local id
+        id="$($READELF -n "$i" | sed 's/.*Build ID: \([0-9a-f]*\).*/\1/; t; d')"
         if [ "${#id}" != 40 ]; then
             echo "could not find build ID of $i, skipping" >&2
             continue
         fi
 
+
         # Extract the debug info.
         echo "separating debug info from $i (build ID $id)"
 
-        destDir=$dst/${id:0:2}
-        destFile=$dst/${id:0:2}/${id:2}.debug
+        local debuginfoDir="$dst/${id:0:2}"
+        local buildIdPrefix="$debuginfoDir/${id:2}"
+        local debuginfoFile="$buildIdPrefix.debug"
+        local executableSymlink="$buildIdPrefix.executable"
+        local sourceSymlink="$buildIdPrefix.source"
+        local sourceOverlaySymlink="$buildIdPrefix.sourceoverlay"
 
-        mkdir -p "$destDir"
+        mkdir -p "$debuginfoDir"
 
-        if [ -f "$destFile" ]; then
+        if [ -f "$debuginfoFile" ]; then
             echo "separate-debug-info: warning: multiple files with build id $id found, overwriting"
         fi
 
         # This may fail, e.g. if the binary is for a different
         # architecture than we're building for.  (This happens with
         # firmware blobs in QEMU.)
-        if $OBJCOPY --only-keep-debug "$i" "$destFile"; then
+        if $OBJCOPY --only-keep-debug "$i" "$debuginfoFile"; then
             # If we succeeded, also a create a symlink <original-name>.debug.
-            ln -sfn ".build-id/${id:0:2}/${id:2}.debug" "$dst/../$(basename "$i")"
+            ln -sfn "$debuginfoFile" "$dst/../$(basename "$i")"
+            # also create a symlink mapping the build-id to the original elf file and the source
+            # debuginfod protocol relies on it
+            ln -sfn "$i" "$executableSymlink"
+            if [ -n "$source" ]; then
+                ln -sfn "$source" "$sourceSymlink"
+            fi
+            if [ -n "$sourceOverlay" ]; then
+                # create it lazily
+                if [ ! -d "$sourceOverlay" ]; then
+                    echo "separate-debug-info: copying patched source files to $sourceOverlay..."
+                    mkdir -p "$sourceOverlay"
+                    pushd "$(dirname "$recordedSourceChecksumsFileName")" || { echo "separate-debug-info: failed to cd parent directory of $recordedSourceChecksumsFileName"; return 1; }
+                    while IFS= read -r -d $'\0'  modifiedSourceFile; do
+                        if [ -z "$modifiedSourceFile" ]; then
+                            continue
+                        fi
+                        # this can happen with files with '\n' in their name
+                        if [ ! -f "$modifiedSourceFile" ]; then
+                            echo "separate-debug-info: cannot save modified source file $modifiedSourceFile: does not exist. ignoring"
+                            continue
+                        fi
+                        mkdir -p "$sourceOverlay/$(dirname "$modifiedSourceFile")"
+                        cp -v "$modifiedSourceFile" "$sourceOverlay/$modifiedSourceFile"
+                    done < <(LANG=C cksum -a "$cksumAlgo" --check --ignore-missing --quiet "$recordedSourceChecksumsFileName" 2>&1 | sed -n -e 's/: FAILED$/\x00/p' | sed -z -e 's/^\n//')
+                    popd || { echo "separate-debug-info: failed to popd" ; return 1; }
+                fi
+                ln -sfn "$sourceOverlay" "$sourceOverlaySymlink"
+            fi
         else
             # If we failed, try to clean up unnecessary directories
             rmdir -p "$dst/${id:0:2}" --ignore-fail-on-non-empty

pkgs/by-name/au/auto-patchelf/source/auto-patchelf.py

@@ -42,6 +42,13 @@ def is_dynamic_executable(elf: ELFFile) -> bool:
     # section but their ELF type is DYN.
     return bool(elf.get_section_by_name(".interp"))
 
+def is_separate_debug_object(elf: ELFFile) -> bool:
+    # objects created by separateDebugInfo = true have all the section headers
+    # of the unstripped objects but those that normal `strip` would have kept
+    # are NOBITS
+    text_section = elf.get_section_by_name(".text")
+    return elf.has_dwarf_info() and bool(text_section) and text_section.header['sh_type'] == "SHT_NOBITS"
+
 
 def get_dependencies(elf: ELFFile) -> list[list[Path]]:
     dependencies = []
@@ -174,6 +181,10 @@ def populate_cache(initial: list[Path], recursive: bool =False) -> None:
 
             try:
                 with open_elf(path) as elf:
+                    if is_separate_debug_object(elf):
+                        print(f"skipping {path} because it looks like a separate debug object")
+                        continue
+
                     osabi = get_osabi(elf)
                     arch = get_arch(elf)
                     rpath = [Path(p) for p in get_rpath(elf)

pkgs/development/compilers/openjdk/generic.nix

@@ -428,6 +428,7 @@ stdenv.mkDerivation (finalAttrs: {
   buildFlags = if atLeast17 then [ "images" ] else [ "all" ];
 
   separateDebugInfo = true;
+  __structuredAttrs = true;
 
   # -j flag is explicitly rejected by the build system:
   #     Error: 'make -jN' is not supported, use 'make JOBS=N'

pkgs/development/interpreters/python/cpython/default.nix

@@ -788,6 +788,7 @@ stdenv.mkDerivation (finalAttrs: {
     ];
 
   separateDebugInfo = true;
+  __structuredAttrs = true;
 
   passthru = passthru // {
     doc = stdenv.mkDerivation {

pkgs/development/libraries/mesa/default.nix

@@ -205,6 +205,7 @@ stdenv.mkDerivation {
   # Keep build-ids so drivers can use them for caching, etc.
   # Also some drivers segfault without this.
   separateDebugInfo = true;
+  __structuredAttrs = true;
 
   # Needed to discover llvm-config for cross
   preConfigure = ''

pkgs/os-specific/linux/systemd/default.nix

@@ -331,6 +331,7 @@ stdenv.mkDerivation (finalAttrs: {
     "dev"
   ] ++ (lib.optional (!buildLibsOnly) "man");
   separateDebugInfo = true;
+  __structuredAttrs = true;
 
   hardeningDisable =
     [

pkgs/stdenv/generic/make-derivation.nix

@@ -324,7 +324,21 @@ let
       doCheck' = doCheck && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
       doInstallCheck' = doInstallCheck && stdenv.buildPlatform.canExecute stdenv.hostPlatform;
 
-      separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux;
+      separateDebugInfo' =
+        let
+          actualValue = separateDebugInfo && stdenv.hostPlatform.isLinux;
+          conflictingOption =
+            attrs ? "disallowedReferences"
+            || attrs ? "disallowedRequisites"
+            || attrs ? "allowedRequisites"
+            || attrs ? "allowedReferences";
+        in
+        if actualValue && conflictingOption && !__structuredAttrs then
+          throw "separateDebugInfo = true in ${
+            attrs.pname or "mkDerivation argument"
+          } requires __structuredAttrs if {dis,}allowedRequisites or {dis,}allowedReferences is set"
+        else
+          actualValue;
       outputs' = outputs ++ optional separateDebugInfo' "debug";
 
       noNonNativeDeps =
@@ -646,10 +660,26 @@ let
                 outputChecks = builtins.listToAttrs (
                   map (name: {
                     inherit name;
-                    value = zipAttrsWith (_: builtins.concatLists) [
-                      (makeOutputChecks attrs)
-                      (makeOutputChecks attrs.outputChecks.${name} or { })
-                    ];
+                    value =
+                      let
+                        raw = zipAttrsWith (_: builtins.concatLists) [
+                          (makeOutputChecks attrs)
+                          (makeOutputChecks attrs.outputChecks.${name} or { })
+                        ];
+                      in
+                      # separateDebugInfo = true will put all sorts of files in
+                      # the debug output which could carry references, but
+                      # that's "normal". Notably it symlinks to the source.
+                      # So disable reference checking for the debug output
+                      if separateDebugInfo' && name == "debug" then
+                        removeAttrs raw [
+                          "allowedReferences"
+                          "allowedRequisites"
+                          "disallowedReferences"
+                          "disallowedRequisites"
+                        ]
+                      else
+                        raw;
                   }) outputs
                 );
               }

pkgs/tools/package-management/lix/common-lix.nix

@@ -128,6 +128,7 @@ stdenv.mkDerivation (finalAttrs: {
     # We don't want the underlying GCC neither!
     stdenv.cc.cc.stdenv.cc.cc
   ];
+  __structuredAttrs = true;
 
   # We only include CMake so that Meson can locate toml11, which only ships CMake dependency metadata.
   dontUseCmakeConfigure = true;