From c1d2b27e850cf2a942eecdcb7aab0a7952017497 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 28 Apr 2016 23:48:13 +0200 Subject: [PATCH 1/4] Just strip everything by default Run strip of each file and discard expected failure types. Also default to stripping the entire output. --- doc/stdenv.xml | 50 +++++++++++---- pkgs/build-support/setup-hooks/strip.sh | 81 +++++++++++++++++++------ 2 files changed, 100 insertions(+), 31 deletions(-) diff --git a/doc/stdenv.xml b/doc/stdenv.xml index 68441ea9393ab..cf17f36cb6589 100644 --- a/doc/stdenv.xml +++ b/doc/stdenv.xml @@ -846,8 +846,9 @@ following: subdirectories of $out to share/. - It strips libraries and executables of debug - information. + It strips libraries and executables of debug information. + This step is crucial to remove references to build inputs. It also reduces + the size of binaries. On Linux, it applies the patchelf command to ELF executables and libraries to remove unused @@ -869,8 +870,27 @@ following: dontStrip - If set, libraries and executables are not - stripped. By default, they are. + Stripping is required to avoid propagating build + inputs as run-time dependencies. Please ensure you understand the + implications before disabling stripping. To disable stripping on selected + files of folders, see dontStripList or + strip{Debug,All}List.If set, + libraries and executables are not stripped. By default, they are. + + + + + dontStripList + List of paths that should not be stripped. Bash patterns + are accepted, and are relative to the package's prefix. For example, + ["share" "modules/*.i" "bin/libannoying.so"] is an + acceptable value. By default it's empty. Avoid trailing slashes as the + underlying find ignores such paths. + Two good reasons to add paths to this list are when + strip fails on the file, breaking the build, and when + the package requires the debug info. In the latter case, you should ensure + that the unstripped executable does not maintain reference to build inputs. + @@ -881,11 +901,12 @@ following: stripAllList - List of directories to search for libraries and - executables from which all symbols should be - stripped. By default, it’s empty. Stripping all symbols is - risky, since it may remove not just debug symbols but also ELF - information necessary for normal execution. + List of paths to search for libraries and executables from + which all symbols should be stripped. See + dontStripList for detailed information about accepted + values. By default, it’s empty. Stripping all symbols is risky, since it + may remove not just debug symbols but also ELF information necessary for + normal execution. @@ -898,10 +919,13 @@ following: stripDebugList - List of directories to search for libraries and - executables from which only debugging-related symbols should be - stripped. It defaults to lib bin - sbin. + List of paths to search for libraries and executables from + which only debugging-related symbols should be stripped. See + dontStripList for detailed information about accepted + values. By default, everything is stripped. The same warning as for + dontStrip applies here. For most purposes, + dontStripList should be used in place of this option. + diff --git a/pkgs/build-support/setup-hooks/strip.sh b/pkgs/build-support/setup-hooks/strip.sh index 6860c9b9cb9ad..4d5324316174d 100644 --- a/pkgs/build-support/setup-hooks/strip.sh +++ b/pkgs/build-support/setup-hooks/strip.sh @@ -4,33 +4,78 @@ fixupOutputHooks+=(_doStrip) _doStrip() { if [ -z "$dontStrip" ]; then - stripDebugList=${stripDebugList:-lib lib32 lib64 libexec bin sbin} - if [ -n "$stripDebugList" ]; then - stripDirs "$stripDebugList" "${stripDebugFlags:--S}" - fi - stripAllList=${stripAllList:-} - if [ -n "$stripAllList" ]; then - stripDirs "$stripAllList" "${stripAllFlags:--s}" + stripDirs "${stripDebugList:-*}" "${stripDebugFlags:--S}" + + if [ -n "${stripAllList:-}" ]; then + stripDirs "${stripAllList:-}" "${stripAllFlags:--s}" fi fi } stripDirs() { - local dirs="$1" local stripFlags="$2" - local dirsNew= - for d in ${dirs}; do - if [ -d "$prefix/$d" ]; then - dirsNew="${dirsNew} $prefix/$d " + # Ensure $prefix is a directory + if [ -f "$prefix" ]; then + header "Stripping (with flags $stripFlags) the file ${prefix}." + echo -ne $prefix'\0' | stripAll "$stripFlags" + stopNest + fi + [ -d "$prefix" ] || return 0 + + # Ensure expansion occurs within $prefix/. + pushd "$prefix" >/dev/null + shopt -s dotglob + local stripDirs=( $1 ) + local ignoreDirs=( $dontStripList ) + shopt -u dotglob + popd >/dev/null + local debugIncluded= debugExcluded= p= + + # `find` fails on missing paths. Remove them. + local roots=() + for p in "${stripDirs[@]}"; do + if [ -e "$prefix/$p" ]; then + roots+=( "$prefix/$p" ) + debugIncluded="$debugIncluded${debugIncluded:+,}$p" fi done - dirs=${dirsNew} + [ "${#roots[@]}" -eq 0 ] && return 0 - if [ -n "${dirs}" ]; then - header "stripping (with flags $stripFlags) in$dirs" - find $dirs -type f -print0 | xargs -0 ${xargsFlags:--r} strip $commonStripFlags $stripFlags || true - stopNest - fi + # Add ignored paths to find command. + local findArgs=( "${roots[@]}" ) + for p in "${ignoreDirs[@]}"; do + if [ -e "$prefix/$p" ]; then + findArgs+=( "-path" "$prefix/$p" "-prune" "-o" ) + debugExcluded="$debugExcluded $p" + fi + done + + header "Stripping (with flags $stripFlags) in $prefix${debugIncluded:+"/{$debugIncluded}"}${debugExcluded:+", skipping$debugExcluded"}." + find "${findArgs[@]}" -type f -print0 | stripAll "$stripFlags" + stopNest +} + +stripAll() { + local stripFlags="$1" + + local _ls f stripOutput + while IFS= read -r -d $'\0' f; do + # Skip very small files, they make strip choke. + _ls=( $(ls -Ln "$f") ) + [ "${_ls[4]}" -lt 4 ] && continue; + + if stripOutput=$(strip $commonStripFlags $stripFlags "$f" 2>&1); then + echo "Stripped $f" + else + # Ignore failures on files that cannot be stripped. + [ "$stripOutput" = "strip:$f: File format not recognized" ] && continue + [ "$stripOutput" = "strip:$f: File truncated" ] && continue + + [ -n "$stripOutput" ] && echo "$stripOutput" + echo "Strip failed on file $f" + false # fail ! + fi + done } From 6dcb5abbb40512f5be72a77c71d7453f2a333bee Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 6 May 2016 00:44:48 +0200 Subject: [PATCH 2/4] swig: stripping share/ fails, don't --- pkgs/development/tools/misc/swig/2.x.nix | 3 +++ pkgs/development/tools/misc/swig/3.x.nix | 3 +++ pkgs/development/tools/misc/swig/default.nix | 3 +++ 3 files changed, 9 insertions(+) diff --git a/pkgs/development/tools/misc/swig/2.x.nix b/pkgs/development/tools/misc/swig/2.x.nix index 48ba12ccd9c95..6731b86eb11b7 100644 --- a/pkgs/development/tools/misc/swig/2.x.nix +++ b/pkgs/development/tools/misc/swig/2.x.nix @@ -23,6 +23,9 @@ stdenv.mkDerivation rec { ./autogen.sh ''; + # Some files in share look like malformed executables. + dontStripList = [ "share/swig" ]; + meta = { description = "SWIG, an interface compiler that connects C/C++ code to higher-level languages"; diff --git a/pkgs/development/tools/misc/swig/3.x.nix b/pkgs/development/tools/misc/swig/3.x.nix index 2d5358b2255f1..8ccef2b379d30 100644 --- a/pkgs/development/tools/misc/swig/3.x.nix +++ b/pkgs/development/tools/misc/swig/3.x.nix @@ -23,6 +23,9 @@ stdenv.mkDerivation rec { ./autogen.sh ''; + # Some files in share look like malformed executables. + dontStripList = [ "share/swig" ]; + meta = with stdenv.lib; { description = "SWIG, an interface compiler that connects C/C++ code to higher-level languages"; homepage = http://swig.org/; diff --git a/pkgs/development/tools/misc/swig/default.nix b/pkgs/development/tools/misc/swig/default.nix index e10fd25d046f0..0f055f1f82ea4 100644 --- a/pkgs/development/tools/misc/swig/default.nix +++ b/pkgs/development/tools/misc/swig/default.nix @@ -14,6 +14,9 @@ stdenv.mkDerivation rec { configureFlags = "--disable-ccache"; + # Some files in share look like malformed executables. + dontStripList = [ "share/swig" ]; + meta = { description = "Interface compiler that connects C/C++ code to higher-level languages"; From 0c718edadc306d4e34a77c89bc3489287d683928 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 6 May 2016 01:13:23 +0200 Subject: [PATCH 3/4] Remove superfluous stripDebugList --- .../misc/avr-gcc-with-avr-libc/default.nix | 12 ++++++++++-- pkgs/os-specific/linux/syslinux/default.nix | 2 -- pkgs/servers/http/apache-httpd/2.2.nix | 2 -- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkgs/development/misc/avr-gcc-with-avr-libc/default.nix b/pkgs/development/misc/avr-gcc-with-avr-libc/default.nix index 237c4e4027f19..b2b0f131b01e6 100644 --- a/pkgs/development/misc/avr-gcc-with-avr-libc/default.nix +++ b/pkgs/development/misc/avr-gcc-with-avr-libc/default.nix @@ -28,8 +28,10 @@ stdenv.mkDerivation { hardeningDisable = [ "format" ]; - # Make sure we don't strip the libraries in lib/gcc/avr. - stripDebugList= [ "bin" "avr/bin" "libexec" ]; + dontStripList = [ + "lib/gcc/avr" # Stripping here makes the whole toolchain unstable + "avr/lib" # Avr binaries need to be stripped with avr-strip; see below. + ]; installPhase = '' # important, without this gcc won't find the binutils executables @@ -67,6 +69,12 @@ stdenv.mkDerivation { popd ''; + postFixup = '' + # Strip avr/lib with avr-strip + alias strip=$out/bin/avr-strip + dontStripList= stripDirs "avr/strip" "''${stripDebugFlags:--S}" + ''; + meta = with stdenv.lib; { description = "AVR development environment including binutils, avr-gcc and avr-libc"; # I've tried compiling the packages separately.. too much hassle. This just works. Fine. diff --git a/pkgs/os-specific/linux/syslinux/default.nix b/pkgs/os-specific/linux/syslinux/default.nix index f4ad94b5085ce..3a16613669ab8 100644 --- a/pkgs/os-specific/linux/syslinux/default.nix +++ b/pkgs/os-specific/linux/syslinux/default.nix @@ -24,8 +24,6 @@ stdenv.mkDerivation rec { substituteInPlace utils/ppmtolss16 gpxe/src/Makefile --replace /usr/bin/perl $(type -P perl) ''; - stripDebugList = "bin sbin share/syslinux/com32"; - makeFlags = [ "BINDIR=$(out)/bin" "SBINDIR=$(out)/sbin" diff --git a/pkgs/servers/http/apache-httpd/2.2.nix b/pkgs/servers/http/apache-httpd/2.2.nix index 8cab241f85c0f..f2115017f8a96 100644 --- a/pkgs/servers/http/apache-httpd/2.2.nix +++ b/pkgs/servers/http/apache-httpd/2.2.nix @@ -56,8 +56,6 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; - stripDebugList = "lib modules bin"; - postInstall = '' mkdir -p $doc/share/doc/httpd mv $out/manual $doc/share/doc/httpd From 1f93a82dc365839a89e3a0817b5b313088bc415a Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 6 May 2016 00:43:54 +0200 Subject: [PATCH 4/4] go: do not strip upstream elf test data --- pkgs/development/compilers/go/1.4.nix | 2 ++ pkgs/development/compilers/go/1.6.nix | 2 ++ pkgs/development/compilers/go/1.7.nix | 2 ++ 3 files changed, 6 insertions(+) diff --git a/pkgs/development/compilers/go/1.4.nix b/pkgs/development/compilers/go/1.4.nix index eb4c64ed33406..5c6dd0d2562ba 100644 --- a/pkgs/development/compilers/go/1.4.nix +++ b/pkgs/development/compilers/go/1.4.nix @@ -118,6 +118,8 @@ stdenv.mkDerivation rec { setupHook = ./setup-hook.sh; + dontStripList = [ "share/go/src/debug/elf/testdata" ]; + meta = with stdenv.lib; { branch = "1.4"; homepage = http://golang.org/; diff --git a/pkgs/development/compilers/go/1.6.nix b/pkgs/development/compilers/go/1.6.nix index 982446f4fdb1c..16ee3a028cb47 100644 --- a/pkgs/development/compilers/go/1.6.nix +++ b/pkgs/development/compilers/go/1.6.nix @@ -146,6 +146,8 @@ stdenv.mkDerivation rec { setupHook = ./setup-hook.sh; + dontStripList = [ "share/go/src/debug/elf/testdata" ]; + disallowedReferences = [ go_bootstrap ]; meta = with stdenv.lib; { diff --git a/pkgs/development/compilers/go/1.7.nix b/pkgs/development/compilers/go/1.7.nix index 0df2e8d6f7e9e..be39cd69fa6e1 100644 --- a/pkgs/development/compilers/go/1.7.nix +++ b/pkgs/development/compilers/go/1.7.nix @@ -149,6 +149,8 @@ stdenv.mkDerivation rec { setupHook = ./setup-hook.sh; + dontStripList = [ "share/go/src/debug/elf/testdata" ]; + disallowedReferences = [ go_bootstrap ]; meta = with stdenv.lib; {