Preview: deterministic build

nixos/doc/manual/default.nix

@@ -55,7 +55,7 @@ in rec {
 
     inherit sources;
 
-    buildInputs = [ libxml2 libxslt ];
+    buildInputs = [ libxml2 libxslt perl ];
 
     buildCommand = ''
       ${copySources}
@@ -83,6 +83,21 @@ in rec {
         --nonet --xinclude --output $dst/ \
         ${docbook5_xsl}/xml/xsl/docbook/xhtml/chunkfast.xsl ./manual.xml
 
+      # Fix the non-deterministic id-generation used by xsltproc
+      # !!! Move this somewhere else
+      perl -0777 -pi -e '
+          # xsltproc html output id remapping
+          # pretty weird that xsltproc cannot do this
+          # Author: Alexander Kjeldaas <ak@formalprivacy.com>
+          my @parts = split(/(href="#|id="|href="#ftn.|id="ftn.)(id.\d+")/g, $_);
+          my %remap = {};
+          for (my $i = 0; $i < @parts; $i += 3) {
+              my $id = @parts[ $i+2 ];
+              $remap{$id} = $i unless exists $remap{$id};
+              @parts[ $i+2 ] = "idp$remap{$id}\"";
+          }
+          $_ = join "", @parts; ' $dst/manual.html
+
       mkdir -p $dst/images/callouts
       cp ${docbook5_xsl}/xml/xsl/docbook/images/callouts/*.gif $dst/images/callouts/
 

nixos/lib/make-system-tarball.sh

@@ -50,7 +50,12 @@ done
 
 mkdir -p $out/tarball
 
-tar cvJf $out/tarball/$fileName.tar.xz * $extraArgs
+
+rm ./env-vars
+
+find * ! -type d -print0 | sort -z |
+  tar -cv --mtime='1970-01-01' -T- --null -f- $extraArgs |
+  xz -c > $out/tarball/$fileName.tar.xz
 
 mkdir -p $out/nix-support
 echo $system > $out/nix-support/system

nixos/modules/system/boot/stage-1.nix

@@ -298,7 +298,7 @@ in
 
     boot.initrd.compressor = mkOption {
       internal = true;
-      default = "gzip -9";
+      default = "gzip -9n";
       type = types.str;
       description = "The compressor to use on the initrd image.";
       example = "xz";

pkgs/build-support/gcc-wrapper/gcc-wrapper.sh

@@ -85,22 +85,32 @@ fi
 extraAfter=($NIX_CFLAGS_COMPILE)
 extraBefore=()
 
+# When enforcing purity, pretend gcc can't find the current date and
+# time
+if test "$NIX_ENFORCE_PURITY" = "1"; then
+    extraBefore=(-D__DATE__=\"Jan\ \ 1\ 1970\"
+	-D__TIME__=\"00:00:01\"
+        -Wno-builtin-macro-redefined
+	"${extraBefore[@]}")
+fi
+
+
 if test "$dontLink" != "1"; then
 
     # Add the flags that should only be passed to the compiler when
     # linking.
-    extraAfter=(${extraAfter[@]} $NIX_CFLAGS_LINK)
+    extraAfter=("${extraAfter[@]}" $NIX_CFLAGS_LINK)
 
     # Add the flags that should be passed to the linker (and prevent
     # `ld-wrapper' from adding NIX_LDFLAGS again).
     for i in $NIX_LDFLAGS_BEFORE; do
-        extraBefore=(${extraBefore[@]} "-Wl,$i")
+        extraBefore=("${extraBefore[@]}" "-Wl,$i")
     done
     for i in $NIX_LDFLAGS; do
 	if test "${i:0:3}" = "-L/"; then
-	    extraAfter=(${extraAfter[@]} "$i")
+	    extraAfter=("${extraAfter[@]}" "$i")
 	else
-	    extraAfter=(${extraAfter[@]} "-Wl,$i")
+	    extraAfter=("${extraAfter[@]}" "-Wl,$i")
 	fi
     done
     export NIX_LDFLAGS_SET=1
@@ -122,11 +132,11 @@ if test "$NIX_DEBUG" = "1"; then
       echo "  $i" >&2
   done
   echo "extraBefore flags to @gccProg@:" >&2
-  for i in ${extraBefore[@]}; do
+  for i in "${extraBefore[@]}"; do
       echo "  $i" >&2
   done
   echo "extraAfter flags to @gccProg@:" >&2
-  for i in ${extraAfter[@]}; do
+  for i in "${extraAfter[@]}"; do
       echo "  $i" >&2
   done
 fi
@@ -140,9 +150,9 @@ fi
 # `-B' flags, since they confuse some programs.  Deep bash magic to
 # apply grep to stderr (by swapping stdin/stderr twice).
 if test -z "$NIX_GCC_NEEDS_GREP"; then
-    @gccProg@ ${extraBefore[@]} "${params[@]}" ${extraAfter[@]}
+    @gccProg@ "${extraBefore[@]}" "${params[@]}" "${extraAfter[@]}"
 else
-    (@gccProg@ ${extraBefore[@]} "${params[@]}" ${extraAfter[@]} 3>&2 2>&1 1>&3- \
+    (@gccProg@ "${extraBefore[@]}" "${params[@]}" "${extraAfter[@]}" 3>&2 2>&1 1>&3- \
         | (grep -v 'file path prefix' || true); exit ${PIPESTATUS[0]}) 3>&2 2>&1 1>&3-
     exit $?
 fi    

pkgs/build-support/kernel/cpio-clean.pl

@@ -9,8 +9,11 @@
 my $cpio = Archive::Cpio->new;
 my $IN = \*STDIN;
 my $ino = 1;
+my %ino_remap = {};
 $cpio->read_with_handler($IN, sub {
         my ($e) = @_;
+        $ino_remap{$e->{inode}} = $ino++ unless exists $ino_remap{$e->{inode}};
+        $e->{inode} = $ino_remap{$e->{inode}};
         $e->{mtime} = 1;
 	$cpio->write_one(\*STDOUT, $e);
     });

pkgs/build-support/kernel/make-initrd.sh

@@ -36,7 +36,11 @@ storePaths=$(perl $pathsFromGraph closure-*)
 
 # Put the closure in a gzipped cpio archive.
 mkdir -p $out
-(cd root && find * -print0 | cpio -o -H newc --null | perl $cpioClean | $compressor > $out/initrd)
+(cd root && find * -print0 | 
+ sort -z |
+ cpio -o -H newc -R 0.0 --null |
+ perl $cpioClean |
+ $compressor > $out/initrd)
 
 if [ -n "$makeUInitrd" ]; then
     mv $out/initrd $out/initrd.gz

pkgs/development/compilers/qcmm/builder.sh

@@ -16,7 +16,7 @@ installPhase() {
     mv $file ${file%.opt}
   done
 
-  find $out/man -type f -exec gzip -9 {} \;
+  find $out/man -type f -exec gzip -9n {} \;
 
   find $out -name \*.a -exec echo stripping {} \; \
             -exec strip -S {} \;

pkgs/development/interpreters/perl/5.16/default.nix

@@ -21,6 +21,9 @@ stdenv.mkDerivation rec {
   patches =
     [ # Do not look in /usr etc. for dependencies.
       ./no-sys-dirs.patch
+      ./no-impure-config-time.patch
+      ./fixed-man-page-date.patch
+      ./no-date-in-perl-binary.patch
     ]
     ++ optional stdenv.isSunOS  ./ld-shared.patch
     ++ stdenv.lib.optional stdenv.isDarwin [ ./cpp-precomp.patch ./no-libutil.patch ] ;
@@ -64,5 +67,31 @@ stdenv.mkDerivation rec {
 
   setupHook = ./setup-hook.sh;
 
+  doCheck = !stdenv.isDarwin;
+
+  # some network-related tests don't work, mostly probably due to our sandboxing
+  # man-heading.t is skipped due to output determinism (no dates)
+  testsToSkip = ''
+    lib/Net/hostent.t \
+    dist/IO/t/{io_multihomed.t,io_sock.t} \
+    dist/Net-Ping/t/*.t \
+    cpan/autodie/t/truncate.t \
+    t/porting/{maintainers.t,regen.t} \
+    cpan/Socket/t/get{name,addr}info.t \
+    cpan/podlators/t/man-heading.t \
+  '' + optionalString stdenv.isFreeBSD ''
+    cpan/CPANPLUS/t/04_CPANPLUS-Module.t \
+    cpan/CPANPLUS/t/20_CPANPLUS-Dist-MM.t \
+  '' + " ";
+
+  postPatch = optionalString (!stdenv.isDarwin) /* this failed on Darwin, no idea why */ ''
+    for test in ${testsToSkip}; do
+      echo "Removing test" $test
+      rm "$test"
+      pat=`echo "$test" | sed 's,/,\\\\/,g'` # just escape slashes
+      sed "/^$pat/d" -i MANIFEST
+    done
+  '';
+
   passthru.libPrefix = "lib/perl5/site_perl";
 }

pkgs/development/interpreters/perl/5.16/fixed-man-page-date.patch

@@ -0,0 +1,11 @@
+--- a/cpan/podlators/lib/Pod/Man.pm	2014-04-07 06:25:23.730505243 +0200
++++ b/cpan/podlators/lib/Pod/Man.pm	2014-04-07 06:26:40.816552603 +0200
+@@ -768,7 +768,7 @@
+     } else {
+         ($name, $section) = $self->devise_title;
+     }
+-    my $date = $$self{date} || $self->devise_date;
++    my $date = "1970-01-01"; # Fixed date for NixOS, orig: $$self{date} || $self->devise_date;
+     $self->preamble ($name, $section, $date)
+         unless $self->bare_output or DEBUG > 9;
+

pkgs/development/interpreters/perl/5.16/no-date-in-perl-binary.patch

@@ -0,0 +1,11 @@
+--- a/perl.c	2014-04-07 07:58:01.402831615 +0200
++++ b/perl.c	2014-04-07 07:59:38.556945298 +0200
+@@ -1754,7 +1754,7 @@
+     PUSHs(Perl_newSVpvn_flags(aTHX_ non_bincompat_options,
+ 			      sizeof(non_bincompat_options) - 1, SVs_TEMP));
+
+-#ifdef __DATE__
++#if 0
+ #  ifdef __TIME__
+     PUSHs(Perl_newSVpvn_flags(aTHX_
+ 			      STR_WITH_LEN("Compiled at " __DATE__ " " __TIME__),

pkgs/development/interpreters/perl/5.16/no-impure-config-time.patch

@@ -0,0 +1,11 @@
+--- a/Configure	2014-04-05 20:21:33.714635700 +0200
++++ b/Configure	2014-04-05 20:23:23.377441026 +0200
+@@ -3609,6 +3609,8 @@
+
+ : who configured the system
+ cf_time=`LC_ALL=C; LANGUAGE=C; export LC_ALL; export LANGUAGE; $date 2>&1`
++cf_time='Thu Jan  1 00:00:01 UTC 1970'
++
+ case "$cf_by" in
+ "")
+ 	cf_by=`(logname) 2>/dev/null`

pkgs/development/interpreters/python/2.7/default.nix

@@ -19,6 +19,9 @@ let
     [ # Look in C_INCLUDE_PATH and LIBRARY_PATH for stuff.
       ./search-path.patch
 
+      # No time in the bdist wininst
+      ./no-time-wininst.patch
+
       # Python recompiles a Python if the mtime stored *in* the
       # pyc/pyo file differs from the mtime of the source file.  This
       # doesn't work in Nix because Nix changes the mtime of files in
@@ -72,7 +75,8 @@ let
       '';
 
     NIX_CFLAGS_COMPILE = optionalString stdenv.isDarwin "-msse2";
-
+    DETERMINISTIC_BUILD = 1;
+    useFakeTime = 1;
     setupHook = ./setup-hook.sh;
 
     postInstall =
@@ -83,6 +87,11 @@ let
         ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
 
         paxmark E $out/bin/python${majorVersion}
+        # !!! This is a stopgap measure for getting deterministic builds.
+        # It disables creation of windows installers and the lib2to3 which
+        # can rewrite python2-programs to python3.
+        rm $out/lib/python${majorVersion}/distutils/command/wininst-*.exe
+        rm $out/lib/python${majorVersion}/lib2to3/Grammar2.7.7.final.0.pickle
       '';
 
     passthru = rec {

pkgs/development/interpreters/python/2.7/no-time-wininst.patch

@@ -0,0 +1,12 @@
+diff -ur Python-2.7.6.orig/Lib/distutils/command/bdist_wininst.py Python-2.7.6/Lib/distutils/command/bdist_wininst.py
+--- Python-2.7.6.orig/Lib/distutils/command/bdist_wininst.py	2013-11-10 08:36:40.000000000 +0100
++++ Python-2.7.6/Lib/distutils/command/bdist_wininst.py	2014-04-11 14:49:49.789235982 +0200
+@@ -245,7 +245,7 @@
+         import time
+         import distutils
+         build_info = "Built %s with distutils-%s" % \
+-                     (time.ctime(time.time()), distutils.__version__)
++                     (time.ctime(0), distutils.__version__)
+         lines.append("build_info=%s" % build_info)
+         return string.join(lines, "\n")
+

pkgs/development/libraries/apr-util/default.nix

@@ -35,6 +35,11 @@ stdenv.mkDerivation rec {
 
   # Give apr1 access to sed for runtime invocations
   postInstall = ''
+    # Determinism changes
+    sed -i -e 's/APU_SOURCE_DIR=".*"/APU_SOURCE_DIR="unknown"/g' \
+           -e 's/APU_BUILD_DIR=".*"/APU_BUILD_DIR="unknown"/g' \
+           $out/bin/apu-1-config
+
     wrapProgram $out/bin/apu-1-config --prefix PATH : "${gnused}/bin"
   '';
 

pkgs/development/libraries/atomic-ops/default.nix

@@ -0,0 +1,28 @@
+{stdenv, fetchgit, autoconf, automake, libtool}:
+
+stdenv.mkDerivation rec {
+  baseName = "atomic-ops";
+  version = "7.4.0";
+  name="${baseName}-${version}";
+
+  buildInputs = [ autoconf automake libtool ];
+
+  preConfigure = ''
+    ./autogen.sh
+  '';
+
+  src = fetchgit {
+    url = "https://github.com/ivmai/libatomic_ops";
+    rev = "a5df11ab031f7541442bac387e2ec5b6c88d8600";
+    sha256 = "0ij9i0m9lq7ipx5mbp0qpr1y95zpgiv8cp11d46sss2fs1jkj1i3";
+  };
+
+  meta = {
+    homepage = "https://github.com/ivmai/libatomic_ops";
+    description = ''
+        This package provides semi-portable access to hardware-provided
+        atomic memory update operations on a number architectures.'';
+    platforms = stdenv.lib.platforms.linux ++ stdenv.lib.platforms.darwin;
+    maintainers = [ stdenv.lib.maintainers.ak ];
+  };
+}

pkgs/development/libraries/glib/default.nix

@@ -100,6 +100,7 @@ stdenv.mkDerivation rec {
     '';
 
   postInstall = ''rm -rvf $out/share/gtk-doc'';
+  DETERMINISTIC_BUILD = 1;
 
   passthru = {
      gioModuleDir = "lib/gio/modules";

pkgs/development/libraries/glibc/2.19/common.nix

@@ -60,6 +60,14 @@ stdenv.mkDerivation ({
       ./fix-math.patch
 
       ./cve-2014-0475.patch
+
+      /* Remove references to the compilation date.  */
+      ./glibc-remove-date-from-compilation-banner.patch
+
+      /* Remove the date and time from nscd.  It is used as a protocol
+         compatibility check, but we assume nix takes care of that for
+         us. */
+      ./glibc-remove-datetime-from-nscd.patch
     ];
 
   postPatch = ''

pkgs/development/libraries/glibc/2.19/glibc-remove-date-from-compilation-banner.patch

@@ -0,0 +1,12 @@
+diff -ur glibc-2.17.orig/csu/Makefile glibc-2.17/csu/Makefile
+--- glibc-2.17.orig/csu/Makefile	2012-12-25 04:02:13.000000000 +0100
++++ glibc-2.17/csu/Makefile	2013-08-19 16:01:57.132378550 +0200
+@@ -172,7 +172,7 @@
+ 		     os=Linux; \
+ 		   fi; \
+ 		   printf '"Compiled on a %s %s system on %s.\\n"\n' \
+-			  "$$os" "$$version" "`date +%Y-%m-%d`";; \
++			  "$$os" "$$version";; \
+ 	   *) ;; \
+ 	 esac; \
+ 	 files="$(all-Banner-files)";				\

pkgs/development/libraries/glibc/2.19/glibc-remove-datetime-from-nscd.patch

@@ -0,0 +1,11 @@
+--- a/nscd/nscd_stat.c	2014-04-08 20:35:24.253715420 +0200
++++ b/nscd/nscd_stat.c	2014-04-08 20:38:32.526634400 +0200
+@@ -37,7 +37,7 @@
+
+
+ /* We use this to make sure the receiver is the same.  */
+-static const char compilation[21] = __DATE__ " " __TIME__;
++static const char compilation[21] = "Thu  1 1970 00:00:01"; /* __DATE__ " " __TIME__; */
+
+ /* Statistic data for one database.  */
+ struct dbstat

pkgs/development/libraries/icu/default.nix

@@ -34,6 +34,7 @@ stdenv.mkDerivation {
     stdenv.lib.optionalString stdenv.isDarwin " --enable-rpath";
 
   enableParallelBuilding = true;
+  useFakeTime = 1;
 
   meta = {
     description = "Unicode and globalization support library";

pkgs/development/libraries/libfaketime/avoid-spurious-lrt.patch

@@ -0,0 +1,12 @@
+diff -ur libfaketime-0.9.5.orig/src/Makefile libfaketime-0.9.5/src/Makefile
+--- libfaketime-0.9.5.orig/src/Makefile	2013-10-13 11:19:30.000000000 +0200
++++ libfaketime-0.9.5/src/Makefile	2014-04-11 21:58:06.285435083 +0200
+@@ -69,7 +69,7 @@
+
+ CFLAGS += -std=gnu99 -Wall -Wextra -Werror -DFAKE_STAT -DFAKE_SLEEP -DFAKE_TIMERS -DFAKE_INTERNAL_CALLS -fPIC -DPREFIX='"'$(PREFIX)'"' -DLIBDIRNAME='"'$(LIBDIRNAME)'"'
+ LIB_LDFLAGS += -shared
+-LDFLAGS += -Wl,--version-script=libfaketime.map -lrt
++LDFLAGS += -Wl,--version-script=libfaketime.map 
+ LDADD += -ldl -lm -lpthread -lrt
+
+ SRC = libfaketime.c