NixOS · MarcWeber · Sep 14, 2013
diff --git a/pkgs/tools/backup/bup/default.nix b/pkgs/tools/backup/bup/default.nix
@@ -1,34 +1,149 @@
 { stdenv, fetchgit, python, pyxattr, pylibacl, setuptools, fuse, git, perl, pandoc, makeWrapper
+, diffutils, writeTextFile, rsync
+, versionedDerivation
+, # Be careful when updating default, git-annex depends on it! (ask Peter!)
+  version ? "bup-0.25-rc1-107-g96c6fa2"
 , par2cmdline, par2Support ? false }:
 
+# - Keep in mind you cannot prune older revisions yet! (2013-06)
+# - Caution! with "bup-0.25-rc1-107-g96c6fa2" Mathijs Kwik has had trouble
+#   causing him to abondon it (see mailinglist)
+
 assert par2Support -> par2cmdline != null;
 
 with stdenv.lib;
 
-stdenv.mkDerivation {
-  name = "bup-0.25-rc1-107-g96c6fa2";
+let
+
+  extraPythonPath = stdenv.lib.concatStringsSep ":"
+          (map (path: "$(toPythonPath ${path})") [ pyxattr pylibacl setuptools fuse ]);
+
+  nix_backup_test = ''
+    # if make test passes the following probably passes, too
+    backup_init(){
+      export BUP_DIR=$TMP/bup
+      PATH=$out/bin:$PATH
+      bup init
+    }
+    backup_make(){
+      ( cd "$1"; tar -cvf - .) | bup split -n backup
+    }
+    backup_restore_latest(){
+      bup join backup | ( cd "$1"; tar -xf - )
+    }
+    backup_verify_integrity_latest(){
+      bup fsck
+    }
+    backup_verify_latest(){
+      # maybe closest would be to mount or use the FTP like server ..
+      true
+    }
+
+    . ${import ../test-case.nix { inherit diffutils writeTextFile; }}
+    backup_test backup 100M
+  '';
+
+in
+
+versionedDerivation "bup" version {
+  "bup-0.25-rc1-107-g96c6fa2" = {
+
+    name = "bup-0.25-rc1-107-g96c6fa2";
+
+    src = fetchgit {
+      url = "https://github.com/bup/bup.git";
+      rev = "96c6fa2a70425fff1e73d2e0945f8e242411ab58";
+      sha256 = "0d9hgyh1g5qcpdvnqv3a5zy67x79yx9qx557rxrnxyzqckp9v75n";
+    };
+
+    patchPhase = ''
+      substituteInPlace Makefile --replace "-Werror" ""
+      for f in "cmd/"* "lib/tornado/"* "lib/tornado/test/"* "t/"* wvtest.py main.py; do
+        test -f $f || continue
+        substituteInPlace $f --replace "/usr/bin/env python" "${python}/bin/python"
+      done
+      substituteInPlace Makefile --replace "./format-subst.pl" "perl ./format-subst.pl"
+      for t in t/*.sh t/compare-trees; do
+        substituteInPlace $t --replace "/usr/bin/env bash" "$(type -p bash)"
+      done
+
+      substituteInPlace wvtestrun --replace "/usr/bin/perl" "${perl}/bin/perl"
+
+      substituteInPlace t/test.sh --replace "/bin/pwd" "$(type -P pwd)"
+    '' + optionalString par2Support ''
+      substituteInPlace cmd/fsck-cmd.py --replace "['par2'" "['${par2cmdline}/bin/par2'"
+    '';
+
+
+    dontAddPrefix = true;
+
+    postInstall = optionalString (elem stdenv.system platforms.linux) ''
+      wrapProgram $out/bin/bup --prefix PYTHONPATH : \
+        ${stdenv.lib.concatStringsSep ":"
+            (map (path: "$(toPythonPath ${path})") [ pyxattr pylibacl setuptools fuse ])}
+
+      ## test it
+
+      export PYTHONPATH=$PYTHONPATH${PYTHONPATH:+:}${extraPythonPath}}
 
-  src = fetchgit {
-    url = "https://github.com/bup/bup.git";
-    rev = "96c6fa2a70425fff1e73d2e0945f8e242411ab58";
-    sha256 = "0d9hgyh1g5qcpdvnqv3a5zy67x79yx9qx557rxrnxyzqckp9v75n";
+      make test
+
+      ${nix_backup_test}
+    '';
+
+  };
+
+  "latest" = {
+    name = "bup-0.25-rc1-107-g96c6fa2";
+
+    src = fetchgit {
+      url = "https://github.com/bup/bup.git";
+      rev = "98a8e2ebb775386cb7e66b1953df46cdbd4b4bd3";
+      sha256 = "ab01c70f0caf993c0c05ec3a1008b5940b433bf2f7bd4e9b995d85e81958c1b7";
+    };
+
+    enableParallelBuilding = true;
+
+    patchPhase = ''
+      substituteInPlace Makefile --replace "-Werror" ""
+      for f in "cmd/"* "lib/tornado/"* "lib/tornado/test/"* "t/"* wvtest.py main.py; do
+        test -f $f || continue
+        substituteInPlace $f --replace "/usr/bin/env python" "${python}/bin/python"
+      done
+      substituteInPlace Makefile --replace "./format-subst.pl" "perl ./format-subst.pl"
+      for t in t/*.sh t/configure-sampledata t/compare-trees; do
+        substituteInPlace $t --replace "/usr/bin/env bash" "$(type -p bash)"
+      done
+      substituteInPlace wvtestrun --replace "/usr/bin/env perl" "${perl}/bin/perl"
+
+      substituteInPlace t/test.sh --replace "/bin/pwd" "$(type -P pwd)"
+    '' + optionalString par2Support ''
+      substituteInPlace cmd/fsck-cmd.py --replace "['par2'" "['${par2cmdline}/bin/par2'"
+    '';
+
+    dontAddPrefix = true;
+
+    # make test failed on hydra once but worked on 3 different machines (?) ..
+    # retrying needed
+    postInstall = optionalString (elem stdenv.system platforms.linux) ''
+      wrapProgram $out/bin/bup --prefix PYTHONPATH : \
+        ${extraPythonPath}
+
+      ## test it
+      make test
+
+      ${nix_backup_test}
+    '';
   };
+} {
 
   buildInputs = [ python git ];
-  nativeBuildInputs = [ pandoc perl makeWrapper ];
-
-  patchPhase = ''
-    substituteInPlace Makefile --replace "-Werror" ""
-    for f in "cmd/"* "lib/tornado/"* "lib/tornado/test/"* "t/"* wvtest.py main.py; do
-      test -f $f || continue
-      substituteInPlace $f --replace "/usr/bin/env python" "${python}/bin/python"
-    done
-    substituteInPlace Makefile --replace "./format-subst.pl" "perl ./format-subst.pl"
-  '' + optionalString par2Support ''
-    substituteInPlace cmd/fsck-cmd.py --replace "['par2'" "['${par2cmdline}/bin/par2'"
-  '';
+  nativeBuildInputs = [
+    pandoc perl makeWrapper
 
-  dontAddPrefix = true;
+    # used by test suite
+    rsync
+  ];
 
   makeFlags = [
     "MANDIR=$(out)/share/man"
@@ -37,11 +152,6 @@ stdenv.mkDerivation {
     "LIBDIR=$(out)/lib/bup"
   ];
 
-  postInstall = optionalString (elem stdenv.system platforms.linux) ''
-    wrapProgram $out/bin/bup --prefix PYTHONPATH : \
-      ${stdenv.lib.concatStringsSep ":"
-          (map (path: "$(toPythonPath ${path})") [ pyxattr pylibacl setuptools fuse ])}
-  '';
 
   meta = {
     homepage = "https://github.com/bup/bup";

diff --git a/pkgs/tools/backup/store-backup/default.nix b/pkgs/tools/backup/store-backup/default.nix
@@ -1,4 +1,4 @@
-{stdenv, which, coreutils, perl, fetchurl, perlPackages, makeWrapper, diffutils , writeScriptBin, bzip2}:
+{stdenv, which, coreutils, perl, fetchurl, perlPackages, makeWrapper, diffutils , writeScriptBin, writeTextFile, bzip2}:
 
 # quick usage:
 # storeBackup.pl --sourceDir /home/user --backupDir /tmp/my_backup_destination
@@ -48,56 +48,31 @@ stdenv.mkDerivation {
     PATH=$PATH:${dummyMount}/bin
 
 
-    { # simple sanity test, test backup/restore of simple store paths
-
+    ## test it
+    backup_init(){
       mkdir backup
+    }
+    latestBackup(){
+      echo backup/default/$(ls -1 backup/default | sort | tail -n 1)
+    }
+    backup_make(){
+      # $1=source
+      $out/bin/storeBackup.pl --sourceDir "$1" --backupDir "backup"
+    }
+    backup_restore_latest(){
+      $out/bin/storeBackupRecover.pl -b "$(latestBackup)" -t "$1" -r /
+    }
 
-      backupRestore(){
-        source="$2"
-        echo =========
-        echo RUNNING TEST "$1" source: "$source"
-        mkdir restored
-
-        $out/bin/storeBackup.pl --sourceDir "$source" --backupDir backup
-        latestBackup=backup/default/$(ls -1 backup/default | sort | tail -n 1)
-        $out/bin/storeBackupRecover.pl -b "$latestBackup" -t restored -r /
-        ${diffutils}/bin/diff -r "$source" restored
-
-        # storeBackupCheckSource should return 0
-        $out/bin/storeBackupCheckSource.pl -s "$source" -b "$latestBackup"
-        # storeBackupCheckSource should return not 0 when using different source
-        ! $out/bin/storeBackupCheckSource.pl -s $TMP -b "$latestBackup"
-
-        # storeBackupCheckBackup should return 0
-        $out/bin/storeBackupCheckBackup.pl -c "$latestBackup"
-
-        chmod -R +w restored
-        rm -fr restored
-      }
-
-      testDir=$TMP/testDir
-
-      mkdir $testDir
-      echo X > $testDir/X
-      ln -s ./X $testDir/Y
-
-      backupRestore 'test 1: backup, restore' $testDir
-
-      # test huge blocks, according to docs files bigger than 100MB get split
-      # into pieces
-      dd if=/dev/urandom bs=100M of=block-1 count=1
-      dd if=/dev/urandom bs=100M of=block-2 count=1
-      cat block-1 block-2 > $testDir/block
-      backupRestore 'test 1 with huge block' $testDir
-
-      cat block-2 block-1 > $testDir/block
-      backupRestore 'test 1 with huge block reversed' $testDir
-
-      backupRestore 'test 2: backup, restore' $out
-      backupRestore 'test 3: backup, restore' $out
-      backupRestore 'test 4: backup diffutils to same backup locations, restore' ${diffutils}
+    backup_verify_integrity_latest(){
+      $out/bin/storeBackupCheckBackup.pl -c "$(latestBackup)"
     }
-  '';
+    backup_verify_latest(){
+      $out/bin/storeBackupCheckSource.pl -s "$1" -b "$(latestBackup)"
+    }
+
+    . ${import ../test-case.nix { inherit diffutils writeTextFile; }}
+    backup_test backup 100M
+'';
 
   meta = {
     description = "Storebackup is a backup suite that stores files on other disks";

diff --git a/pkgs/tools/backup/test-case.nix b/pkgs/tools/backup/test-case.nix
@@ -0,0 +1,104 @@
+# while this test suite is not perfect it will at least provide some guarantees
+# that basic features should be fine ..
+
+/*
+  In order to use the suite you have to define the following functions
+
+    backup_init
+    backup_make source
+    backup_restore_latest target
+    backup_verify_integrity_latest
+    backup_verify_latest source
+
+  use true if a backup system does not implement a feature
+
+  TODO: add test cases for all backup solutions shipping with nixpkgs
+
+  This does not replace the test suites shipping with the backup solutions!
+*/
+
+{diffutils, writeTextFile}:
+
+writeTextFile {
+  name = "backup-test-case";
+  text = ''
+  backup_run_tests_on_source(){
+    local test="$1"
+    local source="$2"
+    local backup="$3"
+    echo =========
+    echo RUNNING TEST "$test" source: "$source"
+    mkdir restored
+
+    backup_make "$source" backup
+
+    { # verify that restoring works
+      backup_restore_latest restored
+      ${diffutils}/bin/diff -r "$source" restored
+      # diff does not make a difference for symlinks, so list them and compare
+      # lists
+      ( cd "$source"; find /var/www/ -type l) | sort > 1
+      ( cd "$restored"; find /var/www/ -type l) | sort > 2
+      diff 1 2
+    }
+
+    { # verify that backup tool thinks so, too:
+      backup_verify_latest "$source" backup
+      # using different source verification must fail:
+      ! backup_verify_latest "$TMP" backup
+    }
+
+    backup_verify_integrity_latest backup
+
+    chmod -R +w restored
+    rm -fr restored
+  }
+
+  backup_test(){
+    # allow getting run time to compare backup solutions
+    echo "START $(date)"
+
+    local block_size="$2"
+
+    backup_init
+
+    if [ -z "$SKIP_SYMLINK_TEST" ]; then
+      { # create first test case directory contentents
+        testDir=$TMP/test-1a
+        mkdir $testDir
+        echo X > $testDir/X
+        ln -s ./X $testDir/Y
+      }
+
+      backup_run_tests_on_source 'test 1a: backup, restore' "$testDir" "$backup"
+    fi
+
+    if [ -z "$SKIP_EMPTY_DIR_TEST" ]; then
+      { # create first test case directory contentents
+        testDir=$TMP/test-1b
+        mkdir -p $testDir/empty-directory
+      }
+
+      backup_run_tests_on_source 'test 1b: backup, restore' "$testDir" "$backup"
+    fi
+
+    testDir=$TMP/test-huge-blocks
+    mkdir $testDir
+    # some backup systems treat huge plocks such as virtual machine images in specific ways
+    # thus create some large files and test them.
+    dd if=/dev/urandom bs=1M of=block-0 count=20
+    dd if=/dev/urandom bs="$block_size" of=block-1 count=1
+    dd if=/dev/urandom bs="$block_size" of=block-2 count=1
+    cat block-0 block-0 block-0 block-1 block-2 block-0 block-0 block-0 > $testDir/block
+    backup_run_tests_on_source 'test 1 with huge block' $testDir
+
+    cat block-2 block-0 block-0 block-1 > $testDir/block
+    backup_run_tests_on_source 'test 1 with huge block reversed' $testDir
+
+    backup_run_tests_on_source 'test 2: backup, restore' $out
+    backup_run_tests_on_source 'test 3: backup, restore' $out
+    backup_run_tests_on_source 'test 4: backup diffutils to same backup locations, restore' ${diffutils}
+    echo "STOP $(date)"
+  }
+  '';
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
@@ -587,6 +587,7 @@ let
     inherit (haskellPackages) pandoc;
     par2Support = (config.bup.par2Support or false);
   };
+  bupLatest = bup.override { version = "latest"; };
 
   atool = callPackage ../tools/archivers/atool { };