put command: add --exclude-if-present option

Co-authored-by: Andrew Chambers <[email protected]>

cli-tests/cli-tests.bats

@@ -6,7 +6,7 @@ unset BUPSTASH_KEY
 unset BUPSTASH_KEY_COMMAND
 
 export CLI_TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-export SCRATCH="$BATS_TMPDIR/bupstash-test-scratch"
+export SCRATCH="${BATS_TMPDIR%/}/bupstash-test-scratch"
 export BUPSTASH_KEY="$SCRATCH/bupstash-test-primary.key"
 export PUT_KEY="$SCRATCH/bupstash-test-put.key"
 export METADATA_KEY="$SCRATCH/bupstash-test-metadata.key"
@@ -378,8 +378,7 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
   test 2 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
 }
 
-# Test the exclusion globbing
-@test "backup exclusions" {
+@test "exclusions" {
   mkdir "$SCRATCH/foo"
   touch "$SCRATCH/foo/bang"
   mkdir "$SCRATCH/foo/bar"
@@ -394,10 +393,11 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
   # Exclude on multiple levels
   # As expected, this also excludes $SCRATCH/foo/bang
   id=$(bupstash put --exclude="$SCRATCH/foo/**/bang" :: "$SCRATCH/foo")
+  bupstash get id=$id | tar -tf -
   test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
 
   # Exclude on multiple levels (should be the same)
-  id=$(bupstash put --exclude="/**/bang" :: "$SCRATCH/foo")
+  id=$(bupstash put --exclude="**/bang" :: "$SCRATCH/foo")
   test 3 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
 
   # Exclude on a single level
@@ -417,6 +417,20 @@ llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\
   ! bupstash put --exclude="*/bar" :: "$SCRATCH/foo"
 }
 
+@test "exclude if exists" {
+  mkdir "$SCRATCH/foo"
+  touch "$SCRATCH/foo/bang"
+  mkdir "$SCRATCH/foo/bar"
+  touch "$SCRATCH/foo/bar/bang"
+  touch "$SCRATCH/foo/bar/.backupignore"
+  mkdir "$SCRATCH/foo/bar/baz"
+  touch "$SCRATCH/foo/bar/baz/bang"
+
+  # Keep . bang bar
+  id=$(bupstash put --exclude-if-present=".backupignore" :: "$SCRATCH/foo")
+  test 4 = "$(bupstash get id=$id | tar -tf - | expr $(wc -l))"
+}
+
 @test "checkpoint plain data" {
   # Excercise the checkpointing code, does not check
   # cache invalidation, that is covered via unit tests.

doc/man/bupstash-put.1.md

@@ -160,6 +160,11 @@ With possible types:
   number of levels, `?` matches a single character, `[…]` matches a single character from
   a given character set (and can also be used to escape the other special characters: `[?]`).
 
+* --exclude-if-present FILENAME:
+  Exclude a directory's content if it contains a file with the given name. May be passed multiple times.
+  This will still backup the folder itself, containing the marker file. Common marker file names are `CACHEDIR.TAG`, `.backupexclude`
+  or `.no-backup`.
+
 * --send-log PATH:
   Path to the send log file, defaults to one of the following, in order, provided
   the appropriate environment variables are set, `$BUPSTASH_SEND_LOG`,

src/client.rs

@@ -310,6 +310,7 @@ impl<'a, 'b, 'c> SendSession<'a, 'b, 'c> {
         &mut self,
         paths: Vec<std::path::PathBuf>,
         exclusions: Vec<glob::Pattern>,
+        exclusion_markers: std::collections::HashSet<std::ffi::OsString>,
     ) -> Result<(), anyhow::Error> {
         let use_stat_cache = self.ctx.use_stat_cache;
 
@@ -322,6 +323,7 @@ impl<'a, 'b, 'c> SendSession<'a, 'b, 'c> {
                 want_xattrs: self.ctx.want_xattrs,
                 want_hash: false,
                 exclusions,
+                exclusion_markers,
             },
         )?
         .background();
@@ -601,6 +603,7 @@ pub enum DataSource {
     Filesystem {
         paths: Vec<std::path::PathBuf>,
         exclusions: Vec<glob::Pattern>,
+        exclusion_markers: std::collections::HashSet<std::ffi::OsString>,
     },
 }
 
@@ -709,8 +712,12 @@ pub fn send(
             ctx.progress.set_message(description);
             session.write_data(&mut data, &mut |_: &Address, _: usize| {})?;
         }
-        DataSource::Filesystem { paths, exclusions } => {
-            session.send_dir(paths, exclusions)?;
+        DataSource::Filesystem {
+            paths,
+            exclusions,
+            exclusion_markers,
+        } => {
+            session.send_dir(paths, exclusions, exclusion_markers)?;
         }
     }
 
@@ -1442,6 +1449,7 @@ pub fn restore_to_local_dir(
             &[to_dir.to_owned()],
             indexer::FsIndexerOptions {
                 exclusions: vec![],
+                exclusion_markers: HashSet::new(),
                 want_xattrs: false,
                 want_hash: true,
                 one_file_system: false,

src/indexer.rs

@@ -245,6 +245,7 @@ pub struct FsIndexer {
 
 pub struct FsIndexerOptions {
     pub exclusions: Vec<glob::Pattern>,
+    pub exclusion_markers: std::collections::HashSet<std::ffi::OsString>,
     pub want_xattrs: bool,
     pub want_hash: bool,
     pub one_file_system: bool,
@@ -427,6 +428,21 @@ impl FsIndexer {
         let mut excluded_paths = Vec::new();
         let mut to_recurse = Vec::new();
 
+        if !self.opts.exclusion_markers.is_empty() {
+            for p in dir_ent_paths.iter() {
+                if self.opts.exclusion_markers.contains(p.file_name().unwrap()) {
+                    dir_ent_paths.retain(|p| {
+                        if !self.opts.exclusion_markers.contains(p.file_name().unwrap()) {
+                            excluded_paths.push(p.to_owned());
+                            return false;
+                        };
+                        true
+                    });
+                    break;
+                }
+            }
+        }
+
         dir_ent_paths.retain(|p| {
             for excl in self.opts.exclusions.iter() {
                 if excl.matches_path(p) {

src/main.rs

@@ -845,6 +845,14 @@ fn put_main(args: Vec<String>) -> Result<(), anyhow::Error> {
         Patterns without any slashes will match any file (`--exclude foo` is equivalent to `--exclude '/**/foo'`).",
         "PATTERN",
     );
+    opts.optmulti(
+        "",
+        "exclude-if-present",
+        "Exclude a directory's content if it contains a file with the given name. May be passed multiple times.
+  This will still backup the folder itself, containing the marker file. Common marker file names are `CACHEDIR.TAG`, `.backupexclude`
+  or `.no-backup`.",
+        "FILENAME",
+    );
     opts.optflag(
         "",
         "one-file-system",
@@ -906,37 +914,54 @@ fn put_main(args: Vec<String>) -> Result<(), anyhow::Error> {
 
         /* An exclude path ending on / won't match anything. */
         if e.ends_with('/') {
-            anyhow::bail!("--exclude option '{}' ends with '/', so it won't match anything", e);
+            anyhow::bail!(
+                "--exclude option '{}' ends with '/', so it won't match anything",
+                e
+            );
         }
-        
+
         /* This check is technically redundant, but it gives a nicer error message. */
         if e.starts_with("./") {
             anyhow::bail!("No relative paths in --exclude");
         }
 
         /* Start with a / to match a path, and leave out slashes to match any file. */
-        if e.starts_with('/') {
+        if e.starts_with('/') || e.starts_with("**/") {
             /* pass */
-        } else if e.contains('/') {
-            anyhow::bail!("--exclude option '{}' must start with a '/'", e);
+        } else if e.contains('/') && !e.contains('[') {
+            /* This is just to help the user with a nicer error message, we do
+             * not take this branch if the pattern contains an escape character. */
+            anyhow::bail!(
+                "--exclude option '{}' contains '/' so must be absolute to match anything",
+                e
+            );
         } else {
             /* Just a file name */
             e = format!("/**/{}", e);
         }
 
-        /* Check for unnormalized segments, as they too won't match anything 
-         * Note that this may interfere with range syntax: `[/./]`. However, these would make no
-         * sense as a range because each character is only given once.
-         */
-        if e.contains("/./") || e.contains("/../") || e.ends_with("/.") || e.ends_with("/..") {
-            anyhow::bail!("--exclude option '{}' must be normalized (no '.' and '..' segments)", e);
+        /* Check for unnormalized segments, as they too won't match anything. Skip this
+         * check if the pattern contains the escape character. */
+        if (e.contains("/./") || e.contains("/../") || e.contains("//")) && !e.contains('[') {
+            anyhow::bail!(
+                "--exclude option '{}' must be normalized (no '.', '..' or '//' path segments)",
+                e
+            );
         }
 
-        let pattern = glob::Pattern::new(&e)
-            .map_err(|err| anyhow::format_err!("--exclude option '{}' is not a valid glob: {}", e, err))?;
+        let pattern = glob::Pattern::new(&e).map_err(|err| {
+            anyhow::format_err!("--exclude option '{}' is not a valid glob: {}", e, err)
+        })?;
+
         exclusions.push(pattern);
     }
 
+    let exclusion_markers = matches
+        .opt_strs("exclude-if-present")
+        .drain(..)
+        .map(std::ffi::OsString::from)
+        .collect();
+
     let one_file_system = matches.opt_present("one-file-system");
 
     let checkpoint_bytes: u64 = match std::env::var("BUPSTASH_CHECKPOINT_BYTES") {
@@ -1068,6 +1093,7 @@ fn put_main(args: Vec<String>) -> Result<(), anyhow::Error> {
                 data_source = client::DataSource::Filesystem {
                     paths: vec![input_path],
                     exclusions,
+                    exclusion_markers,
                 };
             } else if md.is_file() {
                 if default_tags && !tags.contains_key("name") {
@@ -1108,6 +1134,7 @@ fn put_main(args: Vec<String>) -> Result<(), anyhow::Error> {
         data_source = client::DataSource::Filesystem {
             paths: absolute_paths,
             exclusions,
+            exclusion_markers,
         };
     };
 
@@ -1717,6 +1744,7 @@ fn diff_main(args: Vec<String>) -> Result<(), anyhow::Error> {
                 &paths,
                 indexer::FsIndexerOptions {
                     exclusions: vec![],
+                    exclusion_markers: std::collections::HashSet::new(),
                     want_xattrs: matches.opt_present("xattrs"),
                     want_hash: true,
                     one_file_system: false,