make a reasonable header magic format

src/bin/brotli.rs

@@ -399,150 +399,163 @@ fn main() {
   params.quality = 11; // default
   let mut filenames = [std::string::String::new(), std::string::String::new()];
   let mut num_benchmarks = 1;
+  let mut double_dash = false;
   if env::args_os().len() > 1 {
     let mut first = true;
     for argument in env::args() {
       if first {
         first = false;
         continue;
       }
-      if argument == "-catable" || argument == "--catable" {
+      if argument == "--" {
+        double_dash = true;
+        continue;
+      }
+      if (argument == "-catable" || argument == "--catable") && !double_dash {
           params.catable = true;
           continue;
       }
-      if argument == "--dump-dictionary" {
+      if (argument == "-appendable" || argument == "--appendable") && !double_dash {
+          params.appendable = true;
+          continue;
+      }
+      if (argument.starts_with("-magic") || argument.starts_with("--magic")) && !double_dash {
+          params.magic_number = true;
+          continue;
+      }
+      if argument == "--dump-dictionary" && !double_dash {
         util::print_dictionary(util::permute_dictionary());
         return
       }
-      if argument == "-utf8" {
+      if argument == "-utf8" && !double_dash {
           params.mode = BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR;
           continue;
       }
-      if argument == "-msb" {
+      if argument == "-msb" && !double_dash {
           params.mode = BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR;
           continue;
       }
-      if argument == "-lsb" {
+      if argument == "-lsb" && !double_dash {
           params.mode = BrotliEncoderMode::BROTLI_FORCE_LSB_PRIOR;
           continue;
       }
-      if argument == "-signed" {
+      if argument == "-signed" && !double_dash {
           params.mode = BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR;
           continue;
       }
-      if argument == "-i" {
+      if argument == "-i" && !double_dash {
         // display the intermediate representation of metablocks
         params.log_meta_block = true;
         continue;
       }
-      if argument == "-0" || argument == "-q0" {
+      if (argument == "-0" || argument == "-q0") && !double_dash {
         params.quality = 0;
         continue;
       }
-      if argument == "-1" || argument == "-q1" {
+      if (argument == "-1" || argument == "-q1") && !double_dash {
         params.quality = 1;
         continue;
       }
-      if argument == "-2" || argument == "-q2" {
+      if (argument == "-2" || argument == "-q2") && !double_dash {
         params.quality = 2;
         continue;
       }
-      if argument == "-3" || argument == "-q3" {
+      if (argument == "-3" || argument == "-q3") && !double_dash {
         params.quality = 3;
         continue;
       }
-      if argument == "-4" || argument == "-q4" {
+      if (argument == "-4" || argument == "-q4") && !double_dash {
         params.quality = 4;
         continue;
       }
-      if argument == "-5" || argument == "-q5" {
+      if (argument == "-5" || argument == "-q5") && !double_dash {
         params.quality = 5;
         continue;
       }
-      if argument == "-6" || argument == "-q6" {
+      if (argument == "-6" || argument == "-q6") && !double_dash {
         params.quality = 6;
         continue;
       }
-      if argument == "-7" || argument == "-q7" {
+      if (argument == "-7" || argument == "-q7") && !double_dash {
         params.quality = 7;
         continue;
       }
-      if argument == "-8" || argument == "-q8" {
+      if (argument == "-8" || argument == "-q8") && !double_dash {
         params.quality = 8;
         continue;
       }
-      if argument == "-9" || argument == "-q9" {
+      if (argument == "-9" || argument == "-q9") && !double_dash {
         params.quality = 9;
         continue;
       }
-      if argument == "-9.5" || argument == "-q9.5" {
+      if (argument == "-9.5" || argument == "-q9.5") && !double_dash {
         params.quality = 10;
         params.q9_5 = true;
         continue;
       }
-      if argument == "-9.5x" || argument == "-q9.5x" {
+      if (argument == "-9.5x" || argument == "-q9.5x") && !double_dash {
         params.quality = 11;
         params.q9_5 = true;
         continue;
       }
-      if argument == "-10" || argument == "-q10" {
+      if (argument == "-10" || argument == "-q10") && !double_dash {
         params.quality = 10;
         continue;
       }
-      if argument == "-11" || argument == "-q11" {
+      if (argument == "-11" || argument == "-q11") && !double_dash {
         params.quality = 11;
         continue;
       }
-      if argument == "-q9.5y" {
+      if (argument == "-q9.5y") && !double_dash {
           params.quality = 12;
           params.q9_5 = true;
         continue;
       }
-      if argument.starts_with("-l") {
+      if argument.starts_with("-l") && !double_dash {
         params.lgblock = argument.trim_matches('-').trim_matches('l').parse::<i32>().unwrap();
         continue;
       }
-      if argument.starts_with("-bytescore=") {
+      if argument.starts_with("-bytescore=") && !double_dash {
         params.hasher.literal_byte_score = argument.trim_matches('-').trim_matches('b').trim_matches('y').trim_matches('t').trim_matches('e').trim_matches('s').trim_matches('c').trim_matches('o').trim_matches('r').trim_matches('e').trim_matches('=').parse::<i32>().unwrap();
         continue;
       }
-      if argument.starts_with("-w") {
+      if argument.starts_with("-w") && !double_dash {
           params.lgwin = argument.trim_matches('-').trim_matches('w').parse::<i32>().unwrap();
           continue;
       }
-      if argument.starts_with("-l") {
+      if argument.starts_with("-l") && !double_dash {
           params.lgblock = argument.trim_matches('-').trim_matches('l').parse::<i32>().unwrap();
           continue;
       }
-      if argument.starts_with("-findprior") {
+      if argument.starts_with("-findprior") && !double_dash {
           params.prior_bitmask_detection = 1;
           continue;
       }
-      if argument.starts_with("-findspeed=") {
+      if argument.starts_with("-findspeed=") && !double_dash {
           params.cdf_adaptation_detection = argument.trim_matches('-').trim_matches('f').trim_matches('i').trim_matches('n').trim_matches('d').trim_matches('r').trim_matches('a').trim_matches('n').trim_matches('d').trim_matches('o').trim_matches('m').trim_matches('=').parse::<u32>().unwrap() as u8;
           continue;
-      } else if argument == "-findspeed" {
+      } else if argument == "-findspeed" && !double_dash {
           params.cdf_adaptation_detection = 1;
           continue;
       }
-      if argument == "-basicstride" {
+      if argument == "-basicstride" && !double_dash {
           params.stride_detection_quality = 1;
           continue;
-      } else if argument == "-advstride" {
+      } else if argument == "-advstride" && !double_dash {
           params.stride_detection_quality = 3;
           continue;
       } else {
-          if argument == "-stride" {
+          if argument == "-stride" && !double_dash {
               params.stride_detection_quality = 2;
               continue;
           } else {
-              if argument.starts_with("-s") && !argument.starts_with("-speed=") {
+              if (argument.starts_with("-s") && !argument.starts_with("-speed=")) && !double_dash {
                   params.size_hint = argument.trim_matches('-').trim_matches('s').parse::<usize>().unwrap();
                   continue;
               }
           }
       }
-      if argument.starts_with("-speed=") {
+      if argument.starts_with("-speed=") && !double_dash {
           let comma_string = argument.trim_matches('-').trim_matches('s').trim_matches('p').trim_matches('e').trim_matches('e').trim_matches('d').trim_matches('=');
           let split = comma_string.split(",");
           for (index, s) in split.enumerate() {
@@ -568,18 +581,18 @@ fn main() {
           }
           continue;
       }
-      if argument == "-avoiddistanceprefixsearch" {
+      if argument == "-avoiddistanceprefixsearch" && !double_dash {
           params.avoid_distance_prefix_search = true;
       }
-      if argument.starts_with("-b") {
+      if argument.starts_with("-b") && !double_dash {
           num_benchmarks = argument.trim_matches('-').trim_matches('b').parse::<usize>().unwrap();
           continue;
       }
-      if argument == "-c" {
+      if argument == "-c" && !double_dash {
         do_compress = true;
         continue;
       }
-      if argument == "-h" || argument == "-help" || argument == "--help" {
+      if argument == "-h" || argument == "-help" || argument == "--help" && !double_dash {
         println_stderr!("Decompression:\nbrotli [input_file] [output_file]\nCompression:brotli -c -q9.5 -w22 [input_file] [output_file]\nQuality may be one of -q9.5 -q9.5x -q9.5y or -q[0-11] for standard brotli settings.\nOptional size hint -s<size> to direct better compression\n\nThe -i parameter produces a cross human readdable IR representation of the file.\nThis can be ingested by other compressors.\nIR-specific options include:\n-findprior\n-speed=<inc,max,inc,max,inc,max,inc,max>");
         return;
       }

src/enc/backward_references.rs

@@ -90,6 +90,10 @@ pub struct BrotliEncoderParams {
   pub avoid_distance_prefix_search: bool,
   // construct brotli in such a way that it may be concatenated with another brotli file using appropriate bit ops
   pub catable: bool,
+  // construct brotli in such a way that another concatable brotli file may be appended
+  pub appendable: bool,
+  // include a magic number and version number and size_hint at the beginning
+  pub magic_number: bool,
 }
 
 impl Default for BrotliEncoderParams {

src/enc/brotli_bit_stream.rs

@@ -4,6 +4,7 @@
 #![allow(unused_macros)]
 #[cfg(not(feature="no-stdlib"))]
 use std::io::Write;
+use VERSION;
 use super::{v8, s16};
 use super::util::floatX;
 use super::prior_eval;
@@ -2775,3 +2776,31 @@ pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8])
     BrotliWriteBits(1u8, 1u64, storage_ix, storage);
     JumpToByteBoundary(storage_ix, storage);
 }
+pub fn BrotliWriteMetadataMetaBlock(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) {
+    BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last
+    BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1)
+    BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved
+    BrotliWriteBits(2u8, 1u64, storage_ix, storage); // num bytes for length of magic number header
+    BrotliWriteBits(8u8, 11u64, storage_ix, storage); // 1 byte of data: writing 12 for the magic number header
+    JumpToByteBoundary(storage_ix, storage);
+    let magic_number = if params.catable {
+        [0xe1, 0x97, 0x81]
+    } else if params.appendable {
+        [0xe1, 0x97, 0x82]
+    } else {
+        [0xe1, 0x97, 0x80]
+    };
+    let header = [magic_number[0], magic_number[1], magic_number[2], VERSION,
+                        (params.size_hint & 0xff) as u8,
+                        ((params.size_hint >> 8) & 0xff) as u8,
+                        ((params.size_hint >> 16) & 0xff) as u8,
+                        ((params.size_hint >> 24) & 0xff) as u8,
+                        ((params.size_hint >> 32) & 0xff) as u8,
+                        ((params.size_hint >> 40) & 0xff) as u8,
+                        ((params.size_hint >> 48) & 0xff) as u8,
+                        ((params.size_hint >> 56) & 0xff) as u8,
+    ];
+    for magic in header.iter() {
+        BrotliWriteBits(8u8, u64::from(*magic), storage_ix, storage);
+    }
+}

src/enc/encode.rs

@@ -17,7 +17,7 @@ use super::block_split::BlockSplit;
 use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree,
                                BrotliStoreMetaBlock, BrotliStoreMetaBlockFast,
                                BrotliStoreMetaBlockTrivial, BrotliStoreUncompressedMetaBlock,
-                               BrotliWriteEmptyLastMetaBlock,
+                               BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock,
                                MetaBlockSplit, RecoderState, JumpToByteBoundary};
 
 use enc::input_pair::InputReferenceMut;
@@ -103,6 +103,8 @@ pub enum BrotliEncoderParameter {
   BROTLI_PARAM_CM_SPEED_LOW_MAX = 165,
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
+  BROTLI_PARAM_APPENDABLE = 168,
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
 }
 
 pub struct RingBuffer<AllocU8: alloc::Allocator<u8>> {
@@ -345,6 +347,17 @@ pub fn BrotliEncoderSetParameter<Alloc: BrotliAlloc>
   }
   if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_CATABLE as (i32) {
     (*state).params.catable = value != 0;
+    if !(*state).params.appendable {
+      (*state).params.appendable = value != 0;
+    }
+    return 1i32;
+  }
+  if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_APPENDABLE as (i32) {
+    (*state).params.appendable = value != 0;
+    return 1i32;
+  }
+  if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_MAGIC_NUMBER as (i32) {
+    (*state).params.magic_number = value != 0;
     return 1i32;
   }
   0i32
@@ -394,6 +407,8 @@ pub fn BrotliEncoderInitParams() -> BrotliEncoderParams {
            prior_bitmask_detection: 0,
            literal_adaptation: [(0,0);4],
            catable: false,
+           appendable: false,
+           magic_number: false,
            hasher: BrotliHasherParams {
              type_: 6,
              block_bits: 9 - 1,
@@ -595,6 +610,9 @@ fn SanitizeParams(params: &mut BrotliEncoderParams) {
   } else if (*params).lgwin > 24i32 {
     (*params).lgwin = 24i32;
   }
+  if params.catable {
+    params.appendable = true;
+  }
 }
 
 fn ComputeLgBlock(params: &BrotliEncoderParams) -> i32 {
@@ -2506,8 +2524,10 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                                          &mut [interface::StaticCommand],
                                          interface::InputPair, &mut Alloc) {
   let actual_is_last = is_last;
-  if params.catable {
+  if params.appendable {
     is_last = 0;
+  } else {
+    assert_eq!(params.catable, false); // Sanitize Params senforces this constraint
   }
   let wrapped_last_flush_pos: u32 = WrapPosition(last_flush_pos);
   let last_bytes: u16;
@@ -2757,13 +2777,20 @@ fn EncodeData<Alloc: BrotliAlloc,
     let meta_size = core::cmp::max(bytes as usize,
                                    (*s).input_pos_.wrapping_sub((*s).last_flush_pos_) as usize);
     GetBrotliStorage(s,
-                     (2usize).wrapping_mul(meta_size).wrapping_add(503 + 16));
+                     (2usize).wrapping_mul(meta_size).wrapping_add(503 + 24));
   }
   {
     (*s).storage_.slice_mut()[0] = (*s).last_bytes_ as u8;
     (*s).storage_.slice_mut()[1] = ((*s).last_bytes_ >> 8) as u8;
   }
   let mut catable_header_size = 0;
+  if s.is_first_mb && s.params.magic_number {
+      // this could be written more than once if this function is called with 0 or 1 byte of data. that's ok
+      BrotliWriteMetadataMetaBlock(&s.params, &mut storage_ix, (*s).storage_.slice_mut());
+      (*s).last_bytes_ = (*s).storage_.slice()[((storage_ix >> 3i32) as (usize))] as u16 | (
+        ((*s).storage_.slice()[1 + ((storage_ix >> 3i32) as (usize))] as u16)<<8);
+      (*s).last_bytes_bits_ = (storage_ix & 7u32 as (usize)) as (u8);
+  }
   if !s.params.catable {
     s.is_first_mb = false;
   } else if bytes != 0 && (bytes <= 2 || s.is_first_mb) {

src/lib.rs

@@ -16,7 +16,7 @@ extern crate packed_simd;
 extern crate alloc_no_stdlib as alloc;
 extern crate brotli_decompressor;
 pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
-
+pub const VERSION: u8 = 1;
 #[cfg(not(feature="no-stdlib"))]
 pub use alloc::HeapAlloc;
 pub mod enc;