Feature: Add option to force compressed data to be byte aligned

c/arg.h

@@ -70,6 +70,21 @@ size_t set_options(BrotliEncoderParameter *out_encoder_param_keys,
             out_encoder_param_values[ret] = 1;
             ret += 1;
         }
+        if (strstr(argv[i], "-appendable") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_APPENDABLE;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
+        if (strstr(argv[i], "-bytealign") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_BYTE_ALIGN;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
+        if (strstr(argv[i], "-bare") == argv[i]) {
+            out_encoder_param_keys[ret] = BROTLI_PARAM_BARE_STREAM;
+            out_encoder_param_values[ret] = 1;
+            ret += 1;
+        }
     }
     return ret;
 }

c/brotli/encode.h

@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
   BROTLI_PARAM_APPENDABLE = 168,
-  BROTLI_PARAM_MAGIC_NUMBER = 169
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
+  BROTLI_PARAM_NO_DICTIONARY = 170,
+  BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173
 } BrotliEncoderParameter;
 
 /**

c/go/brotli/brotli.go

@@ -43,6 +43,8 @@ type CompressionOptions struct {
 	Catable                       bool
 	Appendable                    bool
 	Magic                         bool
+	ByteAlign                     bool
+	BareStream                    bool
 	Mode                          int
 	LgWin                         byte
 	LgBlock                       byte
@@ -157,6 +159,14 @@ func makeCompressionOptionsStreams(options CompressionOptions,
 		qualityParams = append(qualityParams, C.BROTLI_PARAM_MAGIC_NUMBER)
 		values = append(values, 1)
 	}
+	if options.ByteAlign {
+		qualityParams = append(qualityParams, C.BROTLI_PARAM_BYTE_ALIGN)
+		values = append(values, 1)
+	}
+	if options.BareStream {
+		qualityParams = append(qualityParams, C.BROTLI_PARAM_BARE_STREAM)
+		values = append(values, 1)
+	}
 	if options.Mode != 0 {
 		qualityParams = append(qualityParams, C.BROTLI_PARAM_MODE)
 		values = append(values, C.uint32_t(options.Mode))

c/go/brotli/brotli/encode.h

@@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter {
   BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166,
   BROTLI_PARAM_CATABLE = 167,
   BROTLI_PARAM_APPENDABLE = 168,
-  BROTLI_PARAM_MAGIC_NUMBER = 169
+  BROTLI_PARAM_MAGIC_NUMBER = 169,
+  BROTLI_PARAM_NO_DICTIONARY = 170,
+  BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173,
 } BrotliEncoderParameter;
 
 /**

c/py/brotli.py

@@ -313,6 +313,10 @@ def BrotliParseHeader(raw_data):
 BROTLI_PARAM_CATABLE = 167
 BROTLI_PARAM_APPENDABLE = 168
 BROTLI_PARAM_MAGIC_NUMBER = 169
+BROTLI_PARAM_NO_DICTIONARY = 170
+BROTLI_PARAM_FAVOR_EFFICIENCY = 171
+BROTLI_PARAM_BYTE_ALIGN = 172
+BROTLI_PARAM_BARE_STREAM = 173
 
 #simple test binary
 def main(args):

src/bin/brotli.rs

@@ -524,7 +524,15 @@ fn main() {
           use_work_pool = false;
           continue;
       }
-
+      if (argument == "-bytealign" || argument == "--bytealign") && !double_dash {
+          params.byte_align = true;
+          continue;
+      }
+      if (argument == "-bare" || argument == "--bare") && !double_dash {
+          params.bare_stream = true;
+          params.byte_align = true;
+          continue;
+      }
       if (argument == "-appendable" || argument == "--appendable") && !double_dash {
           params.appendable = true;
           continue;
@@ -746,6 +754,14 @@ fn main() {
       }
       panic!("Unknown Argument {:}", argument);
    }
+   if params.bare_stream && !params.appendable {
+      println_stderr!("bare streams only supported when catable or appendable!");
+      return;
+   }
+   if params.byte_align && !params.appendable {
+      println_stderr!("byte aligned streams only supported when catable or appendable!");
+      return;
+   }
    if filenames[0] != "" {
       let mut input = match File::open(&Path::new(&filenames[0])) {
         Err(why) => panic!("couldn't open {:}\n{:}", filenames[0], why),

src/enc/backward_references/mod.rs

@@ -91,6 +91,11 @@ pub struct BrotliEncoderParams {
   pub large_window: bool,
   // avoid search for the best ndirect vs npostfix parameters for distance
   pub avoid_distance_prefix_search: bool,
+  // inserts an extra empty metadata block before the final empty metablock in
+  // catable/appendable mode so concatination tools can just remove the last byte
+  pub byte_align: bool,
+  // do not emit a stream header or empty last block at end of data
+  pub bare_stream: bool,
   // construct brotli in such a way that it may be concatenated with another brotli file using appropriate bit ops
   pub catable: bool,
   // can use the dictionary (default yes unless catable is set)

src/enc/brotli_bit_stream.rs

@@ -2765,18 +2765,51 @@ pub fn BrotliStoreUncompressedMetaBlock<Cb, Alloc: BrotliAlloc>
   }
 }
 
-
 pub fn BrotliStoreSyncMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
-  BrotliWriteBits(6, 6, storage_ix, storage);
+  BrotliWriteBits(6u8, 6u64, storage_ix, storage);
   JumpToByteBoundary(storage_ix, storage);
 }
 
+pub fn BrotliWritePaddingMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
+  if *storage_ix & 7 != 0 {
+    BrotliWriteBits(6u8, 6u64, storage_ix, storage);
+    JumpToByteBoundary(storage_ix, storage);
+  }
+}
+
 pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) {
     BrotliWriteBits(1u8, 1u64, storage_ix, storage);
     BrotliWriteBits(1u8, 1u64, storage_ix, storage);
     JumpToByteBoundary(storage_ix, storage);
 }
 
+pub fn BrotliWriteRawMetadataMetaBlock(storage_ix: &mut usize, storage: &mut [u8], len: usize, data: &mut [u8]) {
+    BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last
+    BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1)
+    BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved
+    if len > 16777215 {
+        panic!("metadata too large");
+    } else if len > 65535 {
+        BrotliWriteBits(2u8, 3u64, storage_ix, storage);
+        BrotliWriteBits(8u8, ((len >> 16) & 255) as u64, storage_ix, storage);
+        BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage);
+        BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
+    } else if len > 255 {
+        BrotliWriteBits(2u8, 2u64, storage_ix, storage);
+        BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage);
+        BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
+    } else if len == 0 {
+        BrotliWriteBits(2u8, 0u64, storage_ix, storage);
+    } else {
+        BrotliWriteBits(2u8, 1u64, storage_ix, storage);
+        BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage);
+    }
+    JumpToByteBoundary(storage_ix, storage);
+    for index in 0..len {
+        BrotliWriteBits(8u8, data[index] as u64, storage_ix, storage);
+    }
+}
+
 const MAX_SIZE_ENCODING:usize = 10;
 
 fn encode_base_128(mut value: u64)-> (usize, [u8;MAX_SIZE_ENCODING]) {

src/enc/encode.rs

@@ -18,7 +18,8 @@ use super::block_split::BlockSplit;
 use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree,
                                BrotliStoreMetaBlock, BrotliStoreMetaBlockFast,
                                BrotliStoreMetaBlockTrivial, BrotliStoreUncompressedMetaBlock,
-                               BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock,
+                               BrotliWriteEmptyLastMetaBlock,
+                               BrotliWritePaddingMetaBlock, BrotliWriteMetadataMetaBlock,
                                MetaBlockSplit, RecoderState, JumpToByteBoundary};
 
 use enc::input_pair::InputReferenceMut;
@@ -340,6 +341,17 @@ value: u32) -> i32 {
     params.favor_cpu_efficiency = value != 0;
     return 1i32;
   }
+  if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_BYTE_ALIGN as (i32) {
+    params.byte_align = value != 0;
+    return 1i32;
+  }
+  if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_BARE_STREAM as (i32) {
+    params.bare_stream = value != 0;
+    if !params.byte_align {
+      params.byte_align = value != 0;
+    }
+    return 1i32;
+  }
   0i32  
 }
 
@@ -397,6 +409,8 @@ pub fn BrotliEncoderInitParams() -> BrotliEncoderParams {
            cdf_adaptation_detection: 0,
            prior_bitmask_detection: 0,
            literal_adaptation: [(0,0);4],
+           byte_align: false,
+           bare_stream: false,
            catable: false,
            use_dictionary: true,
            appendable: false,
@@ -621,7 +635,12 @@ pub fn SanitizeParams(params: &mut BrotliEncoderParams) {
     }
   }
   if params.catable {
-      params.appendable = true;
+    params.appendable = true;
+  }
+  if params.bare_stream {
+    params.byte_align = true;
+  } else if !params.appendable {
+    params.byte_align = false;
   }
 }
 
@@ -721,7 +740,9 @@ fn EnsureInitialized<Alloc: BrotliAlloc>
     if (*s).params.quality == 0i32 || (*s).params.quality == 1i32 {
       lgwin = brotli_max_int(lgwin, 18i32);
     }
-    EncodeWindowBits(lgwin, s.params.large_window, &mut (*s).last_bytes_, &mut (*s).last_bytes_bits_);
+    if !((*s).params.catable && (*s).params.bare_stream) {
+      EncodeWindowBits(lgwin, s.params.large_window, &mut (*s).last_bytes_, &mut (*s).last_bytes_bits_);
+    }
   }
   if (*s).params.quality == 0i32 {
     InitCommandPrefixCodes(&mut (*s).cmd_depths_[..],
@@ -1948,6 +1969,15 @@ fn DecideOverLiteralContextModeling(input: &[u8],
                      literal_context_map);
   }
 }
+fn WriteEmptyLastBlocksInternal(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) {
+  // insert empty block for byte alignment if required
+  if params.byte_align {
+    BrotliWritePaddingMetaBlock(storage_ix, storage);
+  }
+  if !params.bare_stream {
+    BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
+  }
+}
 fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                           Cb>
             (alloc: &mut Alloc,
@@ -1975,7 +2005,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                                          &mut [interface::StaticCommand],
                                          interface::InputPair, &mut Alloc) {
   let actual_is_last = is_last;
-  if params.appendable {
+  if params.appendable || params.byte_align {
     is_last = 0;
   } else {
     assert_eq!(params.catable, false); // Sanitize Params senforces this constraint
@@ -1986,8 +2016,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
   let literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
   let mut block_params = params.clone();
   if bytes == 0usize {
-    BrotliWriteBits(2usize, 3, storage_ix, storage);
-    *storage_ix = (*storage_ix).wrapping_add(7u32 as (usize)) & !7u32 as (usize);
+    WriteEmptyLastBlocksInternal(params, storage_ix, storage);
     return;
   }
   if ShouldCompress(data,
@@ -2010,7 +2039,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                                      false,
                                      cb);
     if actual_is_last != is_last {
-      BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
+      WriteEmptyLastBlocksInternal(params, storage_ix, storage);
     }
     return;
   }
@@ -2150,7 +2179,7 @@ fn WriteMetaBlockInternal<Alloc: BrotliAlloc,
                                        cb);
   }
   if actual_is_last != is_last {
-    BrotliWriteEmptyLastMetaBlock(storage_ix, storage)
+    WriteEmptyLastBlocksInternal(params, storage_ix, storage);
   }
 }
 
@@ -2247,6 +2276,10 @@ fn EncodeData<Alloc: BrotliAlloc,
       *out_size = catable_header_size;
       s.is_first_mb = IsFirst::HeaderWritten;
     }
+    // fixup for empty stream - note: catable is always appendable
+    if bytes == 0 && s.params.byte_align && s.params.appendable && !s.params.catable {
+      BrotliWritePaddingMetaBlock(&mut storage_ix, (*s).storage_.slice_mut());
+    }
   }
   if let IsFirst::BothCatableBytesWritten = s.is_first_mb {
     // nothing to do here, move along

src/enc/parameters.rs

@@ -29,6 +29,8 @@ pub enum BrotliEncoderParameter {
   BROTLI_PARAM_MAGIC_NUMBER = 169,
   BROTLI_PARAM_NO_DICTIONARY = 170,
   BROTLI_PARAM_FAVOR_EFFICIENCY = 171,
+  BROTLI_PARAM_BYTE_ALIGN = 172,
+  BROTLI_PARAM_BARE_STREAM = 173,
   UNUSED7=7,
   UNUSED8=8,
   UNUSED9=9,
@@ -172,8 +174,6 @@ pub enum BrotliEncoderParameter {
   UNUSED147=147,
   UNUSED148=148,
   UNUSED149=149,
-  UNUSED172=172,
-  UNUSED173=173,
   UNUSED174=174,
   UNUSED175=175,
   UNUSED176=176,