diff --git a/c/arg.h b/c/arg.h index 19bac930..2de71ec6 100644 --- a/c/arg.h +++ b/c/arg.h @@ -70,6 +70,21 @@ size_t set_options(BrotliEncoderParameter *out_encoder_param_keys, out_encoder_param_values[ret] = 1; ret += 1; } + if (strstr(argv[i], "-appendable") == argv[i]) { + out_encoder_param_keys[ret] = BROTLI_PARAM_APPENDABLE; + out_encoder_param_values[ret] = 1; + ret += 1; + } + if (strstr(argv[i], "-bytealign") == argv[i]) { + out_encoder_param_keys[ret] = BROTLI_PARAM_BYTE_ALIGN; + out_encoder_param_values[ret] = 1; + ret += 1; + } + if (strstr(argv[i], "-bare") == argv[i]) { + out_encoder_param_keys[ret] = BROTLI_PARAM_BARE_STREAM; + out_encoder_param_values[ret] = 1; + ret += 1; + } } return ret; } diff --git a/c/brotli/encode.h b/c/brotli/encode.h index 8c2a0b6f..f00b5440 100644 --- a/c/brotli/encode.h +++ b/c/brotli/encode.h @@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter { BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166, BROTLI_PARAM_CATABLE = 167, BROTLI_PARAM_APPENDABLE = 168, - BROTLI_PARAM_MAGIC_NUMBER = 169 + BROTLI_PARAM_MAGIC_NUMBER = 169, + BROTLI_PARAM_NO_DICTIONARY = 170, + BROTLI_PARAM_FAVOR_EFFICIENCY = 171, + BROTLI_PARAM_BYTE_ALIGN = 172, + BROTLI_PARAM_BARE_STREAM = 173 } BrotliEncoderParameter; /** diff --git a/c/go/brotli/brotli.go b/c/go/brotli/brotli.go index 2e4361c0..3596120f 100644 --- a/c/go/brotli/brotli.go +++ b/c/go/brotli/brotli.go @@ -43,6 +43,8 @@ type CompressionOptions struct { Catable bool Appendable bool Magic bool + ByteAlign bool + BareStream bool Mode int LgWin byte LgBlock byte @@ -157,6 +159,14 @@ func makeCompressionOptionsStreams(options CompressionOptions, qualityParams = append(qualityParams, C.BROTLI_PARAM_MAGIC_NUMBER) values = append(values, 1) } + if options.ByteAlign { + qualityParams = append(qualityParams, C.BROTLI_PARAM_BYTE_ALIGN) + values = append(values, 1) + } + if options.BareStream { + qualityParams = append(qualityParams, C.BROTLI_PARAM_BARE_STREAM) + values = append(values, 1) + } if options.Mode != 0 { qualityParams = append(qualityParams, C.BROTLI_PARAM_MODE) values = append(values, C.uint32_t(options.Mode)) diff --git a/c/go/brotli/brotli/encode.h b/c/go/brotli/brotli/encode.h index 3e968975..985df347 100644 --- a/c/go/brotli/brotli/encode.h +++ b/c/go/brotli/brotli/encode.h @@ -224,7 +224,11 @@ typedef enum BrotliEncoderParameter { BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166, BROTLI_PARAM_CATABLE = 167, BROTLI_PARAM_APPENDABLE = 168, - BROTLI_PARAM_MAGIC_NUMBER = 169 + BROTLI_PARAM_MAGIC_NUMBER = 169, + BROTLI_PARAM_NO_DICTIONARY = 170, + BROTLI_PARAM_FAVOR_EFFICIENCY = 171, + BROTLI_PARAM_BYTE_ALIGN = 172, + BROTLI_PARAM_BARE_STREAM = 173, } BrotliEncoderParameter; /** diff --git a/c/py/brotli.py b/c/py/brotli.py index 80af1221..0999be77 100644 --- a/c/py/brotli.py +++ b/c/py/brotli.py @@ -313,6 +313,10 @@ def BrotliParseHeader(raw_data): BROTLI_PARAM_CATABLE = 167 BROTLI_PARAM_APPENDABLE = 168 BROTLI_PARAM_MAGIC_NUMBER = 169 +BROTLI_PARAM_NO_DICTIONARY = 170 +BROTLI_PARAM_FAVOR_EFFICIENCY = 171 +BROTLI_PARAM_BYTE_ALIGN = 172 +BROTLI_PARAM_BARE_STREAM = 173 #simple test binary def main(args): diff --git a/src/bin/brotli.rs b/src/bin/brotli.rs index 3a14cb30..a4b5af00 100755 --- a/src/bin/brotli.rs +++ b/src/bin/brotli.rs @@ -524,7 +524,15 @@ fn main() { use_work_pool = false; continue; } - + if (argument == "-bytealign" || argument == "--bytealign") && !double_dash { + params.byte_align = true; + continue; + } + if (argument == "-bare" || argument == "--bare") && !double_dash { + params.bare_stream = true; + params.byte_align = true; + continue; + } if (argument == "-appendable" || argument == "--appendable") && !double_dash { params.appendable = true; continue; @@ -746,6 +754,14 @@ fn main() { } panic!("Unknown Argument {:}", argument); } + if params.bare_stream && !params.appendable { + println_stderr!("bare streams only supported when catable or appendable!"); + return; + } + if params.byte_align && !params.appendable { + println_stderr!("byte aligned streams only supported when catable or appendable!"); + return; + } if filenames[0] != "" { let mut input = match File::open(&Path::new(&filenames[0])) { Err(why) => panic!("couldn't open {:}\n{:}", filenames[0], why), diff --git a/src/enc/backward_references/mod.rs b/src/enc/backward_references/mod.rs index f280298e..f86dd672 100755 --- a/src/enc/backward_references/mod.rs +++ b/src/enc/backward_references/mod.rs @@ -91,6 +91,11 @@ pub struct BrotliEncoderParams { pub large_window: bool, // avoid search for the best ndirect vs npostfix parameters for distance pub avoid_distance_prefix_search: bool, + // inserts an extra empty metadata block before the final empty metablock in + // catable/appendable mode so concatination tools can just remove the last byte + pub byte_align: bool, + // do not emit a stream header or empty last block at end of data + pub bare_stream: bool, // construct brotli in such a way that it may be concatenated with another brotli file using appropriate bit ops pub catable: bool, // can use the dictionary (default yes unless catable is set) diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs index ad940d9d..da7bd3b6 100755 --- a/src/enc/brotli_bit_stream.rs +++ b/src/enc/brotli_bit_stream.rs @@ -2765,18 +2765,51 @@ pub fn BrotliStoreUncompressedMetaBlock } } - pub fn BrotliStoreSyncMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) { - BrotliWriteBits(6, 6, storage_ix, storage); + BrotliWriteBits(6u8, 6u64, storage_ix, storage); JumpToByteBoundary(storage_ix, storage); } +pub fn BrotliWritePaddingMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) { + if *storage_ix & 7 != 0 { + BrotliWriteBits(6u8, 6u64, storage_ix, storage); + JumpToByteBoundary(storage_ix, storage); + } +} + pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) { BrotliWriteBits(1u8, 1u64, storage_ix, storage); BrotliWriteBits(1u8, 1u64, storage_ix, storage); JumpToByteBoundary(storage_ix, storage); } +pub fn BrotliWriteRawMetadataMetaBlock(storage_ix: &mut usize, storage: &mut [u8], len: usize, data: &mut [u8]) { + BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last + BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1) + BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved + if len > 16777215 { + panic!("metadata too large"); + } else if len > 65535 { + BrotliWriteBits(2u8, 3u64, storage_ix, storage); + BrotliWriteBits(8u8, ((len >> 16) & 255) as u64, storage_ix, storage); + BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage); + BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage); + } else if len > 255 { + BrotliWriteBits(2u8, 2u64, storage_ix, storage); + BrotliWriteBits(8u8, ((len >> 8) & 255) as u64, storage_ix, storage); + BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage); + } else if len == 0 { + BrotliWriteBits(2u8, 0u64, storage_ix, storage); + } else { + BrotliWriteBits(2u8, 1u64, storage_ix, storage); + BrotliWriteBits(8u8, (len & 255) as u64, storage_ix, storage); + } + JumpToByteBoundary(storage_ix, storage); + for index in 0..len { + BrotliWriteBits(8u8, data[index] as u64, storage_ix, storage); + } +} + const MAX_SIZE_ENCODING:usize = 10; fn encode_base_128(mut value: u64)-> (usize, [u8;MAX_SIZE_ENCODING]) { diff --git a/src/enc/encode.rs b/src/enc/encode.rs index 71abd198..15aa3f46 100755 --- a/src/enc/encode.rs +++ b/src/enc/encode.rs @@ -18,7 +18,8 @@ use super::block_split::BlockSplit; use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree, BrotliStoreMetaBlock, BrotliStoreMetaBlockFast, BrotliStoreMetaBlockTrivial, BrotliStoreUncompressedMetaBlock, - BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock, + BrotliWriteEmptyLastMetaBlock, + BrotliWritePaddingMetaBlock, BrotliWriteMetadataMetaBlock, MetaBlockSplit, RecoderState, JumpToByteBoundary}; use enc::input_pair::InputReferenceMut; @@ -340,6 +341,17 @@ value: u32) -> i32 { params.favor_cpu_efficiency = value != 0; return 1i32; } + if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_BYTE_ALIGN as (i32) { + params.byte_align = value != 0; + return 1i32; + } + if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_BARE_STREAM as (i32) { + params.bare_stream = value != 0; + if !params.byte_align { + params.byte_align = value != 0; + } + return 1i32; + } 0i32 } @@ -397,6 +409,8 @@ pub fn BrotliEncoderInitParams() -> BrotliEncoderParams { cdf_adaptation_detection: 0, prior_bitmask_detection: 0, literal_adaptation: [(0,0);4], + byte_align: false, + bare_stream: false, catable: false, use_dictionary: true, appendable: false, @@ -621,7 +635,12 @@ pub fn SanitizeParams(params: &mut BrotliEncoderParams) { } } if params.catable { - params.appendable = true; + params.appendable = true; + } + if params.bare_stream { + params.byte_align = true; + } else if !params.appendable { + params.byte_align = false; } } @@ -721,7 +740,9 @@ fn EnsureInitialized if (*s).params.quality == 0i32 || (*s).params.quality == 1i32 { lgwin = brotli_max_int(lgwin, 18i32); } - EncodeWindowBits(lgwin, s.params.large_window, &mut (*s).last_bytes_, &mut (*s).last_bytes_bits_); + if !((*s).params.catable && (*s).params.bare_stream) { + EncodeWindowBits(lgwin, s.params.large_window, &mut (*s).last_bytes_, &mut (*s).last_bytes_bits_); + } } if (*s).params.quality == 0i32 { InitCommandPrefixCodes(&mut (*s).cmd_depths_[..], @@ -1948,6 +1969,15 @@ fn DecideOverLiteralContextModeling(input: &[u8], literal_context_map); } } +fn WriteEmptyLastBlocksInternal(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) { + // insert empty block for byte alignment if required + if params.byte_align { + BrotliWritePaddingMetaBlock(storage_ix, storage); + } + if !params.bare_stream { + BrotliWriteEmptyLastMetaBlock(storage_ix, storage) + } +} fn WriteMetaBlockInternal (alloc: &mut Alloc, @@ -1975,7 +2005,7 @@ fn WriteMetaBlockInternal