diff --git a/src/bin/brotli.rs b/src/bin/brotli.rs index 73a2ef01..2766480f 100755 --- a/src/bin/brotli.rs +++ b/src/bin/brotli.rs @@ -399,6 +399,7 @@ fn main() { params.quality = 11; // default let mut filenames = [std::string::String::new(), std::string::String::new()]; let mut num_benchmarks = 1; + let mut double_dash = false; if env::args_os().len() > 1 { let mut first = true; for argument in env::args() { @@ -406,143 +407,155 @@ fn main() { first = false; continue; } - if argument == "-catable" || argument == "--catable" { + if argument == "--" { + double_dash = true; + continue; + } + if (argument == "-catable" || argument == "--catable") && !double_dash { params.catable = true; continue; } - if argument == "--dump-dictionary" { + if (argument == "-appendable" || argument == "--appendable") && !double_dash { + params.appendable = true; + continue; + } + if (argument.starts_with("-magic") || argument.starts_with("--magic")) && !double_dash { + params.magic_number = true; + continue; + } + if argument == "--dump-dictionary" && !double_dash { util::print_dictionary(util::permute_dictionary()); return } - if argument == "-utf8" { + if argument == "-utf8" && !double_dash { params.mode = BrotliEncoderMode::BROTLI_FORCE_UTF8_PRIOR; continue; } - if argument == "-msb" { + if argument == "-msb" && !double_dash { params.mode = BrotliEncoderMode::BROTLI_FORCE_MSB_PRIOR; continue; } - if argument == "-lsb" { + if argument == "-lsb" && !double_dash { params.mode = BrotliEncoderMode::BROTLI_FORCE_LSB_PRIOR; continue; } - if argument == "-signed" { + if argument == "-signed" && !double_dash { params.mode = BrotliEncoderMode::BROTLI_FORCE_SIGNED_PRIOR; continue; } - if argument == "-i" { + if argument == "-i" && !double_dash { // display the intermediate representation of metablocks params.log_meta_block = true; continue; } - if argument == "-0" || argument == "-q0" { + if (argument == "-0" || argument == "-q0") && !double_dash { params.quality = 0; continue; } - if argument == "-1" || argument == "-q1" { + if (argument == "-1" || argument == "-q1") && !double_dash { params.quality = 1; continue; } - if argument == "-2" || argument == "-q2" { + if (argument == "-2" || argument == "-q2") && !double_dash { params.quality = 2; continue; } - if argument == "-3" || argument == "-q3" { + if (argument == "-3" || argument == "-q3") && !double_dash { params.quality = 3; continue; } - if argument == "-4" || argument == "-q4" { + if (argument == "-4" || argument == "-q4") && !double_dash { params.quality = 4; continue; } - if argument == "-5" || argument == "-q5" { + if (argument == "-5" || argument == "-q5") && !double_dash { params.quality = 5; continue; } - if argument == "-6" || argument == "-q6" { + if (argument == "-6" || argument == "-q6") && !double_dash { params.quality = 6; continue; } - if argument == "-7" || argument == "-q7" { + if (argument == "-7" || argument == "-q7") && !double_dash { params.quality = 7; continue; } - if argument == "-8" || argument == "-q8" { + if (argument == "-8" || argument == "-q8") && !double_dash { params.quality = 8; continue; } - if argument == "-9" || argument == "-q9" { + if (argument == "-9" || argument == "-q9") && !double_dash { params.quality = 9; continue; } - if argument == "-9.5" || argument == "-q9.5" { + if (argument == "-9.5" || argument == "-q9.5") && !double_dash { params.quality = 10; params.q9_5 = true; continue; } - if argument == "-9.5x" || argument == "-q9.5x" { + if (argument == "-9.5x" || argument == "-q9.5x") && !double_dash { params.quality = 11; params.q9_5 = true; continue; } - if argument == "-10" || argument == "-q10" { + if (argument == "-10" || argument == "-q10") && !double_dash { params.quality = 10; continue; } - if argument == "-11" || argument == "-q11" { + if (argument == "-11" || argument == "-q11") && !double_dash { params.quality = 11; continue; } - if argument == "-q9.5y" { + if (argument == "-q9.5y") && !double_dash { params.quality = 12; params.q9_5 = true; continue; } - if argument.starts_with("-l") { + if argument.starts_with("-l") && !double_dash { params.lgblock = argument.trim_matches('-').trim_matches('l').parse::().unwrap(); continue; } - if argument.starts_with("-bytescore=") { + if argument.starts_with("-bytescore=") && !double_dash { params.hasher.literal_byte_score = argument.trim_matches('-').trim_matches('b').trim_matches('y').trim_matches('t').trim_matches('e').trim_matches('s').trim_matches('c').trim_matches('o').trim_matches('r').trim_matches('e').trim_matches('=').parse::().unwrap(); continue; } - if argument.starts_with("-w") { + if argument.starts_with("-w") && !double_dash { params.lgwin = argument.trim_matches('-').trim_matches('w').parse::().unwrap(); continue; } - if argument.starts_with("-l") { + if argument.starts_with("-l") && !double_dash { params.lgblock = argument.trim_matches('-').trim_matches('l').parse::().unwrap(); continue; } - if argument.starts_with("-findprior") { + if argument.starts_with("-findprior") && !double_dash { params.prior_bitmask_detection = 1; continue; } - if argument.starts_with("-findspeed=") { + if argument.starts_with("-findspeed=") && !double_dash { params.cdf_adaptation_detection = argument.trim_matches('-').trim_matches('f').trim_matches('i').trim_matches('n').trim_matches('d').trim_matches('r').trim_matches('a').trim_matches('n').trim_matches('d').trim_matches('o').trim_matches('m').trim_matches('=').parse::().unwrap() as u8; continue; - } else if argument == "-findspeed" { + } else if argument == "-findspeed" && !double_dash { params.cdf_adaptation_detection = 1; continue; } - if argument == "-basicstride" { + if argument == "-basicstride" && !double_dash { params.stride_detection_quality = 1; continue; - } else if argument == "-advstride" { + } else if argument == "-advstride" && !double_dash { params.stride_detection_quality = 3; continue; } else { - if argument == "-stride" { + if argument == "-stride" && !double_dash { params.stride_detection_quality = 2; continue; } else { - if argument.starts_with("-s") && !argument.starts_with("-speed=") { + if (argument.starts_with("-s") && !argument.starts_with("-speed=")) && !double_dash { params.size_hint = argument.trim_matches('-').trim_matches('s').parse::().unwrap(); continue; } } } - if argument.starts_with("-speed=") { + if argument.starts_with("-speed=") && !double_dash { let comma_string = argument.trim_matches('-').trim_matches('s').trim_matches('p').trim_matches('e').trim_matches('e').trim_matches('d').trim_matches('='); let split = comma_string.split(","); for (index, s) in split.enumerate() { @@ -568,18 +581,18 @@ fn main() { } continue; } - if argument == "-avoiddistanceprefixsearch" { + if argument == "-avoiddistanceprefixsearch" && !double_dash { params.avoid_distance_prefix_search = true; } - if argument.starts_with("-b") { + if argument.starts_with("-b") && !double_dash { num_benchmarks = argument.trim_matches('-').trim_matches('b').parse::().unwrap(); continue; } - if argument == "-c" { + if argument == "-c" && !double_dash { do_compress = true; continue; } - if argument == "-h" || argument == "-help" || argument == "--help" { + if argument == "-h" || argument == "-help" || argument == "--help" && !double_dash { println_stderr!("Decompression:\nbrotli [input_file] [output_file]\nCompression:brotli -c -q9.5 -w22 [input_file] [output_file]\nQuality may be one of -q9.5 -q9.5x -q9.5y or -q[0-11] for standard brotli settings.\nOptional size hint -s to direct better compression\n\nThe -i parameter produces a cross human readdable IR representation of the file.\nThis can be ingested by other compressors.\nIR-specific options include:\n-findprior\n-speed="); return; } diff --git a/src/enc/backward_references.rs b/src/enc/backward_references.rs index 2ae572dc..8cdeb92f 100755 --- a/src/enc/backward_references.rs +++ b/src/enc/backward_references.rs @@ -90,6 +90,10 @@ pub struct BrotliEncoderParams { pub avoid_distance_prefix_search: bool, // construct brotli in such a way that it may be concatenated with another brotli file using appropriate bit ops pub catable: bool, + // construct brotli in such a way that another concatable brotli file may be appended + pub appendable: bool, + // include a magic number and version number and size_hint at the beginning + pub magic_number: bool, } impl Default for BrotliEncoderParams { diff --git a/src/enc/brotli_bit_stream.rs b/src/enc/brotli_bit_stream.rs index 89b99932..53f7306b 100755 --- a/src/enc/brotli_bit_stream.rs +++ b/src/enc/brotli_bit_stream.rs @@ -4,6 +4,7 @@ #![allow(unused_macros)] #[cfg(not(feature="no-stdlib"))] use std::io::Write; +use VERSION; use super::{v8, s16}; use super::util::floatX; use super::prior_eval; @@ -2775,3 +2776,31 @@ pub fn BrotliWriteEmptyLastMetaBlock(storage_ix: &mut usize, storage: &mut [u8]) BrotliWriteBits(1u8, 1u64, storage_ix, storage); JumpToByteBoundary(storage_ix, storage); } +pub fn BrotliWriteMetadataMetaBlock(params: &BrotliEncoderParams, storage_ix: &mut usize, storage: &mut [u8]) { + BrotliWriteBits(1u8, 0u64, storage_ix, storage); // not last + BrotliWriteBits(2u8, 3u64, storage_ix, storage); // MNIBBLES = 0 (pattern 1,1) + BrotliWriteBits(1u8, 0u64, storage_ix, storage); // reserved + BrotliWriteBits(2u8, 1u64, storage_ix, storage); // num bytes for length of magic number header + BrotliWriteBits(8u8, 11u64, storage_ix, storage); // 1 byte of data: writing 12 for the magic number header + JumpToByteBoundary(storage_ix, storage); + let magic_number = if params.catable { + [0xe1, 0x97, 0x81] + } else if params.appendable { + [0xe1, 0x97, 0x82] + } else { + [0xe1, 0x97, 0x80] + }; + let header = [magic_number[0], magic_number[1], magic_number[2], VERSION, + (params.size_hint & 0xff) as u8, + ((params.size_hint >> 8) & 0xff) as u8, + ((params.size_hint >> 16) & 0xff) as u8, + ((params.size_hint >> 24) & 0xff) as u8, + ((params.size_hint >> 32) & 0xff) as u8, + ((params.size_hint >> 40) & 0xff) as u8, + ((params.size_hint >> 48) & 0xff) as u8, + ((params.size_hint >> 56) & 0xff) as u8, + ]; + for magic in header.iter() { + BrotliWriteBits(8u8, u64::from(*magic), storage_ix, storage); + } +} diff --git a/src/enc/encode.rs b/src/enc/encode.rs index 707a66d6..24e2de8f 100755 --- a/src/enc/encode.rs +++ b/src/enc/encode.rs @@ -17,7 +17,7 @@ use super::block_split::BlockSplit; use super::brotli_bit_stream::{BrotliBuildAndStoreHuffmanTreeFast, BrotliStoreHuffmanTree, BrotliStoreMetaBlock, BrotliStoreMetaBlockFast, BrotliStoreMetaBlockTrivial, BrotliStoreUncompressedMetaBlock, - BrotliWriteEmptyLastMetaBlock, + BrotliWriteEmptyLastMetaBlock, BrotliWriteMetadataMetaBlock, MetaBlockSplit, RecoderState, JumpToByteBoundary}; use enc::input_pair::InputReferenceMut; @@ -103,6 +103,8 @@ pub enum BrotliEncoderParameter { BROTLI_PARAM_CM_SPEED_LOW_MAX = 165, BROTLI_PARAM_AVOID_DISTANCE_PREFIX_SEARCH = 166, BROTLI_PARAM_CATABLE = 167, + BROTLI_PARAM_APPENDABLE = 168, + BROTLI_PARAM_MAGIC_NUMBER = 169, } pub struct RingBuffer> { @@ -345,6 +347,17 @@ pub fn BrotliEncoderSetParameter } if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_CATABLE as (i32) { (*state).params.catable = value != 0; + if !(*state).params.appendable { + (*state).params.appendable = value != 0; + } + return 1i32; + } + if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_APPENDABLE as (i32) { + (*state).params.appendable = value != 0; + return 1i32; + } + if p as (i32) == BrotliEncoderParameter::BROTLI_PARAM_MAGIC_NUMBER as (i32) { + (*state).params.magic_number = value != 0; return 1i32; } 0i32 @@ -394,6 +407,8 @@ pub fn BrotliEncoderInitParams() -> BrotliEncoderParams { prior_bitmask_detection: 0, literal_adaptation: [(0,0);4], catable: false, + appendable: false, + magic_number: false, hasher: BrotliHasherParams { type_: 6, block_bits: 9 - 1, @@ -595,6 +610,9 @@ fn SanitizeParams(params: &mut BrotliEncoderParams) { } else if (*params).lgwin > 24i32 { (*params).lgwin = 24i32; } + if params.catable { + params.appendable = true; + } } fn ComputeLgBlock(params: &BrotliEncoderParams) -> i32 { @@ -2506,8 +2524,10 @@ fn WriteMetaBlockInternal> 8) as u8; } let mut catable_header_size = 0; + if s.is_first_mb && s.params.magic_number { + // this could be written more than once if this function is called with 0 or 1 byte of data. that's ok + BrotliWriteMetadataMetaBlock(&s.params, &mut storage_ix, (*s).storage_.slice_mut()); + (*s).last_bytes_ = (*s).storage_.slice()[((storage_ix >> 3i32) as (usize))] as u16 | ( + ((*s).storage_.slice()[1 + ((storage_ix >> 3i32) as (usize))] as u16)<<8); + (*s).last_bytes_bits_ = (storage_ix & 7u32 as (usize)) as (u8); + } if !s.params.catable { s.is_first_mb = false; } else if bytes != 0 && (bytes <= 2 || s.is_first_mb) { diff --git a/src/lib.rs b/src/lib.rs index 8c89932b..2c4fda24 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ extern crate packed_simd; extern crate alloc_no_stdlib as alloc; extern crate brotli_decompressor; pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator}; - +pub const VERSION: u8 = 1; #[cfg(not(feature="no-stdlib"))] pub use alloc::HeapAlloc; pub mod enc;