From 3d7273c36f801fb0da8ae70eec7fe742c317df45 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 28 Aug 2019 14:16:46 -0700 Subject: [PATCH] Use at most 8 threads for the xz stream At preset 6, xz2 uses about 173MB of memory per thread. This adds up quickly -- e.g. over 8GB of memory on a 48-CPU machine. If you happen to try this in a 32-bit build, you'll get `LZMA_MEM_ERROR`. We can limit this to a heuristic maximum number of threads to avoid using so much memory, like xz's [`04_compress_easy_mt` example]. // The number 8 is arbitrarily chosen and may be too low or // high depending on the compression preset and the computer // being used. [`04_compress_easy_mt` example]: https://github.com/xz-mirror/xz/blob/de1f47b2b40e960b7bc3acba754f66dd19705921/doc/examples/04_compress_easy_mt.c#L71 --- src/tarballer.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tarballer.rs b/src/tarballer.rs index 9ff1bd2..fa6ab2c 100644 --- a/src/tarballer.rs +++ b/src/tarballer.rs @@ -47,9 +47,11 @@ impl Tarballer { // Prepare the `.tar.gz` file. let gz = GzEncoder::new(create_new_file(tar_gz)?, flate2::Compression::best()); - // Prepare the `.tar.xz` file. + // Prepare the `.tar.xz` file. Note that preset 6 takes about 173MB of memory + // per thread, so we limit the number of threads to not blow out 32-bit hosts. + // (We could be more precise with `MtStreamBuilder::memusage()` if desired.) let stream = xz2::stream::MtStreamBuilder::new() - .threads(num_cpus::get() as u32) + .threads(Ord::min(num_cpus::get(), 8) as u32) .preset(6) .encoder()?; let xz = XzEncoder::new_stream(create_new_file(tar_xz)?, stream);