diff --git a/Cargo.lock b/Cargo.lock index 8cdb092f..d92f49e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -876,6 +876,7 @@ dependencies = [ "url 2.4.1", "walkdir", "warp", + "zstd", ] [[package]] @@ -4737,3 +4738,31 @@ name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 259fc153..e5157d5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" } dotenv = "0.15" failure = "0.1.3" flate2 = "1" +zstd = "0.13.0" http = "0.2" hyper = "0.14" lazy_static = "1.0" diff --git a/src/report/archives.rs b/src/report/archives.rs index 06fb2553..4f54489e 100644 --- a/src/report/archives.rs +++ b/src/report/archives.rs @@ -8,7 +8,6 @@ use crate::experiments::Experiment; use crate::prelude::*; use crate::report::{compare, Comparison, ReportWriter}; use crate::results::{EncodedLog, EncodingType, ReadResults}; -use flate2::{write::GzEncoder, Compression}; use indexmap::IndexMap; use tar::{Builder as TarBuilder, Header as TarHeader}; use tempfile::tempfile; @@ -156,7 +155,7 @@ fn write_all_archive( // writes to S3 (requiring buffer management etc) while avoiding keeping the blob entirely // in memory. let backing = tempfile()?; - let mut all = TarBuilder::new(GzEncoder::new(backing, Compression::default())); + let mut all = TarBuilder::new(zstd::stream::Encoder::new(backing, 0)?); for entry in iterate(db, ex, crates, config) { let entry = entry?; let mut header = entry.header(); @@ -180,9 +179,9 @@ fn write_all_archive( view = &buffer[..]; } match dest.write_bytes( - "logs-archives/all.tar.gz", + "logs-archives/all.tar.zst", view, - &"application/gzip".parse().unwrap(), + &"application/zstd".parse().unwrap(), EncodingType::Plain, ) { Ok(()) => break, @@ -192,7 +191,7 @@ fn write_all_archive( } else { std::thread::sleep(std::time::Duration::from_secs(2)); warn!( - "retry ({}/{}) writing logs-archives/all.tar.gz ({} bytes) (error: {:?})", + "retry ({}/{}) writing logs-archives/all.tar.zst ({} bytes) (error: {:?})", i, RETRIES, view.len(), @@ -206,7 +205,7 @@ fn write_all_archive( Ok(Archive { name: "All the crates".to_string(), - path: "logs-archives/all.tar.gz".to_string(), + path: "logs-archives/all.tar.zst".to_string(), }) } @@ -229,14 +228,14 @@ pub fn write_logs_archives( by_comparison .entry(entry.comparison) - .or_insert_with(|| TarBuilder::new(GzEncoder::new(Vec::new(), Compression::default()))) + .or_insert_with(|| TarBuilder::new(zstd::stream::Encoder::new(Vec::new(), 3).unwrap())) .append_data(&mut entry.header(), &entry.path, &entry.log_bytes[..])?; } for (comparison, archive) in by_comparison.drain(..) { let data = archive.into_inner()?.finish()?; dest.write_bytes( - format!("logs-archives/{comparison}.tar.gz"), + format!("logs-archives/{comparison}.tar.zst"), &data, &"application/gzip".parse().unwrap(), EncodingType::Plain, @@ -244,7 +243,7 @@ pub fn write_logs_archives( archives.push(Archive { name: format!("{comparison} crates"), - path: format!("logs-archives/{comparison}.tar.gz"), + path: format!("logs-archives/{comparison}.tar.zst"), }); } @@ -261,11 +260,11 @@ mod tests { use crate::prelude::*; use crate::report::DummyWriter; use crate::results::{DatabaseDB, EncodingType, FailureReason, TestResult, WriteResults}; - use flate2::read::GzDecoder; use mime::Mime; use rustwide::logging::LogStorage; use std::io::Read; use tar::Archive; + use zstd::stream::Decoder; #[test] fn test_logs_archives_generation() { @@ -355,20 +354,20 @@ mod tests { assert_eq!( &archives_paths, &[ - "logs-archives/all.tar.gz", - "logs-archives/regressed.tar.gz", - "logs-archives/test-pass.tar.gz", + "logs-archives/all.tar.zst", + "logs-archives/regressed.tar.zst", + "logs-archives/test-pass.tar.zst", ] ); // Load the content of all the archives - let mime: Mime = "application/gzip".parse().unwrap(); - let all_content = writer.get("logs-archives/all.tar.gz", &mime); - let mut all = Archive::new(GzDecoder::new(all_content.as_slice())); - let regressed_content = writer.get("logs-archives/regressed.tar.gz", &mime); - let mut regressed = Archive::new(GzDecoder::new(regressed_content.as_slice())); - let test_pass_content = writer.get("logs-archives/test-pass.tar.gz", &mime); - let mut test_pass = Archive::new(GzDecoder::new(test_pass_content.as_slice())); + let mime: Mime = "application/zstd".parse().unwrap(); + let all_content = writer.get("logs-archives/all.tar.zst", &mime); + let mut all = Archive::new(Decoder::new(all_content.as_slice()).unwrap()); + let regressed_content = writer.get("logs-archives/regressed.tar.zst", &mime); + let mut regressed = Archive::new(Decoder::new(regressed_content.as_slice()).unwrap()); + let test_pass_content = writer.get("logs-archives/test-pass.tar.zst", &mime); + let mut test_pass = Archive::new(Decoder::new(test_pass_content.as_slice()).unwrap()); macro_rules! check_content { ($archive:ident: { $($file:expr => $match:expr,)* }) => {{ @@ -401,7 +400,7 @@ mod tests { }} } - // Check all.tar.gz + // Check all.tar.zst check_content!(all: { format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[0]) => "tc1 crate1", format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[1]) => "tc2 crate1", @@ -409,13 +408,13 @@ mod tests { format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[1]) => "tc2 crate2", }); - // Check regressed.tar.gz + // Check regressed.tar.zst check_content!(regressed: { format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[0]) => "tc1 crate1", format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[1]) => "tc2 crate1", }); - // Check test-pass.tar.gz + // Check test-pass.tar.zst check_content!(test_pass: { format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[0]) => "tc1 crate2", format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[1]) => "tc2 crate2", diff --git a/templates/report/downloads.html b/templates/report/downloads.html index 523b1a88..b8e1a4c4 100644 --- a/templates/report/downloads.html +++ b/templates/report/downloads.html @@ -14,7 +14,7 @@
- Build logs (tar.gz) + Build logs (tar.zst)
{% for archive in available_archives %}