Skip to content

Commit

Permalink
Auto merge of #719 - Mark-Simulacrum:shrink-html-report, r=Mark-Simul…
Browse files Browse the repository at this point in the history
…acrum

Update tera/minifier & cut some memory allocations
  • Loading branch information
bors committed Aug 31, 2024
2 parents bdd7be9 + 1416cf3 commit 28fee8f
Show file tree
Hide file tree
Showing 8 changed files with 279 additions and 108 deletions.
238 changes: 157 additions & 81 deletions Cargo.lock

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
name = "crater"
version = "0.1.0"
edition = "2021"

build = "build.rs"
default-run = "crater"

[profile.dev]
opt-level = 0
Expand All @@ -24,7 +24,7 @@ http = "0.2"
hyper = "0.14"
lazy_static = "1.0"
mime = "0.3.1"
minifier = { version = "0.2", features = ["html"] }
minifier = { version = "0.3", features = ["html"] }
r2d2 = "0.8.2"
r2d2_sqlite = "0.22.0"
rusqlite = { version = "0.29.0", features = ["chrono", "functions", "bundled"] }
Expand All @@ -38,7 +38,7 @@ serde_regex = "1.1.0"
clap = { version = "4", features = ["derive"] }
tar = "0.4.36"
tempfile = "3.0.0"
tera = "0.11.7"
tera = "1.19.1"
toml = "0.8.6"
url = "2"
walkdir = "2"
Expand All @@ -60,7 +60,7 @@ tokio = "1.24"
aws-sdk-s3 = "1.7"
aws-config = { version = "1", features = ["behavior-version-latest"] }
thiserror = "1.0.38"
nix = { version = "0.27.1", features = ["mman"] }
nix = { version = "0.27.1", features = ["mman", "resource"] }

[dev-dependencies]
assert_cmd = "2.0.4"
Expand Down
5 changes: 3 additions & 2 deletions src/assets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ fn build_tera_cache() -> Fallible<Tera> {
}

#[allow(unused_variables)]
pub fn render_template<C: Serialize>(name: &str, context: &C) -> Fallible<String> {
pub fn render_template<C: Serialize>(name: &str, context: C) -> Fallible<String> {
// On debug builds the cache is rebuilt every time to pick up changed templates
let tera_owned: Tera;
let tera;
Expand All @@ -161,7 +161,8 @@ pub fn render_template<C: Serialize>(name: &str, context: &C) -> Fallible<String
tera = &TERA_CACHE;
}

let tera_context = tera::Context::from_serialize(context)?;
Ok(tera
.render(name, context)
.render(name, &tera_context)
.map_err(|e| failure::format_err!("{:?}", e))?)
}
78 changes: 78 additions & 0 deletions src/bin/test-report.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
use crater::experiments::ExperimentDBRecord;
use crater::report::ReportWriter;
use crater::results::EncodingType;
use crater::{config::Config, db::QueryUtils};
use failure::Fallible;
use mime::{self, Mime};
use std::{borrow::Cow, fmt, path::Path};

fn main() {
let mut env = env_logger::Builder::new();
env.filter_module("test_report", log::LevelFilter::Info);
env.filter_module("crater", log::LevelFilter::Info);
env.filter_module("rustwide", log::LevelFilter::Info);
if let Ok(content) = std::env::var("RUST_LOG") {
env.parse_filters(&content);
}
rustwide::logging::init_with(env.build());
let config: Config = toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap();
let db = crater::db::Database::open_at(std::path::Path::new("crater.db")).unwrap();
let experiments = db
.query("SELECT * FROM experiments;", [], |r| {
ExperimentDBRecord::from_row(r)
})
.unwrap();
let experiments: Vec<_> = experiments
.into_iter()
.map(|record| record.into_experiment())
.collect::<Fallible<_>>()
.unwrap();
let ex = experiments.iter().find(|e| e.name == "pr-118920").unwrap();
let rdb = crater::results::DatabaseDB::new(&db);

log::info!("Getting crates...");

let crates = ex.get_crates(&db).unwrap();
let writer = NullWriter;

log::info!("Starting report generation...");
log::info!(
"@ {:?}",
nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF)
.unwrap()
.max_rss()
);
crater::report::gen(&rdb, ex, &crates, &writer, &config, false).unwrap();
log::info!(
"@ {:?}",
nix::sys::resource::getrusage(nix::sys::resource::UsageWho::RUSAGE_SELF)
.unwrap()
.max_rss()
);
}

#[derive(Debug)]
struct NullWriter;

impl ReportWriter for NullWriter {
fn write_bytes<P: AsRef<Path>>(
&self,
_path: P,
_b: &[u8],
_mime: &Mime,
_encoding_type: EncodingType,
) -> Fallible<()> {
// no-op
Ok(())
}
fn write_string<P: AsRef<Path>>(&self, _path: P, _s: Cow<str>, _mime: &Mime) -> Fallible<()> {
// no-op
Ok(())
}
}

impl fmt::Display for NullWriter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
24 changes: 24 additions & 0 deletions src/db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use r2d2::{CustomizeConnection, Pool};
use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::types::ToSql;
use rusqlite::{Connection, Row, Transaction};
use std::path::Path;
use std::sync::Arc;
use std::time::Instant;
use tempfile::NamedTempFile;
Expand Down Expand Up @@ -55,6 +56,11 @@ impl Database {
Database::new(SqliteConnectionManager::file(path), None)
}

pub fn open_at(path: &Path) -> Fallible<Self> {
std::fs::create_dir_all(&*WORK_DIR)?;
Database::new(SqliteConnectionManager::file(path), None)
}

#[cfg(test)]
pub fn temp() -> Fallible<Self> {
let tempfile = NamedTempFile::new()?;
Expand Down Expand Up @@ -191,6 +197,24 @@ pub trait QueryUtils {
})
}

fn query_row<T, F: FnOnce(&Row) -> Fallible<T>>(
&self,
sql: &str,
params: impl rusqlite::Params,
func: F,
) -> Fallible<Option<T>> {
self.with_conn(|conn| {
self.trace(sql, || {
let mut prepared = conn.prepare(sql)?;
let mut rows = prepared.query(params)?;
if let Ok(Some(row)) = rows.next() {
return Ok(Some(func(row)?));
}
Ok(None)
})
})
}

fn trace<T, F: FnOnce() -> T>(&self, sql: &str, f: F) -> T {
let start = Instant::now();
let res = f();
Expand Down
6 changes: 3 additions & 3 deletions src/experiments.rs
Original file line number Diff line number Diff line change
Expand Up @@ -659,7 +659,7 @@ impl Experiment {
}
}

struct ExperimentDBRecord {
pub struct ExperimentDBRecord {
name: String,
mode: String,
cap_lints: String,
Expand All @@ -680,7 +680,7 @@ struct ExperimentDBRecord {
}

impl ExperimentDBRecord {
fn from_row(row: &Row) -> rusqlite::Result<Self> {
pub fn from_row(row: &Row) -> rusqlite::Result<Self> {
Ok(ExperimentDBRecord {
name: row.get("name")?,
mode: row.get("mode")?,
Expand All @@ -702,7 +702,7 @@ impl ExperimentDBRecord {
})
}

fn into_experiment(self) -> Fallible<Experiment> {
pub fn into_experiment(self) -> Fallible<Experiment> {
Ok(Experiment {
name: self.name,
toolchains: [self.toolchain_start.parse()?, self.toolchain_end.parse()?],
Expand Down
9 changes: 5 additions & 4 deletions src/report/html.rs
Original file line number Diff line number Diff line change
Expand Up @@ -256,10 +256,6 @@ fn write_report<W: ReportWriter>(
};

info!("generating {}", to);
let rendered = assets::render_template("report/results.html", &context)
.context("rendering template report/results.html")?;
let html = minifier::html::minify(&rendered);
dest.write_string(to, html.into(), &mime::TEXT_HTML)?;

if output_templates {
dest.write_string(
Expand All @@ -269,6 +265,11 @@ fn write_report<W: ReportWriter>(
)?;
}

let rendered = assets::render_template("report/results.html", context)
.context("rendering template report/results.html")?;
let html = minifier::html::minify(&rendered);
dest.write_string(to, html.into(), &mime::TEXT_HTML)?;

Ok(())
}

Expand Down
19 changes: 5 additions & 14 deletions src/results/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,22 +179,13 @@ impl<'a> ReadResults for DatabaseDB<'a> {
toolchain: &Toolchain,
krate: &Crate,
) -> Fallible<Option<TestResult>> {
let result: Option<String> = self
.db
.query(
"SELECT result FROM results \
Ok(self.db.query_row(
"SELECT result FROM results \
WHERE experiment = ?1 AND toolchain = ?2 AND crate = ?3 \
LIMIT 1;",
[&ex.name, &toolchain.to_string(), &krate.id()],
|row| row.get("result"),
)?
.pop();

if let Some(res) = result {
Ok(Some(res.parse()?))
} else {
Ok(None)
}
[&ex.name, &toolchain.to_string(), &krate.id()],
|row| Ok(row.get_ref("result")?.as_str()?.parse::<TestResult>()?),
)?)
}
}

Expand Down

0 comments on commit 28fee8f

Please sign in to comment.