Skip to content

Commit

Permalink
Add some first snapshot test infrastructure
Browse files Browse the repository at this point in the history
  • Loading branch information
DJMcNab committed Jun 28, 2024
1 parent 318948e commit c4e27f6
Show file tree
Hide file tree
Showing 9 changed files with 281 additions and 12 deletions.
21 changes: 21 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ thiserror = "1.0.61"
# NOTE: Make sure to keep this in sync with the version badge in README.md and vello/README.md
wgpu = { version = "0.20.0" }
log = "0.4.21"
image = { version = "0.25.1", default-features = false }

# Used for examples
clap = "4.5.4"
Expand Down
2 changes: 1 addition & 1 deletion examples/scenes/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ workspace = true
vello = { workspace = true }
anyhow = { workspace = true }
clap = { workspace = true, features = ["derive"] }
image = { version = "0.25.1", default-features = false, features = ["jpeg"] }
image = { workspace = true, features = ["jpeg"] }
rand = "0.8.5"

# for pico_svg
Expand Down
2 changes: 2 additions & 0 deletions vello_tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,5 @@ anyhow = { workspace = true }
pollster = { workspace = true }
png = "0.17.13"
futures-intrusive = { workspace = true }
nv-flip = "0.1.2"
image = { workspace = true, features = ["png"] }
2 changes: 2 additions & 0 deletions vello_tests/snapshots/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*.new.png
!.gitignore
Binary file added vello_tests/snapshots/filled_square.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
29 changes: 18 additions & 11 deletions vello_tests/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ use vello::wgpu::{
};
use vello::{block_on_wgpu, RendererOptions, Scene};

mod snapshot;

pub use snapshot::{snapshot_test, snapshot_test_sync};

pub struct TestParams {
pub width: u32,
pub height: u32,
Expand Down Expand Up @@ -129,35 +133,38 @@ pub async fn render(scene: Scene, params: &TestParams) -> Result<Image> {
}
let data = Blob::new(Arc::new(result_unpadded));
let image = Image::new(data, Format::Rgba8, width, height);
if should_debug_png(&params.name, params.use_cpu) {
if env_var_relates_to("VELLO_DEBUG_TEST", &params.name, params.use_cpu) {
let suffix = if params.use_cpu { "cpu" } else { "gpu" };
let name = format!("{}_{suffix}", &params.name);
debug_png(&image, &name, params)?;
let out_path = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("debug_outputs")
.join(name)
.with_extension("png");
write_png_to_file(params, &out_path, &image)?;
println!("Wrote debug result ({width}x{height}) to {out_path:?}");
}
Ok(image)
}

pub fn debug_png(image: &Image, name: &str, params: &TestParams) -> Result<()> {
pub fn write_png_to_file(
params: &TestParams,
out_path: &std::path::Path,
image: &Image,
) -> Result<(), anyhow::Error> {
let width = params.width;
let height = params.height;
let out_path = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("debug_outputs")
.join(name)
.with_extension("png");
let mut file = File::create(&out_path)?;
let mut encoder = png::Encoder::new(&mut file, width, height);
encoder.set_color(png::ColorType::Rgba);
encoder.set_depth(png::BitDepth::Eight);
let mut writer = encoder.write_header()?;
writer.write_image_data(image.data.data())?;
writer.finish()?;
println!("Wrote result ({width}x{height}) to {out_path:?}");

Ok(())
}

pub fn should_debug_png(name: &str, use_cpu: bool) -> bool {
if let Ok(val) = env::var("VELLO_DEBUG_TEST") {
pub fn env_var_relates_to(env_var: &'static str, name: &str, use_cpu: bool) -> bool {
if let Ok(val) = env::var(env_var) {
if val.eq_ignore_ascii_case("all")
|| val.eq_ignore_ascii_case("cpu") && use_cpu
|| val.eq_ignore_ascii_case("gpu") && !use_cpu
Expand Down
192 changes: 192 additions & 0 deletions vello_tests/src/snapshot.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
// Copyright 2024 the Vello Authors
// SPDX-License-Identifier: Apache-2.0 OR MIT

use core::fmt;
use std::{
io::{self, ErrorKind},
path::{Path, PathBuf},
};

use image::{DynamicImage, ImageError};
use nv_flip::FlipPool;
use vello::{
peniko::{Format, Image},
Scene,
};

use crate::{env_var_relates_to, render, write_png_to_file, TestParams};
use anyhow::{anyhow, bail, Result};

fn snapshot_dir() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("snapshots")
}

#[must_use]
pub struct Snapshot<'a> {
pub pool: Option<FlipPool>,
pub reference_path: PathBuf,
pub update_path: PathBuf,
pub raw_rendered: Image,
pub params: &'a TestParams,
}

impl Snapshot<'_> {
pub fn assert_mean_less_than(&mut self, value: f32) -> Result<()> {
assert!(
value < 0.1,
"Mean should be less than 0.1 in almost all cases for a successful test"
);
if let Some(pool) = &self.pool {
let mean = pool.mean();
if mean > value {
self.handle_failure(format_args!(
"Expected mean to be less than {value}, got {mean}"
))?;
}
} else {
// The image is new, so assertion needed?
}
self.handle_success()?;
Ok(())
}

fn handle_success(&mut self) -> Result<()> {
match std::fs::remove_file(&self.update_path) {
Err(e) if e.kind() == ErrorKind::NotFound => Ok(()),
res => res.map_err(Into::into),
}
}

fn handle_failure(&mut self, message: fmt::Arguments) -> Result<()> {
if env_var_relates_to("VELLO_TEST_UPDATE", &self.params.name, self.params.use_cpu) {
if !self.params.use_cpu {
write_png_to_file(self.params, &self.reference_path, &self.raw_rendered)?;
eprintln!(
"Updated result for updated test {} to {:?}",
self.params.name, &self.reference_path
);
} else {
eprintln!(
"Skipped updating result for test {} as not GPU test",
self.params.name
);
}
} else {
write_png_to_file(self.params, &self.update_path, &self.raw_rendered)?;
eprintln!(
"Wrote result for failing test {} to {:?}\n\
Use `VELLO_TEST_UPDATE=all` to update",
self.params.name, &self.update_path
);
}
bail!("{}", message);
}
}

pub fn snapshot_test_sync(scene: Scene, params: &TestParams) -> Result<Snapshot<'_>> {
pollster::block_on(snapshot_test(scene, params))
}

pub async fn snapshot_test(scene: Scene, params: &TestParams) -> Result<Snapshot> {
let raw_rendered = render(scene, params).await?;

// TODO: A different file for GPU and CPU?
let reference_path = snapshot_dir().join(&params.name).with_extension("png");
let update_extension = if params.use_cpu {
"cpu.new.png"
} else {
"gpu.new.png"
};
let update_path = snapshot_dir()
.join(&params.name)
.with_extension(update_extension);

let expected_data = match image::open(&reference_path) {
Ok(contents) => contents.into_rgb8(),
Err(ImageError::IoError(e)) if e.kind() == io::ErrorKind::NotFound => {
if env_var_relates_to("VELLO_TEST_CREATE", &params.name, params.use_cpu) {
if params.use_cpu {
write_png_to_file(params, &reference_path, &raw_rendered)?;
eprintln!(
"Wrote result for new test {} to {:?}",
params.name, &reference_path
);
} else {
eprintln!(
"Skipped writing result for new test {} as not GPU test",
params.name
);
}
return Ok(Snapshot {
pool: None,
reference_path,
update_path,
raw_rendered,
params,
});
} else {
write_png_to_file(params, &update_path, &raw_rendered)?;
bail!(
"Couldn't find snapshot for test {}. Searched at {:?}\n\
Test result written to {:?}\n\
Use `VELLO_TEST_CREATE=all` to update",
params.name,
reference_path,
update_path
);
}
}
Err(e) => return Err(e.into()),
};

if expected_data.width() != raw_rendered.width || expected_data.height() != raw_rendered.height
{
let mut snapshot = Snapshot {
pool: None,
reference_path,
update_path,
raw_rendered,
params,
};
snapshot.handle_failure(format_args!(
"Got wrong size. Expected ({expected_width}x{expected_height}), found ({actual_width}x{actual_height})",
expected_width = expected_data.width(),
expected_height = expected_data.height(),
actual_width = params.width,
actual_height = params.height
))?;
unreachable!();
}
// Compare the images using nv-flip
assert_eq!(raw_rendered.format, Format::Rgba8);
let rendered_data: DynamicImage = image::RgbaImage::from_raw(
raw_rendered.width,
raw_rendered.height,
raw_rendered.data.as_ref().to_vec(),
)
.ok_or(anyhow!("Couldn't create image"))?
.into();
let rendered_data = rendered_data.to_rgb8();
let expected = nv_flip::FlipImageRgb8::with_data(
expected_data.width(),
expected_data.height(),
&expected_data,
);
let rendered = nv_flip::FlipImageRgb8::with_data(
rendered_data.width(),
rendered_data.height(),
&rendered_data,
);

let error_map = nv_flip::flip(expected, rendered, nv_flip::DEFAULT_PIXELS_PER_DEGREE);

let pool = nv_flip::FlipPool::from_image(&error_map);

Ok(Snapshot {
pool: Some(pool),
reference_path,
update_path,
raw_rendered,
params,
})
}
44 changes: 44 additions & 0 deletions vello_tests/tests/simplest.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright 2024 the Vello Authors
// SPDX-License-Identifier: Apache-2.0 OR MIT

use vello::{
kurbo::{Affine, Rect},
peniko::{Brush, Color},
Scene,
};
use vello_tests::{snapshot_test_sync, TestParams};

#[test]
#[cfg_attr(skip_gpu_tests, ignore)]
fn simple_square_gpu() {
filled_square(false);
}

#[test]
// The fine shader still requires a GPU, and so we still get a wgpu device
// skip this for now
#[cfg_attr(skip_gpu_tests, ignore)]
fn simple_square_cpu() {
filled_square(true);
}

fn filled_square(use_cpu: bool) {
let mut scene = Scene::new();
scene.fill(
vello::peniko::Fill::NonZero,
Affine::IDENTITY,
&Brush::Solid(Color::BLUE),
None,
&Rect::from_center_size((10., 10.), (6., 6.)),
);
let params = TestParams {
use_cpu,
..TestParams::new("filled_square", 20, 20)
};
match snapshot_test_sync(scene, &params)
.and_then(|mut snapshot| snapshot.assert_mean_less_than(0.01))
{
Ok(()) => (),
Err(e) => panic!("{:#}", e),
}
}

0 comments on commit c4e27f6

Please sign in to comment.