diff.rs
raw
//! Perceptual comparison of rendered images, used by the golden-image tests
//! and the cross-backend parity check.
use tiny_skia::Pixmap;
/// Result of comparing two images.
#[derive(Debug, Clone, PartialEq)]
pub struct DiffStats {
/// Fraction of pixels whose maximum channel difference exceeds the
/// per-pixel tolerance, in `0.0..=1.0`.
pub differing_fraction: f64,
/// Largest per-channel absolute difference seen anywhere.
pub max_channel_diff: u8,
/// Mean absolute per-channel difference across the whole image.
pub mean_channel_diff: f64,
/// The worst differing fraction within any single tile, and where it was.
///
/// A whole-image fraction cannot see a *small object in the wrong place*:
/// a scrollbar thumb drawn thirty pixels low is a fraction of a percent of
/// the frame and passes any sane global threshold, while being exactly the
/// kind of structural error this comparison exists to catch. Differences
/// that are really anti-aliasing spread themselves over every edge in the
/// picture; a displaced object piles them into one place.
pub worst_tile: TileStats,
}
/// The worst-disagreeing tile of an image comparison.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct TileStats {
/// Fraction of that tile's pixels that differ, in `0.0..=1.0`.
pub differing_fraction: f64,
/// Its top-left corner, in pixels.
pub origin: (u32, u32),
}
/// Tolerances for declaring two renderings "the same picture".
///
/// The defaults absorb anti-aliasing and rounding differences between
/// independent rasterizers while still catching anything structural: a
/// misplaced shape, a wrong color, missing content.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct DiffTolerance {
/// Per-pixel channel difference at or below which a pixel counts as
/// matching.
pub per_pixel: u8,
/// Maximum fraction of non-matching pixels allowed.
pub max_differing_fraction: f64,
/// Per-pixel channel difference above which a pixel counts as *grossly*
/// different, for the local check only.
///
/// Deliberately far coarser than `per_pixel`, because the local check asks
/// a different question: not "do these disagree" but "is something here in
/// one image and missing in the other". Two rasterizers disagree along
/// every anti-aliased edge by small amounts; only a displaced or absent
/// object swings a pixel from background to foreground. Measured across
/// the parity corpus, this is what separates the two — a 4x margin instead
/// of 1.4x.
pub tile_per_pixel: u8,
/// Maximum fraction of grossly-differing pixels allowed within any one
/// [`TILE`]-sized tile. This is what notices a small displaced object,
/// which the whole-image fraction is structurally unable to see.
pub max_tile_fraction: f64,
}
/// Side of the square tiles [`compare`] measures locally, in pixels. Big
/// enough that an anti-aliased edge crossing one is still a small part of it,
/// small enough that a displaced widget fills one.
pub const TILE: u32 = 16;
impl DiffTolerance {
/// For images produced by the same renderer: everything must match
/// exactly.
pub const EXACT: Self = Self {
per_pixel: 0,
max_differing_fraction: 0.0,
tile_per_pixel: 0,
max_tile_fraction: 0.0,
};
/// For images produced by different renderers of the same scene.
/// Anti-aliased edges legitimately differ, so a small fraction of edge
/// pixels may disagree by any amount, and flat areas may differ by
/// rounding.
pub const CROSS_RENDERER: Self = Self {
per_pixel: 12,
max_differing_fraction: 0.02,
// Measured over the parity corpus: the worst legitimate tile is 9.4%
// and the scrollbar-displacement bug this exists for is 37.5%. The
// threshold sits between them with better than two-fold headroom each
// way.
tile_per_pixel: 40,
max_tile_fraction: 0.20,
};
}
/// Compare two images. Returns `None` if the dimensions differ (which is
/// always a failure, not a matter of degree).
pub fn compare(a: &Pixmap, b: &Pixmap, tolerance: DiffTolerance) -> Option<(DiffStats, bool)> {
if a.width() != b.width() || a.height() != b.height() {
return None;
}
let (w, h) = (a.width(), b.height());
let mut differing = 0u64;
let mut max_diff = 0u8;
let mut total_diff = 0u64;
// Per-tile counts, so a *local* pile-up of differences is visible even
// when the whole-image fraction stays tiny.
let tiles_across = w.div_ceil(TILE);
let mut tile_hits = vec![0u32; (tiles_across * h.div_ceil(TILE)) as usize];
let (da, db) = (a.data(), b.data());
for (index, (pa, pb)) in da.chunks_exact(4).zip(db.chunks_exact(4)).enumerate() {
let mut pixel_max = 0u8;
for (ca, cb) in pa.iter().zip(pb.iter()) {
let d = ca.abs_diff(*cb);
pixel_max = pixel_max.max(d);
total_diff += u64::from(d);
}
max_diff = max_diff.max(pixel_max);
if pixel_max > tolerance.per_pixel {
differing += 1;
}
if pixel_max > tolerance.tile_per_pixel {
let (x, y) = (index as u32 % w, index as u32 / w);
tile_hits[(y / TILE * tiles_across + x / TILE) as usize] += 1;
}
}
// Each tile against its own size: the edge tiles of an image are smaller,
// and judging them against a full tile's area would quietly excuse them.
let mut worst = TileStats {
differing_fraction: 0.0,
origin: (0, 0),
};
for (index, hits) in tile_hits.iter().enumerate() {
let (tx, ty) = (index as u32 % tiles_across, index as u32 / tiles_across);
let (ox, oy) = (tx * TILE, ty * TILE);
let area = u64::from((w - ox).min(TILE)) * u64::from((h - oy).min(TILE));
if area == 0 {
continue;
}
let fraction = f64::from(*hits) / area as f64;
if fraction > worst.differing_fraction {
worst = TileStats {
differing_fraction: fraction,
origin: (ox, oy),
};
}
}
let pixel_count = u64::from(w) * u64::from(h);
let stats = DiffStats {
differing_fraction: differing as f64 / pixel_count as f64,
max_channel_diff: max_diff,
mean_channel_diff: total_diff as f64 / (pixel_count * 4) as f64,
worst_tile: worst,
};
let passes = stats.differing_fraction <= tolerance.max_differing_fraction
&& stats.worst_tile.differing_fraction <= tolerance.max_tile_fraction;
Some((stats, passes))
}
#[cfg(test)]
mod tests {
use super::*;
fn solid(width: u32, height: u32, rgba: [u8; 4]) -> Pixmap {
let mut pixmap = Pixmap::new(width, height).unwrap();
for px in pixmap.data_mut().chunks_exact_mut(4) {
px.copy_from_slice(&rgba);
}
pixmap
}
#[test]
fn identical_images_match_exactly() {
let a = solid(8, 8, [10, 20, 30, 255]);
let (stats, passes) = compare(&a, &a.clone(), DiffTolerance::EXACT).unwrap();
assert!(passes);
assert_eq!(stats.max_channel_diff, 0);
}
/// Draw a filled rect into a white image.
fn with_rect(w: u32, h: u32, rect: (u32, u32, u32, u32), rgba: [u8; 4]) -> Pixmap {
let mut pixmap = solid(w, h, [255, 255, 255, 255]);
let (x0, y0, rw, rh) = rect;
let data = pixmap.data_mut();
for y in y0..(y0 + rh).min(h) {
for x in x0..(x0 + rw).min(w) {
let i = ((y * w + x) * 4) as usize;
data[i..i + 4].copy_from_slice(&rgba);
}
}
pixmap
}
/// The case a whole-image fraction cannot see: a small object in the
/// wrong place. This is a scrollbar thumb — six pixels wide — drawn
/// thirty pixels lower than it belongs, which is what a real backend bug
/// did while every global threshold stayed happy.
#[test]
fn a_small_displaced_object_fails_even_though_the_frame_barely_differs() {
let (w, h) = (420, 320);
let thumb = [96, 96, 96, 255];
let a = with_rect(w, h, (410, 40, 6, 120), thumb);
let b = with_rect(w, h, (410, 70, 6, 120), thumb);
let (stats, passes) = compare(&a, &b, DiffTolerance::CROSS_RENDERER).unwrap();
assert!(
stats.differing_fraction < 0.01,
"the premise: this is a trivial fraction of the frame ({:.4}%)",
stats.differing_fraction * 100.0
);
assert!(
!passes,
"and it must fail anyway — worst tile {:.1}% at {:?}",
stats.worst_tile.differing_fraction * 100.0,
stats.worst_tile.origin
);
}
/// The other half of the claim: differences that really are anti-aliasing
/// are spread along every edge, so no one tile piles up.
#[test]
fn edge_noise_spread_across_the_image_still_passes() {
let (w, h) = (256, 256);
let mut a = solid(w, h, [255, 255, 255, 255]);
let mut b = solid(w, h, [255, 255, 255, 255]);
// A diagonal line, one pixel off in one image — an anti-aliased edge
// disagreeing everywhere it goes.
for i in 0..h {
let ia = ((i * w + i) * 4) as usize;
a.data_mut()[ia..ia + 4].copy_from_slice(&[0, 0, 0, 255]);
let ib = ((i * w + (i + 1).min(w - 1)) * 4) as usize;
b.data_mut()[ib..ib + 4].copy_from_slice(&[0, 0, 0, 255]);
}
let (stats, passes) = compare(&a, &b, DiffTolerance::CROSS_RENDERER).unwrap();
assert!(
passes,
"spread-out edge disagreement is what the tolerance is for; \
worst tile {:.1}%",
stats.worst_tile.differing_fraction * 100.0
);
}
#[test]
fn small_rounding_passes_cross_renderer() {
let a = solid(8, 8, [10, 20, 30, 255]);
let b = solid(8, 8, [12, 18, 33, 255]);
let (_, passes) = compare(&a, &b, DiffTolerance::CROSS_RENDERER).unwrap();
assert!(passes);
let (_, exact) = compare(&a, &b, DiffTolerance::EXACT).unwrap();
assert!(!exact);
}
#[test]
fn structural_difference_fails() {
let a = solid(8, 8, [10, 20, 30, 255]);
let b = solid(8, 8, [200, 20, 30, 255]);
let (stats, passes) = compare(&a, &b, DiffTolerance::CROSS_RENDERER).unwrap();
assert!(!passes);
assert_eq!(stats.differing_fraction, 1.0);
}
#[test]
fn size_mismatch_is_none() {
let a = solid(8, 8, [0, 0, 0, 255]);
let b = solid(9, 8, [0, 0, 0, 255]);
assert!(compare(&a, &b, DiffTolerance::EXACT).is_none());
}
}