From 27e2d402f9c7823ccf5dcee90bec94db9c2f6fc7 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Sat, 13 Feb 2021 16:19:10 +0000 Subject: [PATCH] reftests: Allow minor differences to be tolerated Based on an earlier patch by Michael Biebl, with additional inspiration from librsvg's reftests. Each .ui or .node reftest can have an accompanying .keyfile file like this: [reftest] tolerated-diff-level=20 tolerated-diff-pixels=1000 If the image differs, but the number of pixels that differ is no more than tolerated-diff-pixels and the differences are no more than tolerated-diff-level, then we treat it as a success with warnings, save the .diff.png for analysis, and use g_test_incomplete() to record the test-case as "TODO". Signed-off-by: Simon McVittie Forwarded: https://gitlab.gnome.org/GNOME/gtk/-/merge_requests/3195 Applied-upstream: no, upstream want reftests to be a strict pass/fail with identical results required Gbp-Pq: Name reftests-Allow-minor-differences-to-be-tolerated.patch --- testsuite/reftests/gtk-reftest.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c index 5b3d21aee8..e526bc8d98 100644 --- a/testsuite/reftests/gtk-reftest.c +++ b/testsuite/reftests/gtk-reftest.c @@ -260,6 +260,12 @@ save_image (cairo_surface_t *surface, g_free (filename); } +static char * +get_test_keyfile (const char *ui_file) +{ + return get_test_file (ui_file, ".keyfile", TRUE); +} + static void test_ui_file (GFile *file) { @@ -296,10 +302,34 @@ test_ui_file (GFile *file) if (diff_image) { + char *keyfile_path = get_test_keyfile (ui_file); + GKeyFile *keyfile = g_key_file_new (); + guint64 tolerated_diff = 0; + guint64 tolerated_pixels = 0; + + if (keyfile_path != NULL) + { + GError *error = NULL; + g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error); + g_assert_no_error (error); + tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL); + g_test_message ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels", tolerated_diff); + tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL); + g_test_message ("Different pixels tolerated: %" G_GUINT64_FORMAT, tolerated_pixels); + } + g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels", pixels_changed, pixels, max_diff); save_image (diff_image, ui_file, ".diff.png"); - g_test_fail (); + cairo_surface_destroy (diff_image); + + if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels) + g_test_message ("not right, but close enough?"); + else + g_test_fail (); + + g_key_file_unref (keyfile); + g_free (keyfile_path); } remove_extra_css (provider); -- 2.30.2