reftests: Allow minor differences to be tolerated

Based on an earlier patch by Michael Biebl, with additional inspiration
from librsvg's reftests.

Each .ui or .node reftest can have an accompanying .keyfile file
like this:

    [reftest]
    tolerated-diff-level=20
    tolerated-diff-pixels=1000

If the image differs, but the number of pixels that differ is no more
than tolerated-diff-pixels and the differences are no more than
tolerated-diff-level, then we treat it as a success with warnings, save
the .diff.png for analysis, and use g_test_incomplete() to record the
test-case as "TODO".

Signed-off-by: Simon McVittie <smcv@debian.org>
Forwarded: https://gitlab.gnome.org/GNOME/gtk/-/merge_requests/3195
Applied-upstream: no, upstream want reftests to be a strict pass/fail with identical results required

Gbp-Pq: Name reftests-Allow-minor-differences-to-be-tolerated.patch

diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c
index 5b3d21aee82fe27faedb4aaf524136cc44f714cd..e526bc8d9840c904c3d688781eef67905b9d1c07 100644 (file)
--- a/testsuite/reftests/gtk-reftest.c
+++ b/testsuite/reftests/gtk-reftest.c
@@ -260,6 +260,12 @@ save_image (cairo_surface_t *surface,
   g_free (filename);
 }
 
+static char *
+get_test_keyfile (const char *ui_file)
+{
+  return get_test_file (ui_file, ".keyfile", TRUE);
+}
+
 static void
 test_ui_file (GFile *file)
 {
@@ -296,10 +302,34 @@ test_ui_file (GFile *file)
 
   if (diff_image)
     {
+      char *keyfile_path = get_test_keyfile (ui_file);
+      GKeyFile *keyfile = g_key_file_new ();
+      guint64 tolerated_diff = 0;
+      guint64 tolerated_pixels = 0;
+
+      if (keyfile_path != NULL)
+        {
+          GError *error = NULL;
+          g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error);
+          g_assert_no_error (error);
+          tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL);
+          g_test_message ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels", tolerated_diff);
+          tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL);
+          g_test_message ("Different pixels tolerated: %" G_GUINT64_FORMAT, tolerated_pixels);
+        }
+
       g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels",
                       pixels_changed, pixels, max_diff);
       save_image (diff_image, ui_file, ".diff.png");
-      g_test_fail ();
+      cairo_surface_destroy (diff_image);
+
+      if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels)
+        g_test_message ("not right, but close enough?");
+      else
+        g_test_fail ();
+
+      g_key_file_unref (keyfile);
+      g_free (keyfile_path);
     }
 
   remove_extra_css (provider);