reftest_compare_surfaces: Report how much the images differ

In unattended/non-interactive/autobuilder environments where the images
are not trivially accessible, this provides a way to distinguish between
totally different rendering and more subtle issues.

Signed-off-by: Simon McVittie <smcv@debian.org>
Forwarded: https://gitlab.gnome.org/GNOME/gtk/-/merge_requests/3195
Applied-upstream: no, upstream want reftests to be a strict pass/fail with identical results required

Gbp-Pq: Topic debian
Gbp-Pq: Name reftest_compare_surfaces-Report-how-much-the-images-diffe.patch

diff --git a/testsuite/gsk/compare-render.c b/testsuite/gsk/compare-render.c
index 7ba25ad961f12a0025517b887a31a20b533c8ff8..e26b16c0209c5c7d1e4ca3ae1832ea682917abd9 100644 (file)
--- a/testsuite/gsk/compare-render.c
+++ b/testsuite/gsk/compare-render.c
@@ -229,6 +229,9 @@ main (int argc, char **argv)
   GError *error = NULL;
   GOptionContext *context;
   GdkTexture *diff_texture;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
+  guint pixels = 0;
 
   (g_test_init) (&argc, &argv, NULL);
 
@@ -283,9 +286,12 @@ main (int argc, char **argv)
     }
 
   /* Now compare the two */
-  diff_texture = reftest_compare_textures (rendered_texture, reference_texture);
+  diff_texture = reftest_compare_textures (rendered_texture, reference_texture,
+                                           &max_diff, &pixels_changed, &pixels);
   if (diff_texture)
     {
+      g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+               pixels_changed, pixels, max_diff);
       save_image (diff_texture, node_file, ".diff.png");
       g_object_unref (diff_texture);
       success = FALSE;
@@ -317,10 +323,13 @@ main (int argc, char **argv)
 
       save_image (reference_texture, node_file, "-flipped.ref.png");
 
-      diff_texture = reftest_compare_textures (rendered_texture, reference_texture);
+      diff_texture = reftest_compare_textures (rendered_texture, reference_texture,
+                                               &max_diff, &pixels_changed, &pixels);
 
       if (diff_texture)
         {
+          g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+                   pixels_changed, pixels, max_diff);
           save_image (diff_texture, node_file, "-flipped.diff.png");
           g_object_unref (diff_texture);
           success = FALSE;
@@ -390,10 +399,13 @@ main (int argc, char **argv)
 
       save_image (reference_texture, node_file, "-repeated.ref.png");
 
-      diff_texture = reftest_compare_textures (rendered_texture, reference_texture);
+      diff_texture = reftest_compare_textures (rendered_texture, reference_texture,
+                                               &max_diff, &pixels_changed, &pixels);
 
       if (diff_texture)
         {
+          g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+                   pixels_changed, pixels, max_diff);
           save_image (diff_texture, node_file, "-repeated.diff.png");
           g_object_unref (diff_texture);
           success = FALSE;
@@ -427,10 +439,13 @@ main (int argc, char **argv)
 
       save_image (reference_texture, node_file, "-rotated.ref.png");
 
-      diff_texture = reftest_compare_textures (rendered_texture, reference_texture);
+      diff_texture = reftest_compare_textures (rendered_texture, reference_texture,
+                                               &max_diff, &pixels_changed, &pixels);
 
       if (diff_texture)
         {
+          g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+                   pixels_changed, pixels, max_diff);
           save_image (diff_texture, node_file, "-rotated.diff.png");
           g_object_unref (diff_texture);
           success = FALSE;
@@ -473,10 +488,13 @@ main (int argc, char **argv)
 
       save_image (reference_texture, node_file, "-masked.ref.png");
 
-      diff_texture = reftest_compare_textures (rendered_texture, reference_texture);
+      diff_texture = reftest_compare_textures (rendered_texture, reference_texture,
+                                               &max_diff, &pixels_changed, &pixels);
 
       if (diff_texture)
         {
+          g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n",
+                   pixels_changed, pixels, max_diff);
           save_image (diff_texture, node_file, "-masked.diff.png");
           g_object_unref (diff_texture);
           success = FALSE;

diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c
index 2ba47a45932e39150065fc70bd46b954c0c87b9f..046751d6db9d185036ac4ba0266d5a5ef7fc4c5b 100644 (file)
--- a/testsuite/reftests/gtk-reftest.c
+++ b/testsuite/reftests/gtk-reftest.c
@@ -341,6 +341,9 @@ test_ui_file (GFile *file)
   char *ui_file, *reference_file;
   GdkTexture *ui_image, *reference_image, *diff_image;
   GtkStyleProvider *provider;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
+  guint pixels = 0;
 
   ui_file = g_file_get_path (file);
 
@@ -372,12 +375,16 @@ test_ui_file (GFile *file)
   if (reference_image == NULL)
     reference_image = gdk_memory_texture_new (1, 1, GDK_MEMORY_DEFAULT, g_bytes_new ((guchar[4]) {0, 0, 0, 0}, 4), 4);
 
-  diff_image = reftest_compare_textures (ui_image, reference_image);
+  diff_image = reftest_compare_textures (ui_image, reference_image,
+                                         &max_diff, &pixels_changed, &pixels);
 
   save_image (ui_image, ui_file, ".out.png");
   save_image (reference_image, ui_file, ".ref.png");
+
   if (diff_image)
     {
+      g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels",
+                      pixels_changed, pixels, max_diff);
       save_node (g_object_get_data (G_OBJECT (ui_image), "source-render-node"), ui_file, ".out.node");
       save_node (g_object_get_data (G_OBJECT (reference_image), "source-render-node"), ui_file, ".ref.node");
       save_image (diff_image, ui_file, ".diff.png");

diff --git a/testsuite/reftests/reftest-compare.c b/testsuite/reftests/reftest-compare.c
index bb6c6ebc1a6f411bd8ee5fb2d18e26c3623970cf..202fcf53ad22a27e409dee1add5191ef81e2efa3 100644 (file)
--- a/testsuite/reftests/reftest-compare.c
+++ b/testsuite/reftests/reftest-compare.c
@@ -35,12 +35,16 @@ buffer_diff_core (const guchar *buf_a,
                  const guchar *buf_b,
                   int           stride_b,
                  int           width,
-                 int           height)
+                 int           height,
+                  guint        *max_diff_out,
+                  guint        *pixels_changed_out)
 {
   int x, y;
   guchar *buf_diff = NULL;
   int stride_diff = 0;
   GdkTexture *diff = NULL;
+  guint max_diff = 0;
+  guint pixels_changed = 0;
 
   for (y = 0; y < height; y++)
     {
@@ -84,6 +88,10 @@ buffer_diff_core (const guchar *buf_a,
               guint channel_diff;
 
               channel_diff = ABS (value_a - value_b);
+
+              if (channel_diff > max_diff)
+                max_diff = channel_diff;
+
               channel_diff *= 4;  /* emphasize */
               if (channel_diff)
                 channel_diff += 128; /* make sure it's visible */
@@ -92,6 +100,8 @@ buffer_diff_core (const guchar *buf_a,
               diff_pixel |= channel_diff << (channel * 8);
             }
 
+          pixels_changed++;
+
           if ((diff_pixel & 0x00ffffff) == 0)
             {
               /* alpha only difference, convert to luminance */
@@ -105,12 +115,21 @@ buffer_diff_core (const guchar *buf_a,
       }
   }
 
+  if (max_diff_out != NULL)
+    *max_diff_out = max_diff;
+
+  if (pixels_changed_out != NULL)
+    *pixels_changed_out = pixels_changed;
+
   return diff;
 }
 
 GdkTexture *
 reftest_compare_textures (GdkTexture *texture1,
-                          GdkTexture *texture2)
+                          GdkTexture *texture2,
+                          guint      *max_diff_out,
+                          guint      *pixels_changed_out,
+                          guint      *pixels_out)
 {
   int w, h;
   guchar *data1, *data2;
@@ -126,10 +145,13 @@ reftest_compare_textures (GdkTexture *texture1,
 
   diff = buffer_diff_core (data1, w * 4,
                            data2, w * 4,
-                           w, h);
+                           w, h, max_diff_out, pixels_changed_out);
 
   g_free (data1);
   g_free (data2);
 
+  if (pixels_out != NULL)
+    *pixels_out = w * h;
+
   return diff;
 }

diff --git a/testsuite/reftests/reftest-compare.h b/testsuite/reftests/reftest-compare.h
index d5e02a1a7dd4619ffc0301f647c14a6d9877bd94..b71636e27c86ec6d30a10644ba34783d637ad948 100644 (file)
--- a/testsuite/reftests/reftest-compare.h
+++ b/testsuite/reftests/reftest-compare.h
@@ -24,7 +24,10 @@ G_BEGIN_DECLS
 
 G_MODULE_EXPORT
 GdkTexture *            reftest_compare_textures        (GdkTexture             *texture1,
-                                                         GdkTexture             *texture2);
+                                                         GdkTexture             *texture2,
+                                                         guint                  *max_diff_out,
+                                                         guint                  *pixels_changed_out,
+                                                         guint                  *pixels_out);
 
 G_END_DECLS