d/rules, d/run-tests.sh: Add infrastructure to ignore failing GSK tests

Similar to reftests, these can be fragile against varying versions of
dependencies.

diff --git a/debian/ignore.keyfile b/debian/ignore.keyfile
new file mode 100644 (file)
index 0000000..2894620
--- /dev/null
+++ b/debian/ignore.keyfile
@@ -0,0 +1,3 @@
+[reftest]
+tolerated-diff-level=255
+tolerated-diff-pixels=4000

diff --git a/debian/rules b/debian/rules
index 02cea336205712b0d268ede9fc16fe162d5b3a42..b5ad8896f6214fce44a0963062d47d92853c1b46 100755 (executable)
--- a/debian/rules
+++ b/debian/rules
@@ -217,6 +217,15 @@ $(info Architecture with qemu buildds detected, increasing test timeout a lot)
 test_timeout_multiplier = 20
 endif
 
+fuzzy_gsk_compare = \
+       $(NULL)
+
+ignore_gsk_compare = \
+       $(NULL)
+
+xfail_gsk_compare = \
+       $(NULL)
+
 fuzzy_reftests = \
        border-image-url \
        label-attribute-preference \
@@ -225,6 +234,9 @@ fuzzy_reftests = \
        quit-mnemonic \
        $(NULL)
 
+ignore_reftests = \
+       $(NULL)
+
 xfail_reftests = \
        label-sizing \
        sizegroups-evolution-identity-page \
@@ -246,7 +258,11 @@ endif
 override_dh_auto_test:
        set -e; for backend in $(test_backends); do \
                env BUILDDIR=debian/build/deb \
+                       FUZZY_GSK_COMPARE="$(fuzzy_gsk_compare)" \
                        FUZZY_REFTESTS="$(fuzzy_reftests)" \
+                       IGNORE_GSK_COMPARE="$(ignore_gsk_compare)" \
+                       IGNORE_REFTESTS="$(ignore_reftests)" \
+                       XFAIL_GSK_COMPARE="$(xfail_gsk_compare)" \
                        XFAIL_REFTESTS="$(xfail_reftests)" \
                        BACKEND=$$backend \
                                debian/run-tests.sh -t $(test_timeout_multiplier) \

diff --git a/debian/run-tests.sh b/debian/run-tests.sh
index 751ae77565032594096c8b8aa44383f0489fcfe5..3337f285f2990f540ab00ec56a333fffa333c6dd 100755 (executable)
--- a/debian/run-tests.sh
+++ b/debian/run-tests.sh
@@ -5,7 +5,12 @@ set -ex
 BUILDDIR=${BUILDDIR:-debian/build/deb}
 BACKEND=${BACKEND:-x11}
 
+FUZZY_GSK_COMPARE=${FUZZY_GSK_COMPARE:-}
+IGNORE_GSK_COMPARE=${IGNORE_GSK_COMPARE:-}
+XFAIL_GSK_COMPARE=${XFAIL_GSK_COMPARE:-}
+
 FUZZY_REFTESTS=${FUZZY_REFTESTS:-}
+IGNORE_REFTESTS=${IGNORE_REFTESTS:-}
 XFAIL_REFTESTS=${XFAIL_REFTESTS:-}
 
 test_data="$(mktemp -d -t debian-test-data-XXXXXXXX)"
@@ -15,9 +20,13 @@ cleanup() {
     rm -rf "$test_data"
 
     # Avoid incremental builds with -nc leaking settings into the next build
-    for reftest in $FUZZY_REFTESTS; do
+    for reftest in $FUZZY_REFTESTS $IGNORE_REFTESTS; do
         rm -f "testsuite/reftests/$reftest.keyfile"
     done
+
+    for reftest in $FUZZY_GSK_COMPARE $IGNORE_GSK_COMPARE; do
+        rm -f "testsuite/gsk/compare/$reftest.keyfile"
+    done
 }
 
 trap 'cleanup' EXIT INT
@@ -36,6 +45,18 @@ for reftest in $FUZZY_REFTESTS; do
     cp debian/close-enough.keyfile "testsuite/reftests/$reftest.keyfile"
 done
 
+for reftest in $FUZZY_GSK_COMPARE; do
+    cp debian/close-enough.keyfile "testsuite/gsk/compare/$reftest.keyfile"
+done
+
+for reftest in $IGNORE_REFTESTS; do
+    cp debian/ignore.keyfile "testsuite/reftests/$reftest.keyfile"
+done
+
+for reftest in $IGNORE_GSK_COMPARE; do
+    cp debian/ignore.keyfile "testsuite/gsk/compare/$reftest.keyfile"
+done
+
 # So that gsettings can find the (uninstalled) gtk schemas
 mkdir -p "$test_data/glib-2.0/schemas/"
 cp gtk/org.gtk.* "$test_data/glib-2.0/schemas/"
@@ -67,6 +88,12 @@ for reftest in $XFAIL_REFTESTS; do
     rm -f "$BUILDDIR/testsuite/reftests/output/$BACKEND/$reftest.diff.png"
 done
 
+for renderer in cairo gl; do
+    for reftest in $XFAIL_GSK_COMPARE; do
+        rm -f "$BUILDDIR/testsuite/gsk/compare/$renderer/$BACKEND/$reftest.diff.png"
+    done
+done
+
 if [ -e "$test_data/tests-failed" ]; then
     head -v -n-0 "$BUILDDIR/meson-logs/testlog-$BACKEND.txt"
 fi