81 lines
2.8 KiB
Diff
81 lines
2.8 KiB
Diff
From: Simon McVittie <smcv@debian.org>
|
|
Date: Sat, 13 Feb 2021 16:19:10 +0000
|
|
Subject: reftests: Allow minor differences to be tolerated
|
|
|
|
Based on an earlier patch by Michael Biebl, with additional inspiration
|
|
from librsvg's reftests.
|
|
|
|
Each .ui or .node reftest can have an accompanying .keyfile file
|
|
like this:
|
|
|
|
[reftest]
|
|
tolerated-diff-level=20
|
|
tolerated-diff-pixels=1000
|
|
|
|
If the image differs, but the number of pixels that differ is no more
|
|
than tolerated-diff-pixels and the differences are no more than
|
|
tolerated-diff-level, then we treat it as a success with warnings, save
|
|
the .diff.png for analysis, and use g_test_incomplete() to record the
|
|
test-case as "TODO".
|
|
|
|
Signed-off-by: Simon McVittie <smcv@debian.org>
|
|
Forwarded: https://gitlab.gnome.org/GNOME/gtk/-/merge_requests/3195
|
|
Applied-upstream: no, upstream want reftests to be a strict pass/fail with identical results required
|
|
---
|
|
testsuite/reftests/gtk-reftest.c | 32 +++++++++++++++++++++++++++++++-
|
|
1 file changed, 31 insertions(+), 1 deletion(-)
|
|
|
|
diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c
|
|
index 814dc67..b772cd5 100644
|
|
--- a/testsuite/reftests/gtk-reftest.c
|
|
+++ b/testsuite/reftests/gtk-reftest.c
|
|
@@ -260,6 +260,12 @@ save_image (cairo_surface_t *surface,
|
|
g_free (filename);
|
|
}
|
|
|
|
+static char *
|
|
+get_test_keyfile (const char *ui_file)
|
|
+{
|
|
+ return get_test_file (ui_file, ".keyfile", TRUE);
|
|
+}
|
|
+
|
|
static void
|
|
test_ui_file (GFile *file)
|
|
{
|
|
@@ -296,10 +302,34 @@ test_ui_file (GFile *file)
|
|
|
|
if (diff_image)
|
|
{
|
|
+ char *keyfile_path = get_test_keyfile (ui_file);
|
|
+ GKeyFile *keyfile = g_key_file_new ();
|
|
+ guint64 tolerated_diff = 0;
|
|
+ guint64 tolerated_pixels = 0;
|
|
+
|
|
+ if (keyfile_path != NULL)
|
|
+ {
|
|
+ GError *error = NULL;
|
|
+ g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error);
|
|
+ g_assert_no_error (error);
|
|
+ tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL);
|
|
+ g_test_message ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels", tolerated_diff);
|
|
+ tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL);
|
|
+ g_test_message ("Different pixels tolerated: %" G_GUINT64_FORMAT, tolerated_pixels);
|
|
+ }
|
|
+
|
|
g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels",
|
|
pixels_changed, pixels, max_diff);
|
|
save_image (diff_image, ui_file, ".diff.png");
|
|
- g_test_fail ();
|
|
+ cairo_surface_destroy (diff_image);
|
|
+
|
|
+ if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels)
|
|
+ g_test_message ("not right, but close enough?");
|
|
+ else
|
|
+ g_test_fail ();
|
|
+
|
|
+ g_key_file_unref (keyfile);
|
|
+ g_free (keyfile_path);
|
|
}
|
|
|
|
remove_extra_css (provider);
|