diff --git a/testsuite/gsk/compare-render.c b/testsuite/gsk/compare-render.c index 562a82e5d769f36003c46186488942b44a0cde42..a4afd757ae0fda3890850ecd67a08ac3d5b519eb 100644 --- a/testsuite/gsk/compare-render.c +++ b/testsuite/gsk/compare-render.c @@ -2,7 +2,7 @@ #include #include #include -#include "reftest-compare.h" +#include "../reftests/reftest-compare.h" static char *arg_output_dir = NULL; @@ -98,6 +98,12 @@ get_output_file (const char *file, return result; } +static char * +get_test_keyfile (const char *node_file) +{ + return file_replace_extension (node_file, ".node", ".keyfile"); +} + static void save_image (cairo_surface_t *surface, const char *test_name, @@ -232,14 +238,55 @@ main (int argc, char **argv) } else { + guint max_diff = 0; + guint pixels_changed = 0; + guint pixels = 0; + /* Now compare the two */ - diff_surface = reftest_compare_surfaces (rendered_surface, reference_surface); + diff_surface = reftest_compare_surfaces (rendered_surface, reference_surface, + &max_diff, &pixels_changed, &pixels); if (diff_surface) { - save_image (diff_surface, node_file, ".diff.png"); + char *keyfile_path = get_test_keyfile (node_file); + GKeyFile *keyfile = g_key_file_new (); + guint64 tolerated_diff = 0; + guint64 tolerated_pixels = 0; + guint64 accepted_diff = 0; + guint64 accepted_pixels = 0; + + if (keyfile_path != NULL && g_file_test (keyfile_path, G_FILE_TEST_EXISTS)) + { + GError *error = NULL; + g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error); + g_assert_no_error (error); + accepted_diff = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-level", NULL); + g_print ("Maximum difference accepted: %" G_GUINT64_FORMAT " levels\n", accepted_diff); + accepted_pixels = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-pixels", NULL); + g_print ("Different pixels accepted: %" G_GUINT64_FORMAT "\n", accepted_pixels); + tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL); + g_print ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels\n", tolerated_diff); + tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL); + g_print ("Different pixels tolerated: %" G_GUINT64_FORMAT "\n", tolerated_pixels); + } + + g_print ("%u (out of %u) pixels differ from reference by up to %u levels\n", + pixels_changed, pixels, max_diff); + + if (max_diff > accepted_diff || pixels_changed > accepted_pixels) + save_image (diff_surface, node_file, ".diff.png"); + cairo_surface_destroy (diff_surface); - success = FALSE; + + if (max_diff <= accepted_diff && pixels_changed <= accepted_pixels) + g_print ("differences are within acceptable range\n"); + else if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels) + g_print ("not right, but close enough\n"); + else + g_test_fail (); + + g_key_file_unref (keyfile); + g_free (keyfile_path); } } diff --git a/testsuite/gsk/meson.build b/testsuite/gsk/meson.build index 1c4505610467299aeeab9d464355ba8561be20c8..844e76c4753f516156b9f0fce5a37c178c025a84 100644 --- a/testsuite/gsk/meson.build +++ b/testsuite/gsk/meson.build @@ -3,7 +3,7 @@ testdatadir = join_paths(installed_test_datadir, 'gsk') compare_render = executable( 'compare-render', - ['compare-render.c', 'reftest-compare.c'], + ['compare-render.c', '../reftests/reftest-compare.c'], dependencies: libgtk_dep, c_args: common_cflags, install: get_option('install-tests'), diff --git a/testsuite/gsk/reftest-compare.c b/testsuite/gsk/reftest-compare.c deleted file mode 100644 index 6cb8af7fef51395bed8987bf12b4e94e4e329fbb..0000000000000000000000000000000000000000 --- a/testsuite/gsk/reftest-compare.c +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2011 Red Hat Inc. - * - * Author: - * Benjamin Otte - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library. If not, see . - */ - -#include "config.h" - -#include "reftest-compare.h" - -static void -get_surface_size (cairo_surface_t *surface, - int *width, - int *height) -{ - cairo_t *cr; - double x1, x2, y1, y2; - - cr = cairo_create (surface); - cairo_clip_extents (cr, &x1, &y1, &x2, &y2); - cairo_destroy (cr); - - g_assert (x1 == 0 && y1 == 0); - g_assert (x2 > 0 && y2 > 0); - g_assert ((int) x2 == x2 && (int) y2 == y2); - - *width = x2; - *height = y2; -} - -static cairo_surface_t * -coerce_surface_for_comparison (cairo_surface_t *surface, - int width, - int height) -{ - cairo_surface_t *coerced; - cairo_t *cr; - - coerced = cairo_image_surface_create (CAIRO_FORMAT_ARGB32, - width, - height); - cr = cairo_create (coerced); - - cairo_set_source_surface (cr, surface, 0, 0); - cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE); - cairo_paint (cr); - - cairo_destroy (cr); - - g_assert (cairo_surface_status (coerced) == CAIRO_STATUS_SUCCESS); - - return coerced; -} - -/* Compares two CAIRO_FORMAT_ARGB32 buffers, returning NULL if the - * buffers are equal or a surface containing a diff between the two - * surfaces. - * - * This function should be rewritten to compare all formats supported by - * cairo_format_t instead of taking a mask as a parameter. - * - * This function is originally from cairo:test/buffer-diff.c. - * Copyright © 2004 Richard D. Worth - */ -static cairo_surface_t * -buffer_diff_core (const guchar *buf_a, - int stride_a, - const guchar *buf_b, - int stride_b, - int width, - int height) -{ - int x, y; - guchar *buf_diff = NULL; - int stride_diff = 0; - cairo_surface_t *diff = NULL; - - for (y = 0; y < height; y++) - { - const guint32 *row_a = (const guint32 *) (buf_a + y * stride_a); - const guint32 *row_b = (const guint32 *) (buf_b + y * stride_b); - guint32 *row = (guint32 *) (buf_diff + y * stride_diff); - - for (x = 0; x < width; x++) - { - int channel; - guint32 diff_pixel = 0; - - /* check if the pixels are the same */ - if (row_a[x] == row_b[x]) - continue; - - if (diff == NULL) - { - diff = cairo_image_surface_create (CAIRO_FORMAT_RGB24, - width, - height); - g_assert (cairo_surface_status (diff) == CAIRO_STATUS_SUCCESS); - buf_diff = cairo_image_surface_get_data (diff); - stride_diff = cairo_image_surface_get_stride (diff); - row = (guint32 *) (buf_diff + y * stride_diff); - } - - /* calculate a difference value for all 4 channels */ - for (channel = 0; channel < 4; channel++) - { - int value_a = (row_a[x] >> (channel*8)) & 0xff; - int value_b = (row_b[x] >> (channel*8)) & 0xff; - guint channel_diff; - - channel_diff = ABS (value_a - value_b); - channel_diff *= 4; /* emphasize */ - if (channel_diff) - channel_diff += 128; /* make sure it's visible */ - if (channel_diff > 255) - channel_diff = 255; - diff_pixel |= channel_diff << (channel*8); - } - - if ((diff_pixel & 0x00ffffff) == 0) - { - /* alpha only difference, convert to luminance */ - guint8 alpha = diff_pixel >> 24; - diff_pixel = alpha * 0x010101; - } - - row[x] = diff_pixel; - } - } - - return diff; -} - -cairo_surface_t * -reftest_compare_surfaces (cairo_surface_t *surface1, - cairo_surface_t *surface2) -{ - int w1, h1, w2, h2, w, h; - cairo_surface_t *coerced1, *coerced2, *diff; - - get_surface_size (surface1, &w1, &h1); - get_surface_size (surface2, &w2, &h2); - w = MAX (w1, w2); - h = MAX (h1, h2); - coerced1 = coerce_surface_for_comparison (surface1, w, h); - coerced2 = coerce_surface_for_comparison (surface2, w, h); - - diff = buffer_diff_core (cairo_image_surface_get_data (coerced1), - cairo_image_surface_get_stride (coerced1), - cairo_image_surface_get_data (coerced2), - cairo_image_surface_get_stride (coerced2), - w, h); - - cairo_surface_destroy (coerced1); - cairo_surface_destroy (coerced2); - - return diff; -} - diff --git a/testsuite/gsk/reftest-compare.h b/testsuite/gsk/reftest-compare.h deleted file mode 100644 index b889e1659726ba7dfbfe69e092143d1a59a6fc9f..0000000000000000000000000000000000000000 --- a/testsuite/gsk/reftest-compare.h +++ /dev/null @@ -1,30 +0,0 @@ -/* GTK - The GIMP Toolkit - * Copyright (C) 2014 Red Hat, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library. If not, see . - */ - -#ifndef __REFTEST_COMPARE_H__ -#define __REFTEST_COMPARE_H__ - -#include - -G_BEGIN_DECLS - -cairo_surface_t * reftest_compare_surfaces (cairo_surface_t *surface1, - cairo_surface_t *surface2); - -G_END_DECLS - -#endif /* __REFTEST_COMPARE_H__ */ diff --git a/testsuite/reftests/gtk-reftest.c b/testsuite/reftests/gtk-reftest.c index 7e2be7ee1d9be40a9e1252260bb9f8bdc340ad4f..791e95c8491c8adea4827eebbaf0c8420746db70 100644 --- a/testsuite/reftests/gtk-reftest.c +++ b/testsuite/reftests/gtk-reftest.c @@ -290,12 +290,21 @@ save_image (cairo_surface_t *surface, g_free (filename); } +static char * +get_test_keyfile (const char *ui_file) +{ + return get_test_file (ui_file, ".keyfile", TRUE); +} + static void test_ui_file (GFile *file) { char *ui_file, *reference_file; cairo_surface_t *ui_image, *reference_image, *diff_image; GtkStyleProvider *provider; + guint max_diff = 0; + guint pixels_changed = 0; + guint pixels = 0; ui_file = g_file_get_path (file); @@ -315,14 +324,51 @@ test_ui_file (GFile *file) } g_free (reference_file); - diff_image = reftest_compare_surfaces (ui_image, reference_image); + diff_image = reftest_compare_surfaces (ui_image, reference_image, + &max_diff, &pixels_changed, &pixels); save_image (ui_image, ui_file, ".out.png"); save_image (reference_image, ui_file, ".ref.png"); + if (diff_image) { - save_image (diff_image, ui_file, ".diff.png"); - g_test_fail (); + char *keyfile_path = get_test_keyfile (ui_file); + GKeyFile *keyfile = g_key_file_new (); + guint64 tolerated_diff = 0; + guint64 tolerated_pixels = 0; + guint64 accepted_diff = 0; + guint64 accepted_pixels = 0; + + if (keyfile_path != NULL) + { + GError *error = NULL; + g_key_file_load_from_file (keyfile, keyfile_path, G_KEY_FILE_NONE, &error); + g_assert_no_error (error); + accepted_diff = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-level", NULL); + g_test_message ("Maximum difference accepted: %" G_GUINT64_FORMAT " levels", accepted_diff); + accepted_pixels = g_key_file_get_uint64 (keyfile, "reftest", "accepted-diff-pixels", NULL); + g_test_message ("Different pixels accepted: %" G_GUINT64_FORMAT, accepted_pixels); + tolerated_diff = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-level", NULL); + g_test_message ("Maximum difference tolerated: %" G_GUINT64_FORMAT " levels", tolerated_diff); + tolerated_pixels = g_key_file_get_uint64 (keyfile, "reftest", "tolerated-diff-pixels", NULL); + g_test_message ("Different pixels tolerated: %" G_GUINT64_FORMAT, tolerated_pixels); + } + + g_test_message ("%u (out of %u) pixels differ from reference by up to %u levels", + pixels_changed, pixels, max_diff); + + if (max_diff > accepted_diff || pixels_changed > accepted_pixels) + save_image (diff_image, ui_file, ".diff.png"); + + if (max_diff <= accepted_diff && pixels_changed <= accepted_pixels) + g_test_message ("differences are within acceptable range"); + else if (max_diff <= tolerated_diff && pixels_changed <= tolerated_pixels) + g_test_incomplete ("not right, but close enough"); + else + g_test_fail (); + + g_key_file_unref (keyfile); + g_free (keyfile_path); } remove_extra_css (provider); diff --git a/testsuite/reftests/image-compare.c b/testsuite/reftests/image-compare.c index 07bb79e4e06ed9991e760ed269723f4b7351130b..714f4fcadcf9999c5589cb66694e2bac8c55db56 100644 --- a/testsuite/reftests/image-compare.c +++ b/testsuite/reftests/image-compare.c @@ -41,7 +41,7 @@ main (int argc, char **argv) image1 = cairo_image_surface_create_from_png (argv[1]); image2 = cairo_image_surface_create_from_png (argv[2]); - diff = reftest_compare_surfaces (image1, image2); + diff = reftest_compare_surfaces (image1, image2, NULL, NULL, NULL); if (opt_filename && diff) cairo_surface_write_to_png (diff, opt_filename); diff --git a/testsuite/reftests/reftest-compare.c b/testsuite/reftests/reftest-compare.c index 651eac9ca967136e616a3570dc3bc715645af60a..274fd52f36f0d6522ecb1e1a3f677e637126086d 100644 --- a/testsuite/reftests/reftest-compare.c +++ b/testsuite/reftests/reftest-compare.c @@ -83,12 +83,16 @@ buffer_diff_core (const guchar *buf_a, const guchar *buf_b, int stride_b, int width, - int height) + int height, + guint *max_diff_out, + guint *pixels_changed_out) { int x, y; guchar *buf_diff = NULL; int stride_diff = 0; cairo_surface_t *diff = NULL; + guint max_diff = 0; + guint pixels_changed = 0; for (y = 0; y < height; y++) { @@ -121,17 +125,23 @@ buffer_diff_core (const guchar *buf_a, { int value_a = (row_a[x] >> (channel*8)) & 0xff; int value_b = (row_b[x] >> (channel*8)) & 0xff; - guint channel_difff; - - channel_difff = ABS (value_a - value_b); - channel_difff *= 4; /* emphasize */ - if (channel_difff) - channel_difff += 128; /* make sure it's visible */ - if (channel_difff > 255) - channel_difff = 255; - diff_pixel |= channel_difff << (channel * 8); + guint channel_diff; + + channel_diff = ABS (value_a - value_b); + + if (channel_diff > max_diff) + max_diff = channel_diff; + + channel_diff *= 4; /* emphasize */ + if (channel_diff) + channel_diff += 128; /* make sure it's visible */ + if (channel_diff > 255) + channel_diff = 255; + diff_pixel |= channel_diff << (channel * 8); } + pixels_changed++; + if ((diff_pixel & 0x00ffffff) == 0) { /* alpha only difference, convert to luminance */ @@ -143,12 +153,21 @@ buffer_diff_core (const guchar *buf_a, } } + if (max_diff_out != NULL) + *max_diff_out = max_diff; + + if (pixels_changed_out != NULL) + *pixels_changed_out = pixels_changed; + return diff; } cairo_surface_t * reftest_compare_surfaces (cairo_surface_t *surface1, - cairo_surface_t *surface2) + cairo_surface_t *surface2, + guint *max_diff_out, + guint *pixels_changed_out, + guint *pixels_out) { int w1, h1, w2, h2, w, h; cairo_surface_t *coerced1, *coerced2, *diff; @@ -164,11 +183,14 @@ reftest_compare_surfaces (cairo_surface_t *surface1, cairo_image_surface_get_stride (coerced1), cairo_image_surface_get_data (coerced2), cairo_image_surface_get_stride (coerced2), - w, h); + w, h, max_diff_out, pixels_changed_out); cairo_surface_destroy (coerced1); cairo_surface_destroy (coerced2); + if (pixels_out != NULL) + *pixels_out = w * h; + return diff; } diff --git a/testsuite/reftests/reftest-compare.h b/testsuite/reftests/reftest-compare.h index 551b1c5a9291002a396b2049bede8aef44df9b0e..c6e001c505ba6f126c76f4c3f8621189cd2cbf2c 100644 --- a/testsuite/reftests/reftest-compare.h +++ b/testsuite/reftests/reftest-compare.h @@ -24,7 +24,10 @@ G_BEGIN_DECLS G_MODULE_EXPORT cairo_surface_t * reftest_compare_surfaces (cairo_surface_t *surface1, - cairo_surface_t *surface2); + cairo_surface_t *surface2, + guint *max_diff_out, + guint *pixels_changed_out, + guint *pixels_out); G_END_DECLS