Commit aa05b66a authored by Øyvind Kolås's avatar Øyvind Kolås

tests: Add performance tracking framework

This adds a performance tracking framework that can run a set of tests over
specified git revisions. The ruby script for generating the reports comes from
similar performance tracking in GEGL. The framework permits evaluating new
tests against older version of clutter.

The tests themselves go through a few hoops for disabling framerate limiting in
both mesa and clutter.

When running make check the tests will be run and lines of the form:

@ test-state: 40.51 fps

will be left in the output, a script can scrape these lines out of a build log
on a buildbot to in other ways track performance.
parent fa1350b8
......@@ -32,4 +32,7 @@ gcov:
test-report full-report:
$(MAKE) -C tests/conform $(@)
.PHONY: gcov test-report full-report
perf-report:
$(MAKE) -C tests/performance $(@)
.PHONY: gcov test-report full-report perf-report
......@@ -949,6 +949,7 @@ AC_CONFIG_FILES([
tests/interactive/Makefile
tests/interactive/wrapper.sh
tests/micro-bench/Makefile
tests/performance/Makefile
doc/Makefile
doc/reference/Makefile
......
SUBDIRS = accessibility data interactive micro-bench
SUBDIRS = accessibility data interactive micro-bench performance
if BUILD_TESTS
SUBDIRS += conform
endif
DIST_SUBDIRS = accessibility data conform interactive micro-bench
DIST_SUBDIRS = accessibility data conform interactive micro-bench performance
EXTRA_DIST = README
......
......@@ -4,15 +4,15 @@ The conform/ tests should be non-interactive unit-tests that verify a single
feature is behaving as documented. See conform/ADDING_NEW_TESTS for more
details.
The micro-bench/ tests should be focused perfomance test, ideally testing a
single metric. Please never forget that these tests are synthetec and if you
are using them then you understand what metric is being tested. They probably
don't reflect any real world application loads and the intention is that you
use these tests once you have already determined the crux of your problem and
need focused feedback that your changes are indeed improving matters. There is
no exit status requirements for these tests, but they should give clear
feedback as to their performance. If the framerate is the feedback metric, then
the test should forcibly enable FPS debugging.
The performance/ tests are performance tests, both focused tests testing single
metrics and larger tests. These tests are used to report one or more
performance markers for the build of Clutter. Each performance marker is picked
up from the standard output of running the tests from strings having the form
"\n@ marker-name: 42.23" where 'marker-name' and '42.23' are the key/value pairs
of a single metric. Each test can provide multiple key/value pairs. Note that
if framerate is the feedback metric the test should forcibly enable FPS
debugging itself. The file test-common.h contains utility function helping to
do fps reporting.
The interactive/ tests are any tests whose status can not be determined without
a user looking at some visual output, or providing some manual input etc. This
......
# A makefile based framework for testing performance commits in retrospect,
# based on work done by pippin@gimp.org done for GEGL, original code placed in the public domain.
SELF = Makefile-retrospect
MAKE_FLAGS = -j3 -k
CC = "ccache gcc" # if you do not have ccache replace with just gcc
PROJECT_PATH = ../../
# mute makes echoing of commands
.SILENT:
# replace sequential with random to build a random subset
all: reset sequential
#all: reset random
retry:
rm -rf reports/`cat jobs | tail -n1`*
make -f $(SELF)
prepare:
# uncomment these to make sure cpu is in high performance mode
#sudo sh -c 'echo performance > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor || true'
#sudo sh -c 'echo performance > /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor || true'
reset:
rm -rf jobs jobs
# remove checkout dir to have a full reset on each invokation
rm -rf checkout
# create clone
git clone -s $(PROJECT_PATH) checkout
mkdir reports > /dev/null 2>&1 || true
make -f $(SELF) jobs
make -f $(SELF) prepare
jobs: joblist
./makejobs.rb joblist > jobs
sequential:
for a in `cat jobs`;do make -f $(SELF) reports/$$a;done
random:
for a in `cat jobs|sort`;do make -f $(SELF) reports/$$a;done
reports/%:
# check out revision
(cd checkout; git checkout `echo $@|sed s:reports/::`)
# write header for report
git log -1 `echo $@|sed s:reports/::` > $@ || true
# clean previous build
rm -rf install; mkdir install
# build revision
(cd checkout; if [ ! -f Makefile ]; then CC=$(CC) ./autogen.sh --disable-introspection --prefix=`pwd`/../install; fi ; \
make $(MAKE_FLAGS) ; make -k install ) > $@.log 2>&1 || true
# testing
make -f Makefile-tests clean;\
make -f Makefile-tests; sync;\
make -f Makefile-tests check >> $@ || true
# update report.pdf / report.png
./create-report.rb
echo
clean:
rm -rf reports jobs report.pdf report.png checkout install
make -f Makefile-tests clean
CFILES = $(wildcard *.c)
bins = $(subst ,,$(CFILES:.c=))
all: $(bins)
%: %.c
PKG_CONFIG_PATH=install/lib/pkgconfig:$(PKG_CONFIG_PATH) $(CC) -DTESTS_DATA_DIR=\"../data/\" `pkg-config clutter-1.0 --cflags --libs` -Wall -O2 -o $@ $<
check: $(bins)
for a in $(bins); do \
LD_LIBRARY_PATH=install/lib:$(LD_LIBRARY_PATH) ./$$a;\
done
clean:
rm -f $(bins)
include $(top_srcdir)/build/autotools/Makefile.am.silent
noinst_PROGRAMS = \
test-picking \
test-text-perf \
test-state \
test-state-interactive \
test-state-hidden \
test-state-mini \
test-state-pick
INCLUDES = \
-I$(top_srcdir)/ \
-I$(top_srcdir)/clutter \
-I$(top_srcdir)/clutter/cogl \
-I$(top_builddir)/clutter \
-I$(top_builddir)/clutter/cogl
LDADD = $(top_builddir)/clutter/libclutter-@CLUTTER_SONAME_INFIX@-@CLUTTER_API_VERSION@.la
AM_CFLAGS = \
$(CLUTTER_CFLAGS) \
$(MAINTAINER_CFLAGS) \
-DG_DISABLE_SINGLE_INCLUDES \
-DTESTS_DATA_DIR=\""$(top_srcdir)/tests/data/"\"
perf-report: check
check:
for a in $(noinst_PROGRAMS);do ./$$a;done;true
AM_LDFLAGS = $(CLUTTER_LIBS)
test_picking_SOURCES = test-picking.c
test_text_perf_SOURCES = test-text-perf.c
test_state_SOURCES = test-state.c
test_state_hidden_SOURCES = test-state-hidden.c
test_state_pick_SOURCES = test-state-pick.c
test_state_interactive_SOURCES = test-state-interactive.c
test_state_mini_SOURCES = test-state-mini.c
EXTRA_DIST = Makefile-retrospect Makefile-tests create-report.rb test-common.h
-include $(top_srcdir)/build/autotools/Makefile.am.gitignore
#!/usr/bin/env ruby
#
# ruby program to generate a performance report in PDF/png over git revisions, based on work
# originally done for gegl by pippin@gimp.org, the original program is in the public domain.
require 'cairo'
def cairo_surface(w,h)
surface = Cairo::PDFSurface.new("report.pdf", w,h)
cr = Cairo::Context.new(surface)
yield(cr)
end
class Database
def initialize()
@vals = Hash.new
@runs = Array.new
@colors = [
[0,1,0, 0.8],
[0,1,1, 0.8],
[1,0,0, 0.8],
[1,0,1, 0.8],
[1,1,0, 0.8],
#[0.5,0.5,0.5,0.8],
# gray doesnt have sufficient contrast against background
[0.5,0.5,1, 0.8],
[0.5,1,0.5, 0.8],
[0.5,1,1, 0.8],
[1,0.5,0.5, 0.8],
[1,0.5,1, 0.8],
[1,1,0.5, 0.8],
[1,1,1, 0.8],
]
@width = 1800
@height = 500
@marginlx = 10
@marginrx = 180
@rgap = 40
@marginy = 10
end
def val_max(key)
max=0
@runs.each { |run|
val = @vals[key][run]
if val and val > max
max = val
end
}
max
end
def val_min(key)
min=9999990
@runs.each { |run|
val = @vals[key][run]
min = val if val and val < min
}
#min
0 # this shows the relative noise in measurements better
end
def add_run(run)
@runs = @runs + [run]
end
def add_entry(run, name, val)
if !@vals[name]
@vals[name]=Hash.new
end
# check if there is an existing value,
# and perhaps have different behaviors
# associated with
@vals[name][run] = val.to_f
end
def drawbg cr
cr.set_source_rgba(0.2, 0.2, 0.2, 1)
cr.paint
i=0
@runs.each { |run|
if i % 2 == 1
cr.move_to 1.0 * i / @runs.length * (@width - @marginlx-@marginrx) + @marginlx, 0 * (@height - @marginy*2) + @marginy
cr.line_to 1.0 * i / @runs.length * (@width - @marginlx-@marginrx) + @marginlx, 1.0 * (@height - @marginy*2) + @marginy
cr.rel_line_to(1.0 / @runs.length * (@width - @marginlx-@marginrx), 0)
cr.rel_line_to(0, -(@height - @marginy*2))
cr.set_source_rgba([0.25,0.25,0.25,1])
cr.fill
end
i+=1
}
end
def drawtext cr
i = 0
@runs.each { |run|
y = i * 10 + 20
while y > @height - @marginy
y = y - @height + @marginy + 10
end
cr.move_to 1.0 * i / @runs.length * (@width - @marginlx-@marginrx) + @marginlx, y
cr.set_source_rgba(0.6,0.6,0.6,1)
cr.show_text(run[0..6])
i+=1
}
end
def draw_limits cr, key
cr.move_to @width - @marginrx + @rgap, 20
cr.set_source_rgba(1.0, 1.0, 1.0, 1.0)
cr.show_text(" #{val_max(key)} ")
cr.move_to @width - @marginrx + @rgap, @height - @marginy
cr.show_text(" #{val_min(key)} ")
end
def draw_val cr, key, valno
min = val_min(key)
max = val_max(key)
cr.set_source_rgba(@colors[valno])
cr.move_to(@width - @marginrx + @rgap, valno * 14 + @marginy + 20)
cr.show_text(key)
cr.line_width = 2
cr.new_path
i = 0
@runs.each { |run|
val = @vals[key][run]
if val
cr.line_to 1.0 * (i+0.5) / @runs.length * (@width - @marginlx-@marginrx) + @marginlx,
(1.0 - ((val-min) * 1.0 / (max - min))) * (@height - @marginy*2) + @marginy
end
i = i + 1
}
cr.stroke
end
def create_report
cairo_surface(@width, @height) { |cr|
drawbg cr
valno = 0
@vals.each { |key, value|
draw_val cr, key, valno
valno += 1
}
drawtext cr
cr.target.write_to_png("report.png")
valno = 0
@vals.each { |key, value|
cr.show_page
drawbg cr
draw_val cr, key, valno
drawtext cr
draw_limits cr, key
valno += 1
}
}
end
end
generator = Database.new
items = File.open('jobs').each { |rev|
rev.strip!
generator.add_run(rev)
filename = "reports/" + rev;
if File.exist?(filename)
File.open(filename).each { |line|
if line =~ /^@ (.*):(.*)/
generator.add_entry(rev, $1, $2)
end
}
end
}
generator.create_report
# This file lists the commits that we want to do retrospective testing
# of, aborting the retrospective testing and adjusting this file will
# add the commit range to be tested without having to redo the initial one
# thus allowing a sparse distributed range to first be tested, with
# detailed ranges added in later.
master~500..master % 32 # every 32th commit, to provide context
# various spans of commits around which "interesting things happen"
#eeac7~1..985a4
#6c6e93d~1..3142b15
#0486c56..88b026
#732eecf..8cfb158
#b499696..b77d9a6
#ba09e9c..7bdbbe6
#b424bd7..c1878
#bc58de4..d03c3a6
#5640a6..a29623e
#6fd2663..6ec9c32
# 012e4ab..1a8d577 #
#120d759~4..2235e70
# ef8be9e25ebe77fc63055191cc48af53d731c108 - actor: Use paint volumes to always queue clipped redraws
# 5d16000 going up, and 3b78949 going down,. was a case of clipped redraws being broken
#!/usr/bin/env ruby
# this ruby script generates a chronologically sorted
res = ""
input = File.read(ARGV[0])
input = input.gsub(/#.*/, "")
input.split("\n").each {|a|
if a =~ /([^ ]*)\.\.([^ ]*) %(.*)/
res += `git log #{$1}..#{$2} | grep '^commit' | sed 's/commit //' | sed -n '0~#{$3}p'`
elsif a =~ /([^ ]*)\.\.([^ ]*)/
res += `git log #{$1}..#{$2} | grep '^commit' | sed 's/commit //'`
else
res += `echo #{a}`
end
}
all = `git log | grep '^commit' | sed 's/commit//' `
all.split("\n").reverse.each {|a|
if res.match(a.strip) != nil
puts "#{a.strip}"
end
}
#include <stdlib.h>
#include <glib.h>
#include <clutter/clutter.h>
static GTimer *testtimer = NULL;
static gint testframes = 0;
static float testmaxtime = 1.0;
/* initialize environment to be suitable for fps testing */
void clutter_perf_fps_init (void)
{
/* Force not syncing to vblank, we want free-running maximum FPS */
g_setenv ("vblank_mode", "0", FALSE);
g_setenv ("CLUTTER_VBLANK", "none", FALSE);
/* also overrride internal default FPS */
g_setenv ("CLUTTER_DEFAULT_FPS", "1000", FALSE);
if (g_getenv ("CLUTTER_PERFORMANCE_TEST_DURATION"))
testmaxtime = atof(g_getenv("CLUTTER_PERFORMANCE_TEST_DURATION"));
else
testmaxtime = 10.0;
g_random_set_seed (12345678);
}
static void perf_stage_paint_cb (ClutterStage *stage, gpointer *data);
static gboolean perf_fake_mouse_cb (gpointer stage);
void clutter_perf_fps_start (ClutterStage *stage)
{
g_signal_connect (stage, "paint", G_CALLBACK (perf_stage_paint_cb), NULL);
}
void clutter_perf_fake_mouse (ClutterStage *stage)
{
g_timeout_add (1000/60, perf_fake_mouse_cb, stage);
}
void clutter_perf_fps_report (const gchar *id)
{
g_print ("\n@ %s: %.2f fps \n",
id, testframes / g_timer_elapsed (testtimer, NULL));
}
static void perf_stage_paint_cb (ClutterStage *stage, gpointer *data)
{
if (!testtimer)
testtimer = g_timer_new ();
testframes ++;
if (g_timer_elapsed (testtimer, NULL) > testmaxtime)
{
clutter_main_quit ();
}
}
static void wrap (gfloat *value, gfloat min, gfloat max)
{
if (*value > max)
*value = min;
else if (*value < min)
*value = max;
}
static gboolean perf_fake_mouse_cb (gpointer stage)
{
ClutterEvent *event = clutter_event_new (CLUTTER_MOTION);
static ClutterInputDevice *device = NULL;
int i;
static float x = 0.0;
static float y = 0.0;
static float xd = 0.0;
static float yd = 0.0;
static gboolean inited = FALSE;
gfloat w, h;
if (!inited) /* XXX:
force clutter to do handle our motion events,
by forcibly updating the input device's state
this shoudl be possible to do in a better
manner in the future, a versioning check
will have to be added when this is possible
without a hack... and the means to do the
hack is deprecated
*/
{
ClutterEvent *event2 = clutter_event_new (CLUTTER_ENTER);
device = clutter_device_manager_get_core_device (clutter_device_manager_get_default (), CLUTTER_POINTER_DEVICE);
event2->crossing.stage = stage;
event2->crossing.source = stage;
event2->crossing.x = 10;
event2->crossing.y = 10;
event2->crossing.device = device;
event2->crossing.related = NULL;
clutter_input_device_update_from_event (device, event2, TRUE);
clutter_event_put (event2);
clutter_event_free (event2);
inited = TRUE;
}
clutter_actor_get_size (stage, &w, &h);
event->motion.stage = stage;
event->motion.device = device;
/* called about every 60fps, and do 10 picks per stage */
for (i = 0; i < 10; i++)
{
event->motion.x = x;
event->motion.y = y;
clutter_event_put (event);
x += xd;
y += yd;
xd += g_random_double_range (-0.1, 0.1);
yd += g_random_double_range (-0.1, 0.1);
wrap (&x, 0, w);
wrap (&y, 0, h);
xd = CLAMP(xd, -1.3, 1.3);
yd = CLAMP(yd, -1.3, 1.3);
}
clutter_event_free (event);
return TRUE;
}
#include <math.h>
#include <stdlib.h>
#include <clutter/clutter.h>
#include "test-common.h"
#define N_ACTORS 100
#define N_EVENTS 5
static gint n_actors = N_ACTORS;
static gint n_events = N_EVENTS;
static GOptionEntry entries[] = {
{
"num-actors", 'a',
0,
G_OPTION_ARG_INT, &n_actors,
"Number of actors", "ACTORS"
},
{
"num-events", 'e',
0,
G_OPTION_ARG_INT, &n_events,
"Number of events", "EVENTS"
},
{ NULL }
};
static gboolean
motion_event_cb (ClutterActor *actor, ClutterEvent *event, gpointer user_data)
{
return FALSE;
}
static void
do_events (ClutterActor *stage)
{
glong i;
static gdouble angle = 0;
for (i = 0; i < n_events; i++)
{
angle += (2.0 * M_PI) / (gdouble)n_actors;
while (angle > M_PI * 2.0)
angle -= M_PI * 2.0;
/* If we synthesized events, they would be motion compressed;
* calling get_actor_at_position() doesn't have that problem
*/
clutter_stage_get_actor_at_pos (CLUTTER_STAGE (stage),
CLUTTER_PICK_REACTIVE,
256.0 + 206.0 * cos (angle),
256.0 + 206.0 * sin (angle));
}
}
static gboolean queue_redraw (gpointer data)
{
ClutterActor *stage = CLUTTER_ACTOR (data);
clutter_actor_queue_redraw (stage);
do_events (stage);
return TRUE;
}
int
main (int argc, char **argv)
{
glong i;
gdouble angle;
const ClutterColor black = { 0x00, 0x00, 0x00, 0xff };
ClutterColor color = { 0x00, 0x00, 0x00, 0xff };
ClutterActor *stage, *rect;
clutter_perf_fps_init ();
if (CLUTTER_INIT_SUCCESS !=
clutter_init_with_args (&argc, &argv,
NULL,
entries,
NULL,
NULL))
{
g_warning ("Failed to initialize clutter");
return -1;
}
stage = clutter_stage_get_default ();
clutter_actor_set_size (stage, 512, 512);
clutter_stage_set_color (CLUTTER_STAGE (stage), &black);
printf ("Picking performance test with "
"%d actors and %d events per frame\n",
n_actors,
n_events);
for (i = n_actors - 1; i >= 0; i--)
{
angle = ((2.0 * M_PI) / (gdouble) n_actors) * i;
color.red = (1.0 - ABS ((MAX (0, MIN (n_actors/2.0 + 0, i))) /
(gdouble)(n_actors/4.0) - 1.0)) * 255.0;
color.green = (1.0 - ABS ((MAX (0, MIN (n_actors/2.0 + 0,
fmod (i + (n_actors/3.0)*2, n_actors)))) /
(gdouble)(n_actors/4) - 1.0)) * 255.0;
color.blue = (1.0 - ABS ((MAX (0, MIN (n_actors/2.0 + 0,
fmod ((i + (n_actors/3.0)), n_actors)))) /
(gdouble)(n_actors/4.0) - 1.0)) * 255.0;
rect = clutter_rectangle_new_with_color (&color);
clutter_actor_set_size (rect, 100, 100);
clutter_actor_set_anchor_point_from_gravity (rect,
CLUTTER_GRAVITY_CENTER);
clutter_actor_set_position (rect,
256 + 206 * cos (angle),
256 + 206 * sin (angle));
clutter_actor_set_reactive (rect, TRUE);
g_signal_connect (rect, "motion-event",
G_CALLBACK (motion_event_cb), NULL);
clutter_container_add_actor (CLUTTER_CONTAINER (stage), rect);
}
clutter_actor_show (stage);
clutter_perf_fps_start (CLUTTER_STAGE (stage));
g_idle_add (queue_redraw, (gpointer)stage);
clutter_main ();
clutter_perf_fps_report ("test-picking");
return 0;
}
#include <stdlib.h>
#include <math.h>
#include <gmodule.h>
#include <clutter/clutter.h>
#include "test-common.h"
#define STAGE_WIDTH 160
#define STAGE_HEIGHT 120
#define ACTOR_WIDTH 8
#define ACTOR_HEIGHT 8
#define COLS (STAGE_WIDTH/ACTOR_WIDTH)
#define ROWS (STAGE_HEIGHT/ACTOR_HEIGHT)
#define TOTAL (ROWS*COLS)
static void completed (ClutterState *state,
gpointer data)
{
if (g_str_equal (clutter_state_get_state (state), "right"))
{
/* skip straight to left state when reaching right */
clutter_state_warp_to_state (state, "left");
}
else if (g_str_equal (clutter_state_get_state (state), "active"))
clutter_state_set_state (state, "right");
else
{
clutter_state_set_state (state, "active");
}
}
static ClutterActor *new_rect (gint r,
gint g,
gint b,
gint a)
{
ClutterColor *color = clutter_color_new (r, g, b, a);
ClutterActor *group = clutter_group_new ();
ClutterActor *rectangle = clutter_rectangle_new_with_color (color);
gchar *file = g_build_filename (TESTS_DATA_DIR, "redhand.png", NULL);
g_free (file);
clutter_actor_set_size (rectangle, ACTOR_WIDTH,ACTOR_HEIGHT);
clutter_color_free (color);
clutter_container_add (CLUTTER_CONTAINER (group), rectangle, NULL);