Commit 73c0decd authored by Martyn Russell's avatar Martyn Russell

tests: Updated gtester scripts to improve testing system

We imported Makefile.decl from glib when it was using Subversion, a long time
ago. The script had a few issues (like running all unit tests twice on make
distcheck) and needed to be updated.

I've stolen a latest version from GLib's glib.mk and also their m4 macros
which are required to make this all work too.

This gives us the advantage of defining installable and uninstallable test
cases as well as test data, scripts and other useful stuff.

The debug output also looks quite good.
And distcheck passes :)
parent f9ab05df
......@@ -71,7 +71,7 @@ gen-ChangeLog:
functional-test:
cd tests/functional-tests/ && $(MAKE) $(AM_MAKEFLAGS) $@
EXTRA_DIST = \
EXTRA_DIST += \
ChangeLog.pre-0-6-93 \
gitlog-to-changelog \
intltool-extract.in \
......@@ -79,7 +79,7 @@ EXTRA_DIST = \
intltool-update.in \
tracker.doap
DISTCLEANFILES = \
DISTCLEANFILES += \
intltool-extract \
intltool-merge \
intltool-update
......
# Stolen from: https://git.gnome.org/browse/glib/tree/glib.mk
GTESTER = gtester # for non-GLIB packages
GTESTER_REPORT = gtester-report # for non-GLIB packages
#GTESTER = $(top_builddir)/glib/gtester # for the GLIB package
#GTESTER_REPORT = $(top_builddir)/glib/gtester-report # for the GLIB package
NULL =
# initialize variables for unconditional += appending
BUILT_SOURCES =
BUILT_EXTRA_DIST =
CLEANFILES = *.log *.trs
DISTCLEANFILES =
MAINTAINERCLEANFILES =
EXTRA_DIST =
TEST_PROGS =
installed_test_LTLIBRARIES =
installed_test_PROGRAMS =
installed_test_SCRIPTS =
nobase_installed_test_DATA =
noinst_LTLIBRARIES =
noinst_PROGRAMS =
noinst_SCRIPTS =
noinst_DATA =
check_LTLIBRARIES =
check_PROGRAMS =
check_SCRIPTS =
check_DATA =
TESTS =
# test-nonrecursive: run tests only in cwd
#if OS_UNIX
test-nonrecursive: ${TEST_PROGS}
@test -z "${TEST_PROGS}" || G_TEST_SRCDIR="$(abs_srcdir)" G_TEST_BUILDDIR="$(abs_builddir)" G_DEBUG=gc-friendly MALLOC_CHECK_=2 MALLOC_PERTURB_=$$(($${RANDOM:-256} % 256)) ${GTESTER} --verbose ${TEST_PROGS}
#else
test-nonrecursive:
#endif
.PHONY: test-nonrecursive
.PHONY: lcov genlcov lcov-clean
# use recursive makes in order to ignore errors during check
lcov:
-$(MAKE) $(AM_MAKEFLAGS) -k check
$(MAKE) $(AM_MAKEFLAGS) genlcov
# we have to massage the lcov.info file slightly to hide the effect of libtool
# placing the objects files in the .libs/ directory separate from the *.c
# we also have to delete tests/.libs/libmoduletestplugin_*.gcda
genlcov:
$(AM_V_GEN) rm -f $(top_builddir)/tests/.libs/libmoduletestplugin_*.gcda; \
$(LTP) --quiet --directory $(top_builddir) --capture --output-file glib-lcov.info --test-name GLIB_PERF --no-checksum --compat-libtool --ignore-errors source; \
$(LTP) --quiet --output-file glib-lcov.info --remove glib-lcov.info docs/reference/\* /tmp/\* gio/tests/gdbus-object-manager-example/\* ; \
LANG=C $(LTP_GENHTML) --quiet --prefix $(top_builddir) --output-directory glib-lcov --title "GLib Code Coverage" --legend --frames --show-details glib-lcov.info --ignore-errors source
@echo "file://$(abs_top_builddir)/glib-lcov/index.html"
lcov-clean:
if test -n "$(LTP)"; then \
$(LTP) --quiet --directory $(top_builddir) -z; \
fi
# run tests in cwd as part of make check
check-local: test-nonrecursive
# We support a fairly large range of possible variables. It is expected that all types of files in a test suite
# will belong in exactly one of the following variables.
#
# Testing rules for make
# Original file: http://svn.gnome.org/svn/glib/trunk/Makefile.decl
# First, we support the usual automake suffixes, but in lowercase, with the customary meaning:
#
# test: run all tests in cwd and subdirs
# test-report: run tests in subdirs and generate report
# perf-report: run tests in subdirs with -m perf and generate report
# full-report: like test-report: with -m perf and -m slow
# test_programs, test_scripts, test_data, test_ltlibraries
#
# The above are used to list files that are involved in both uninstalled and installed testing. The
# test_programs and test_scripts are taken to be actual testcases and will be run as part of the test suite.
# Note that _data is always used with the nobase_ automake variable name to ensure that installed test data is
# installed in the same way as it appears in the package layout.
#
# In order to mark a particular file as being only for one type of testing, use 'installed' or 'uninstalled',
# like so:
#
# installed_test_programs, uninstalled_test_programs
# installed_test_scripts, uninstalled_test_scripts
# installed_test_data, uninstalled_test_data
# installed_test_ltlibraries, uninstalled_test_ltlibraries
#
# Additionally, we support 'extra' infixes for programs and scripts. This is used for support programs/scripts
# that should not themselves be run as testcases (but exist to be used from other testcases):
#
# test_extra_programs, installed_test_extra_programs, uninstalled_test_extra_programs
# test_extra_scripts, installed_test_extra_scripts, uninstalled_test_extra_scripts
#
# Additionally, for _scripts and _data, we support the customary dist_ prefix so that the named script or data
# file automatically end up in the tarball.
#
# dist_test_scripts, dist_test_data, dist_test_extra_scripts
# dist_installed_test_scripts, dist_installed_test_data, dist_installed_test_extra_scripts
# dist_uninstalled_test_scripts, dist_uninstalled_test_data, dist_uninstalled_test_extra_scripts
#
# Note that no file is automatically disted unless it appears in one of the dist_ variables. This follows the
# standard automake convention of not disting programs scripts or data by default.
#
# test_programs, test_scripts, uninstalled_test_programs and uninstalled_test_scripts (as well as their disted
# variants) will be run as part of the in-tree 'make check'. These are all assumed to be runnable under
# gtester. That's a bit strange for scripts, but it's possible.
extractmodulesdir = $(libdir)/tracker-$(TRACKER_API_VERSION)/extract-modules
extractrulesdir = $(datadir)/tracker/extract-rules
# we use test -z "$(TEST_PROGS)" above, so make sure we have no extra whitespace...
TEST_PROGS += $(strip $(test_programs) $(test_scripts) $(uninstalled_test_programs) $(uninstalled_test_scripts) \
$(dist_test_scripts) $(dist_uninstalled_test_scripts))
GTESTER = gtester # for non-GLIB packages
GTESTER_REPORT = gtester-report
#if OS_WIN32
#TESTS += $(test_programs) $(test_scripts) $(uninstalled_test_programs) $(uninstalled_test_scripts) \
# $(dist_test_scripts) $(dist_uninstalled_test_scripts)
#endif
# initialize variables for unconditional += appending
TEST_PROGS =
# Note: build even the installed-only targets during 'make check' to ensure that they still work.
# We need to do a bit of trickery here and manage disting via EXTRA_DIST instead of using dist_ prefixes to
# prevent automake from mistreating gmake functions like $(wildcard ...) and $(addprefix ...) as if they were
# filenames, including removing duplicate instances of the opening part before the space, eg. '$(addprefix'.
all_test_programs = $(test_programs) $(uninstalled_test_programs) $(installed_test_programs) \
$(test_extra_programs) $(uninstalled_test_extra_programs) $(installed_test_extra_programs)
all_test_scripts = $(test_scripts) $(uninstalled_test_scripts) $(installed_test_scripts) \
$(test_extra_scripts) $(uninstalled_test_extra_scripts) $(installed_test_extra_scripts)
all_dist_test_scripts = $(dist_test_scripts) $(dist_uninstalled_test_scripts) $(dist_installed_test_scripts) \
$(dist_test_extra_scripts) $(dist_uninstalled_test_extra_scripts) $(dist_installed_test_extra_scripts)
all_test_scripts += $(all_dist_test_scripts)
EXTRA_DIST += $(all_dist_test_scripts)
all_test_data = $(test_data) $(uninstalled_test_data) $(installed_test_data)
all_dist_test_data = $(dist_test_data) $(dist_uninstalled_test_data) $(dist_installed_test_data)
all_test_data += $(all_dist_test_data)
EXTRA_DIST += $(all_dist_test_data)
all_test_ltlibs = $(test_ltlibraries) $(uninstalled_test_ltlibraries) $(installed_test_ltlibraries)
if ENABLE_ALWAYS_BUILD_TESTS
noinst_LTLIBRARIES += $(all_test_ltlibs)
noinst_PROGRAMS += $(all_test_programs)
noinst_SCRIPTS += $(all_test_scripts)
noinst_DATA += $(all_test_data)
else
check_LTLIBRARIES += $(all_test_ltlibs)
check_PROGRAMS += $(all_test_programs)
check_SCRIPTS += $(all_test_scripts)
check_DATA += $(all_test_data)
endif
if ENABLE_INSTALLED_TESTS
installed_test_PROGRAMS += $(test_programs) $(installed_test_programs) \
$(test_extra_programs) $(installed_test_extra_programs)
installed_test_SCRIPTS += $(test_scripts) $(installed_test_scripts) \
$(test_extra_scripts) $(test_installed_extra_scripts)
installed_test_SCRIPTS += $(dist_test_scripts) $(dist_test_extra_scripts) \
$(dist_installed_test_scripts) $(dist_installed_test_extra_scripts)
nobase_installed_test_DATA += $(test_data) $(installed_test_data)
nobase_installed_test_DATA += $(dist_test_data) $(dist_installed_test_data)
installed_test_LTLIBRARIES += $(test_ltlibraries) $(installed_test_ltlibraries)
installed_testcases = $(test_programs) $(installed_test_programs) \
$(test_scripts) $(installed_test_scripts) \
$(dist_test_scripts) $(dist_installed_test_scripts)
installed_test_meta_DATA = $(installed_testcases:=.test)
%.test: %$(EXEEXT) Makefile
$(AM_V_GEN) (echo '[Test]' > $@.tmp; \
echo 'Type=session' >> $@.tmp; \
echo 'Exec=$(installed_testdir)/$(notdir $<)' >> $@.tmp; \
mv $@.tmp $@)
### testing rules
# test: run all tests in cwd and subdirs
test: ${TEST_PROGS} all
@ test -z "${TEST_PROGS}" || ${GTESTER} --verbose ${TEST_PROGS}
@ for subdir in $(SUBDIRS) . ; do \
test "$$subdir" = "." -o "$$subdir" = "po" || \
( cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) $@ ) || exit $? ; \
done
# test-report: run tests in subdirs and generate report
# perf-report: run tests in subdirs with -m perf and generate report
# full-report: like test-report: with -m perf and -m slow
test-report perf-report full-report: ${TEST_PROGS}
@test -z "${TEST_PROGS}" || { \
case $@ in \
test-report) test_options="-k";; \
perf-report) test_options="-k -m=perf";; \
full-report) test_options="-k -m=perf -m=slow";; \
esac ; \
if test -z "$$GTESTER_LOGDIR" ; then \
${GTESTER} --verbose $$test_options -o test-report.xml ${TEST_PROGS} ; \
elif test -n "${TEST_PROGS}" ; then \
${GTESTER} --verbose $$test_options -o `mktemp "$$GTESTER_LOGDIR/log-XXXXXX"` ${TEST_PROGS} ; \
fi ; \
}
@ ignore_logdir=true ; \
if test -z "$$GTESTER_LOGDIR" ; then \
GTESTER_LOGDIR=`mktemp -d "\`pwd\`/.testlogs-XXXXXX"`; export GTESTER_LOGDIR ; \
ignore_logdir=false ; \
fi ; \
for subdir in $(SUBDIRS) . ; do \
test "$$subdir" = "." -o "$$subdir" = "po" || \
( cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) $@ ) || exit $? ; \
done ; \
$$ignore_logdir || { \
echo '<?xml version="1.0"?>' > $@.xml ; \
echo '<report-collection>' >> $@.xml ; \
for lf in `ls -L "$$GTESTER_LOGDIR"/.` ; do \
sed '1,1s/^<?xml\b[^>?]*?>//' <"$$GTESTER_LOGDIR"/"$$lf" >> $@.xml ; \
done ; \
echo >> $@.xml ; \
echo '</report-collection>' >> $@.xml ; \
rm -rf "$$GTESTER_LOGDIR"/ ; \
${GTESTER_REPORT} --version 2>/dev/null 1>&2 ; test "$$?" != 0 || ${GTESTER_REPORT} $@.xml >$@.html ; \
}
.PHONY: test test-report perf-report full-report
# run make test as part of make check
check-local: test
CLEANFILES += $(installed_test_meta_DATA)
endif
......@@ -182,3 +182,36 @@ AC_DEFUN([IDT_COMPILE_WARNINGS],[
WARN_VALACFLAGS="$warning_valaflags"
AC_SUBST(WARN_VALACFLAGS)
]) dnl IDT_COMPILE_WARNINGS
dnl Stolen from https://git.gnome.org/browse/glib/tree/m4macros/glibtests.m4
dnl GLIB_TESTS
dnl
AC_DEFUN([GLIB_TESTS],
[
AC_ARG_ENABLE(installed-tests,
AS_HELP_STRING([--enable-installed-tests],
[Enable installation of some test cases]),
[case ${enableval} in
yes) ENABLE_INSTALLED_TESTS="1" ;;
no) ENABLE_INSTALLED_TESTS="" ;;
*) AC_MSG_ERROR([bad value ${enableval} for --enable-installed-tests]) ;;
esac])
AM_CONDITIONAL([ENABLE_INSTALLED_TESTS], test "$ENABLE_INSTALLED_TESTS" = "1")
AC_ARG_ENABLE(always-build-tests,
AS_HELP_STRING([--enable-always-build-tests],
[Enable always building tests during 'make all']),
[case ${enableval} in
yes) ENABLE_ALWAYS_BUILD_TESTS="1" ;;
no) ENABLE_ALWAYS_BUILD_TESTS="" ;;
*) AC_MSG_ERROR([bad value ${enableval} for --enable-always-build-tests]) ;;
esac])
AM_CONDITIONAL([ENABLE_ALWAYS_BUILD_TESTS], test "$ENABLE_ALWAYS_BUILD_TESTS" = "1")
if test "$ENABLE_INSTALLED_TESTS" = "1"; then
AC_SUBST(installed_test_metadir, [${datadir}/installed-tests/]AC_PACKAGE_NAME)
AC_SUBST(installed_testdir, [${libexecdir}/installed-tests/]AC_PACKAGE_NAME)
fi
])
......@@ -113,6 +113,7 @@ AC_PROG_INSTALL
AC_PROG_MAKE_SET
GLIB_GSETTINGS
GLIB_TESTS
# Initialize libtool
LT_PREREQ([2.2])
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = dbus languages icons ontologies miners gschemas
# First we fix the VERSION and libexecdir in the desktop files
......
include $(top_srcdir)/Makefile.decl
configdir = $(datadir)/tracker
config_DATA = \
......
include $(top_srcdir)/Makefile.decl
gsettings_ENUM_NAMESPACE = org.freedesktop.Tracker
gsettings_ENUM_FILES = $(top_srcdir)/src/libtracker-common/tracker-enums.h
......
include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/16x16/apps
icon_DATA = # Empty
......
include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/22x22/apps
icon_DATA = # Empty
......
include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/24x24/apps
icon_DATA = # Empty
......
include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/32x32/apps
icon_DATA = # Empty
......
include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/48x48/apps
icon_DATA = # Empty
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = 16x16 22x22 24x24 32x32 48x48 scalable
gtk_update_icon_cache = gtk-update-icon-cache -f -t $(datadir)/icons/hicolor
......
include $(top_srcdir)/Makefile.decl
icondir = $(datadir)/icons/hicolor/scalable/apps
icon_DATA = # Empty
......
include $(top_srcdir)/Makefile.decl
configdir = $(datadir)/tracker/languages
config_DATA = \
......
include $(top_srcdir)/Makefile.decl
desktop_in_in_files = \
tracker-miner-extract.desktop.in.in \
tracker-miner-userguides.desktop.in.in \
......
include $(top_srcdir)/Makefile.decl
maemo_ontology = \
91-maemo.ontology 91-maemo.description
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = manpages design tools reference
# require gtk-doc when making dist
......
include $(top_srcdir)/Makefile.decl
architecture_docs = \
tracker-store.dia \
tracker-miner-fs.dia \
tracker-structural-view.dot
EXTRA_DIST = $(architecture_docs)
include $(top_srcdir)/Makefile.decl
tp = tracker-preferences.1
tn = tracker-needle.1
tmrss = tracker-miner-rss.1
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = \
libtracker-control \
libtracker-sparql \
......
include $(top_srcdir)/Makefile.decl
AUTOMAKE_OPTIONS = 1.6
# The name of the module.
......
include $(top_srcdir)/Makefile.decl
AUTOMAKE_OPTIONS = 1.6
# The name of the module.
......
include $(top_srcdir)/Makefile.decl
AUTOMAKE_OPTIONS = 1.6
# The name of the module.
......
include $(top_srcdir)/Makefile.decl
AUTOMAKE_OPTIONS = 1.6
# Generation of ontology.png using ontology-graph
......
include $(top_srcdir)/Makefile.decl
noinst_PROGRAMS = ontology-graph ttl2sgml
AM_CPPFLAGS = \
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = \
libtracker-extract \
libtracker-miner \
......
include $(top_srcdir)/Makefile.decl
module_flags = -module -avoid-version -no-undefined
AM_CPPFLAGS = \
......
include $(top_srcdir)/Makefile.decl
noinst_PROGRAMS = tracker-miner-test
AM_CPPFLAGS = \
......
include $(top_srcdir)/Makefile.decl
noinst_PROGRAMS = async-connection class-signal
AM_CPPFLAGS = $(BUILD_CFLAGS) \
......
include $(top_srcdir)/Makefile.decl
EXTRA_DIST = \
initial-data.ttl \
rss_viewer.ui \
......
include $(top_srcdir)/Makefile.decl
if HAVE_TRACKER_FTS
libstemmer_dir = libstemmer
endif
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
$(GVDB_CFLAGS)
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
noinst_LTLIBRARIES = libtracker-bus.la
AM_VALAFLAGS = \
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
AM_VALAFLAGS = \
--includedir=libtracker-data \
--header tracker-sparql-query.h \
......
include $(top_srcdir)/Makefile.decl
noinst_LTLIBRARIES = libtracker-direct.la
AM_VALAFLAGS = \
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
# Include list of shared sources:
# Defines:
# $(libtracker_miner_monitor_sources)
......
include $(top_srcdir)/Makefile.decl
lib_LTLIBRARIES = libtracker-sparql-@TRACKER_API_VERSION@.la
AM_VALAFLAGS = \
......
include $(top_srcdir)/Makefile.decl
noinst_LTLIBRARIES = libtracker-sparql.la
libtracker_sparql_la_VALAFLAGS = \
......
include $(top_srcdir)/Makefile.decl
SUBDIRS =
if HAVE_TRACKER_MINER_FS
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-DSHAREDIR=\""$(datadir)"\" \
......
include $(top_srcdir)/Makefile.decl
SUBDIRS =
if HAVE_TRACKER_MINER_EVOLUTION
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = chrome defaults
extensiondir = $(datadir)/xul-ext/trackerfox
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = skin locale content
include $(top_srcdir)/Makefile.decl
extensiondir = $(datadir)/xul-ext/trackerfox/chrome/content
extension_DATA = \
about.xul \
......
include $(top_srcdir)/Makefile.decl
# General
localesdir = $(datadir)/xul-ext/trackerfox/chrome/locale/
......
include $(top_srcdir)/Makefile.decl
extensiondir = $(datadir)/xul-ext/trackerfox/chrome/skin
extension_DATA = \
overlay.css
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = preferences
include $(top_srcdir)/Makefile.decl
extensiondir = $(datadir)/xul-ext/trackerfox/defaults/preferences
extension_DATA = \
prefs.js
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-I$(top_srcdir)/src \
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = chrome defaults
extensiondir = $(datadir)/xul-ext/trackerbird
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = skin locale content
include $(top_srcdir)/Makefile.decl
extensiondir = $(datadir)/xul-ext/trackerbird/chrome/content
extension_DATA = \
mailstore.js \
......
include $(top_srcdir)/Makefile.decl
# General
localesdir = $(datadir)/xul-ext/trackerbird/chrome/locale/
......
include $(top_srcdir)/Makefile.decl
extensiondir = $(datadir)/xul-ext/trackerbird/chrome/skin
extension_DATA = \
overlay.css
......
include $(top_srcdir)/Makefile.decl
SUBDIRS = preferences
include $(top_srcdir)/Makefile.decl
extensiondir = $(datadir)/xul-ext/trackerbird/defaults/preferences
extension_DATA = \
prefs.js
......
include $(top_srcdir)/Makefile.decl
AM_CPPFLAGS = \
$(BUILD_CFLAGS) \
-DLOCALEDIR=\""$(localedir)"\" \
......