Commit a46a5294 authored by Tim Janik's avatar Tim Janik

gtester: implemented logic to handle failing tests, self tests, and validate XML reports.

* gtester.c: terminate when tests failed. keep XML valid when test cases fail.
restart test binaries when tests fail, resuming after the last processed test.
support --gtester-selftest to run gtester itself as test program.
support --test-arg=<arg> to pass args along to test programs. added
main_selftest() which does a simplistic fixture test. fail if exit
code of test programs is not 0.

* gtestframework.h: added G_TEST_LOG_SKIP_CASE test log message type.

* gtestframework.c: support --GTestSkipCount=<n> to skip a number of tests.

* tests/Makefile.am: added test-report: for demonstration purposes.
added gtester-xmllint-check: and hooked it up into check:, this rule calls
gtester as test program, running it's selftest, and then uses xmllint to
validate the generate XML test log file.

svn path=/trunk/; revision=5904
parent f347900e
......@@ -23,6 +23,7 @@
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/wait.h>
#include <errno.h>
#include <signal.h>
......@@ -30,6 +31,8 @@
#define READ_BUFFER_SIZE 4096
/* --- prototypes --- */
static int main_selftest (int argc,
char **argv);
static void parse_args (gint *argc_p,
gchar ***argv_p);
......@@ -38,17 +41,21 @@ static GIOChannel *ioc_report = NULL;
static gboolean gtester_quiet = FALSE;
static gboolean gtester_verbose = FALSE;
static gboolean gtester_list_tests = FALSE;
static gboolean gtester_selftest = FALSE;
static gboolean subtest_running = FALSE;
static gint subtest_exitstatus = 0;
static gboolean subtest_io_pending = FALSE;
static gboolean subtest_faill = TRUE;
static gboolean subtest_quiet = TRUE;
static gboolean subtest_verbose = FALSE;
static gboolean subtest_mode_fatal = FALSE;
static gboolean subtest_mode_fatal = TRUE;
static gboolean subtest_mode_perf = FALSE;
static gboolean subtest_mode_quick = TRUE;
static const gchar *subtest_seedstr = NULL;
static GSList *subtest_paths = NULL;
static GSList *subtest_args = NULL;
static gboolean testcase_open = FALSE;
static guint testcase_count = 0;
static guint testcase_fail_count = 0;
static const gchar *output_filename = NULL;
static guint log_indent = 0;
static gint log_fd = -1;
......@@ -79,6 +86,33 @@ test_log_printfe (const char *format,
g_free (result);
}
static void
terminate (void)
{
kill (getpid(), SIGTERM);
abort();
}
static void
testcase_close (long double duration,
guint exit_status,
guint n_forks)
{
g_return_if_fail (testcase_open > 0);
test_log_printfe ("%s<duration>%.6Lf</duration>\n", sindent (log_indent), duration);
test_log_printfe ("%s<status exit-status=\"%d\" n-forks=\"%d\"/>\n",
sindent (log_indent), exit_status, n_forks);
log_indent -= 2;
test_log_printfe ("%s</testcase>\n", sindent (log_indent));
testcase_open--;
if (gtester_verbose)
g_print ("%s\n", exit_status ? "FAIL" : "OK");
if (exit_status)
testcase_fail_count += 1;
if (subtest_mode_fatal && testcase_fail_count)
terminate();
}
static void
test_log_msg (GTestLogMsg *msg)
{
......@@ -97,6 +131,7 @@ test_log_msg (GTestLogMsg *msg)
g_print ("%s\n", msg->strings[0]);
break;
case G_TEST_LOG_START_CASE:
testcase_count++;
if (gtester_verbose)
{
gchar *sc = g_strconcat (msg->strings[0], ":", NULL);
......@@ -110,16 +145,19 @@ test_log_msg (GTestLogMsg *msg)
test_log_printfe ("%s<testcase path=\"%s\">\n", sindent (log_indent), msg->strings[0]);
log_indent += 2;
break;
case G_TEST_LOG_SKIP_CASE:
if (TRUE && gtester_verbose) // enable to debug test case skipping logic
{
gchar *sc = g_strconcat (msg->strings[0], ":", NULL);
gchar *sleft = g_strdup_printf ("%-68s", sc);
g_free (sc);
g_print ("%70s SKIPPED\n", sleft);
g_free (sleft);
}
test_log_printfe ("%s<testcase path=\"%s\" skipped=\"1\"/>\n", sindent (log_indent), msg->strings[0]);
break;
case G_TEST_LOG_STOP_CASE:
g_return_if_fail (testcase_open > 0);
test_log_printfe ("%s<duration>%.6Lf</duration>\n", sindent (log_indent), msg->nums[2]);
test_log_printfe ("%s<status exit-status=\"%d\" n-forks=\"%d\"/>\n",
sindent (log_indent), (int) msg->nums[0], (int) msg->nums[1]);
log_indent -= 2;
test_log_printfe ("%s</testcase>\n", sindent (log_indent));
testcase_open--;
if (gtester_verbose)
g_print ("OK\n");
testcase_close (msg->nums[2], (int) msg->nums[0], (int) msg->nums[1]);
break;
case G_TEST_LOG_MIN_RESULT:
case G_TEST_LOG_MAX_RESULT:
......@@ -181,6 +219,10 @@ child_watch_cb (GPid pid,
gpointer data)
{
g_spawn_close_pid (pid);
if (WIFEXITED (status)) /* normal exit */
subtest_exitstatus = WEXITSTATUS (status);
else /* signal or core dump, etc */
subtest_exitstatus = 0xffffffff;
subtest_running = FALSE;
}
......@@ -202,12 +244,13 @@ unset_cloexec_fdp (gpointer fdp_data)
}
static gboolean
launch_test_binary (const char *binary)
launch_test_binary (const char *binary,
guint skip_tests)
{
GTestLogBuffer *tlb;
GSList *slist, *free_list = NULL;
GError *error = NULL;
const gchar *argv[20 + g_slist_length (subtest_paths)];
const gchar *argv[99 + g_slist_length (subtest_args) + g_slist_length (subtest_paths)];
GPid pid = 0;
gint report_pipe[2] = { -1, -1 };
gint i = 0;
......@@ -223,12 +266,13 @@ launch_test_binary (const char *binary)
/* setup argv */
argv[i++] = binary;
for (slist = subtest_args; slist; slist = slist->next)
argv[i++] = (gchar*) slist->data;
// argv[i++] = "--debug-log";
if (subtest_quiet)
argv[i++] = "--quiet";
if (subtest_verbose)
argv[i++] = "--verbose";
// argv[i++] = "--debug-log";
argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestLogFD=%u", report_pipe[1]));
if (!subtest_mode_fatal)
argv[i++] = "--keep-going";
if (subtest_mode_quick)
......@@ -237,12 +281,15 @@ launch_test_binary (const char *binary)
argv[i++] = "-m=slow";
if (subtest_mode_perf)
argv[i++] = "-m=perf";
if (gtester_list_tests)
argv[i++] = "-l";
if (subtest_seedstr)
argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--seed=%s", subtest_seedstr));
argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestLogFD=%u", report_pipe[1]));
if (skip_tests)
argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestSkipCount=%u", skip_tests));
for (slist = subtest_paths; slist; slist = slist->next)
argv[i++] = queue_gfree (&free_list, g_strdup_printf ("-p=%s", (gchar*) slist->data));
if (gtester_list_tests)
argv[i++] = "-l";
argv[i++] = NULL;
g_spawn_async_with_pipes (NULL, /* g_get_current_dir() */
......@@ -302,22 +349,39 @@ launch_test_binary (const char *binary)
static void
launch_test (const char *binary)
{
gboolean success;
gboolean success = TRUE;
GTimer *btimer = g_timer_new();
subtest_faill = FALSE;
gboolean need_restart;
testcase_count = 0;
testcase_fail_count = 0;
if (!gtester_quiet)
g_print ("TEST: %s... ", binary);
retry:
test_log_printfe ("%s<testbinary path=\"%s\">\n", sindent (log_indent), binary);
log_indent += 2;
g_timer_start (btimer);
success = launch_test_binary (binary);
subtest_exitstatus = 0;
success &= launch_test_binary (binary, testcase_count);
success &= subtest_exitstatus == 0;
need_restart = testcase_open != 0;
if (testcase_open)
testcase_close (0, -999, 0);
g_timer_stop (btimer);
test_log_printfe ("%s<duration>%.6f</duration>\n", sindent (log_indent), g_timer_elapsed (btimer, NULL));
log_indent -= 2;
test_log_printfe ("%s</testbinary>\n", sindent (log_indent));
if (need_restart)
{
/* restart test binary, skipping processed test cases */
goto retry;
}
if (!gtester_quiet)
g_print ("%s: %s\n", subtest_faill || !success ? "FAIL" : "PASS", binary);
g_print ("%s: %s\n", testcase_fail_count || !success ? "FAIL" : "PASS", binary);
g_timer_destroy (btimer);
if (subtest_mode_fatal && !success)
terminate();
}
static void
......@@ -361,6 +425,12 @@ parse_args (gint *argc_p,
g_log_set_always_fatal (fatal_mask);
argv[i] = NULL;
}
else if (strcmp (argv[i], "--gtester-selftest") == 0)
{
gtester_selftest = TRUE;
argv[i] = NULL;
break; // stop parsing regular gtester arguments
}
else if (strcmp (argv[i], "-h") == 0 || strcmp (argv[i], "--help") == 0)
{
usage (FALSE);
......@@ -391,6 +461,18 @@ parse_args (gint *argc_p,
}
argv[i] = NULL;
}
else if (strcmp ("--test-arg", argv[i]) == 0 || strncmp ("--test-arg=", argv[i], 11) == 0)
{
gchar *equal = argv[i] + 10;
if (*equal == '=')
subtest_args = g_slist_prepend (subtest_args, equal + 1);
else if (i + 1 < argc)
{
argv[i++] = NULL;
subtest_args = g_slist_prepend (subtest_args, argv[i]);
}
argv[i] = NULL;
}
else if (strcmp ("-o", argv[i]) == 0 || strncmp ("-o=", argv[i], 3) == 0)
{
gchar *equal = argv[i] + 2;
......@@ -491,6 +573,8 @@ main (int argc,
g_set_prgname (argv[0]);
parse_args (&argc, &argv);
if (gtester_selftest)
return main_selftest (argc, argv);
if (argc <= 1)
{
......@@ -521,3 +605,30 @@ main (int argc,
return 0;
}
static void
fixture_setup (guint *fix)
{
g_assert_cmphex (*fix, ==, 0);
*fix = 0xdeadbeef;
}
static void
fixture_test (guint *fix)
{
g_assert_cmphex (*fix, ==, 0xdeadbeef);
}
static void
fixture_teardown (guint *fix)
{
g_assert_cmphex (*fix, ==, 0xdeadbeef);
}
static int
main_selftest (int argc,
char **argv)
{
/* gtester main() for --gtester-selftest invokations */
g_test_init (&argc, &argv, NULL);
g_test_add ("/gtester/fixture-test", guint, fixture_setup, fixture_test, fixture_teardown);
return g_test_run();
}
......@@ -67,6 +67,8 @@ static gchar *test_run_seedstr = NULL;
static GRand *test_run_rand = NULL;
static gchar *test_run_name = "";
static guint test_run_forks = 0;
static guint test_run_count = 0;
static guint test_skip_count = 0;
static GTimer *test_user_timer = NULL;
static double test_user_stamp = 0;
static GSList *test_paths = NULL;
......@@ -224,6 +226,18 @@ parse_args (gint *argc_p,
}
argv[i] = NULL;
}
else if (strcmp ("--GTestSkipCount", argv[i]) == 0 || strncmp ("--GTestSkipCount=", argv[i], 17) == 0)
{
gchar *equal = argv[i] + 16;
if (*equal == '=')
test_skip_count = g_ascii_strtoull (equal + 1, NULL, 0);
else if (i + 1 < argc)
{
argv[i++] = NULL;
test_skip_count = g_ascii_strtoull (argv[i], NULL, 0);
}
argv[i] = NULL;
}
else if (strcmp ("-p", argv[i]) == 0 || strncmp ("-p=", argv[i], 3) == 0)
{
gchar *equal = argv[i] + 2;
......@@ -593,6 +607,11 @@ test_case_run (GTestCase *tc)
gchar *old_name;
old_name = test_run_name;
test_run_name = g_strconcat (old_name, "/", tc->name, NULL);
if (++test_run_count <= test_skip_count)
{
g_test_log (G_TEST_LOG_SKIP_CASE, test_run_name, NULL, 0, NULL);
return 0;
}
if (test_run_list)
{
g_print ("%s\n", test_run_name);
......
......@@ -173,6 +173,7 @@ typedef enum {
G_TEST_LOG_ERROR, // s:msg
G_TEST_LOG_START_BINARY, // s:binaryname s:seed
G_TEST_LOG_LIST_CASE, // s:testpath
G_TEST_LOG_SKIP_CASE, // s:testpath
G_TEST_LOG_START_CASE, // s:testpath
G_TEST_LOG_STOP_CASE, // d:status d:nforks d:elapsed
G_TEST_LOG_MIN_RESULT, // s:blurb d:result
......
......@@ -11,8 +11,18 @@ TEST_PROGS += testing
testing_SOURCES = testing.c
testing_LDADD = $(progs_ldadd)
# exemplary unit test rules
test:
${GTESTER} --verbose ${TEST_PROGS}
.PHONY: test
test-report:
${GTESTER} --verbose -k -o testreport.xml ${TEST_PROGS}
.PHONY: test test-report
check-local: test
# some testing of gtester funcitonality
XMLLINT=xmllint
gtester-xmllint-check: # check testreport xml with xmllint if present
${GTESTER} -k --quiet -o tmpsample.xml --test-arg=--gtester-selftest ${GTESTER}
${XMLLINT} --version 2>/dev/null; test "$$?" != 0 || ${XMLLINT} --noout tmpsample.xml
check-am: gtester-xmllint-check
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment