From 88bac46287c648a5bddbe3ac862d44c0038f05dc Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 20 Dec 2018 17:45:46 +0000 Subject: [PATCH 1/7] gtestutils: Add regression test for combining -s with --tap Signed-off-by: Simon McVittie --- glib/tests/testing-helper.c | 10 ++++++++++ glib/tests/testing.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/glib/tests/testing-helper.c b/glib/tests/testing-helper.c index b0e1f4a989..c472778a51 100644 --- a/glib/tests/testing-helper.c +++ b/glib/tests/testing-helper.c @@ -101,6 +101,16 @@ main (int argc, g_test_add_func ("/incomplete", test_incomplete); g_test_add_func ("/fail", test_fail); } + else if (g_strcmp0 (argv1, "skip-options") == 0) + { + /* The caller is expected to skip some of these with + * -p, -s and/or --GTestSkipCount */ + g_test_add_func ("/a", test_pass); + g_test_add_func ("/b/a", test_pass); + g_test_add_func ("/b/b", test_pass); + g_test_add_func ("/c/a", test_pass); + g_test_add_func ("/d/a", test_pass); + } else { g_assert_not_reached (); diff --git a/glib/tests/testing.c b/glib/tests/testing.c index 03b9ca2c4d..225f29941a 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -1063,6 +1063,39 @@ test_tap (void) g_assert_no_error (error); g_ptr_array_unref (argv); + + g_test_message ("-s"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/a"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/b"); + g_ptr_array_add (argv, "-s"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "1..5\n")); + g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP by request")); + /* "-s /b" would skip a test named exactly /b, but not a test named + * /b/anything */ + g_assert_nonnull (strstr (output, "\nok 2 /b/a\n")); + g_assert_nonnull (strstr (output, "\nok 3 /b/b\n")); + g_assert_nonnull (strstr (output, "\nok 4 /c/a # SKIP by request")); + g_assert_nonnull (strstr (output, "\nok 5 /d/a\n")); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); } int -- GitLab From 14082191e8e85ad0209a76febf662bf3a7e6825a Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 4 Jan 2019 12:29:06 +0000 Subject: [PATCH 2/7] gtestutils: Make --tap compatible with --GTestSkipCount The undocumented --GTestSkipCount option is internal to the deprecated gtester tool and rather obscure, but it's straightforward to support by making G_TEST_LOG_SKIP_CASE produce TAP output similar to what already happened when we emitted G_TEST_LOG_STOP_CASE with result G_TEST_RUN_SKIPPED. I might as well do that while I'm looking at the interaction between the --tap, -p and -s options. Signed-off-by: Simon McVittie --- glib/gtestutils.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/glib/gtestutils.c b/glib/gtestutils.c index edf5ba1b5e..2418ac6da3 100644 --- a/glib/gtestutils.c +++ b/glib/gtestutils.c @@ -994,6 +994,10 @@ g_test_log (GTestLogType lbit, if (result == G_TEST_RUN_SKIPPED || result == G_TEST_RUN_INCOMPLETE) test_skipped_count++; break; + case G_TEST_LOG_SKIP_CASE: + if (test_tap_log) + g_print ("ok %d %s # SKIP\n", test_run_count, string1); + break; case G_TEST_LOG_MIN_RESULT: if (test_tap_log) g_print ("# min perf: %s\n", string1); @@ -1538,10 +1542,10 @@ void /* sanity check */ if (test_tap_log) { - if (test_paths || test_startup_skip_count) + if (test_paths) { /* Not invoking every test (even if SKIPped) breaks the "1..XX" plan */ - g_printerr ("%s: -p and --GTestSkipCount options are incompatible with --tap\n", + g_printerr ("%s: -p option is incompatible with --tap\n", (*argv)[0]); exit (1); } -- GitLab From efa56aa9579b2fc5412c11ebe5fe7f0586a0bfc7 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 4 Jan 2019 12:59:35 +0000 Subject: [PATCH 3/7] gtestutils: Test the combination of --tap and --GTestSkipCount Signed-off-by: Simon McVittie --- glib/tests/testing.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index 225f29941a..a2804cdd83 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -1064,6 +1064,33 @@ test_tap (void) g_ptr_array_unref (argv); + g_test_message ("--GTestSkipCount"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "--GTestSkipCount"); + g_ptr_array_add (argv, "2"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "1..5\n")); + g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP\n")); + g_assert_nonnull (strstr (output, "\nok 2 /b/a # SKIP\n")); + g_assert_nonnull (strstr (output, "\nok 3 /b/b\n")); + g_assert_nonnull (strstr (output, "\nok 4 /c/a\n")); + g_assert_nonnull (strstr (output, "\nok 5 /d/a\n")); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + g_test_message ("-s"); argv = g_ptr_array_new (); g_ptr_array_add (argv, (char *) testing_helper); -- GitLab From b24cdffd5cb0954e7b2dd24f6d65058fec7f9c83 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 4 Jan 2019 12:53:04 +0000 Subject: [PATCH 4/7] gtestutils: Allow combining --tap with -p The -p option is documented, and can be used to select and repeat test-cases. This is particularly useful when debugging a single failure among a large number of test-cases, or when debugging a test-case that you suspect influences another test-case by leaking global state. Until now, -p was only supported with GLib's default (GLib-specific) textual output format, and not with the standardized TAP format that we are now encouraging. If we are considering making TAP the new default (see glib#1619) it should get feature-equivalence with the current default. Because -p allows test-cases to be re-ordered and repeated, and an entry in the test_paths list can match any number of test-cases (including zero), we don't know ahead of time how many test-cases we are going to run. TAP allows the "plan" to be deferred to the end, exactly to support situations like this. Signed-off-by: Simon McVittie --- glib/gtestutils.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/glib/gtestutils.c b/glib/gtestutils.c index 2418ac6da3..9d92bf74e1 100644 --- a/glib/gtestutils.c +++ b/glib/gtestutils.c @@ -942,17 +942,24 @@ g_test_log (GTestLogType lbit, case G_TEST_LOG_START_SUITE: if (test_tap_log) { + /* We only print the TAP "plan" (1..n) ahead of time if we did + * not use the -p option to select specific tests to be run. */ if (string1[0] != 0) g_print ("# Start of %s tests\n", string1); - else + else if (test_paths == NULL) g_print ("1..%d\n", test_count); } break; case G_TEST_LOG_STOP_SUITE: if (test_tap_log) { + /* If we didn't print the TAP "plan" at the beginning because + * we were using -p, we need to print how many tests we ran at + * the end instead. */ if (string1[0] != 0) g_print ("# End of %s tests\n", string1); + else if (test_paths != NULL) + g_print ("1..%d\n", test_run_count); } break; case G_TEST_LOG_STOP_CASE: @@ -1539,18 +1546,6 @@ void test_tmpdir = g_getenv ("G_TEST_TMPDIR"); } - /* sanity check */ - if (test_tap_log) - { - if (test_paths) - { - /* Not invoking every test (even if SKIPped) breaks the "1..XX" plan */ - g_printerr ("%s: -p option is incompatible with --tap\n", - (*argv)[0]); - exit (1); - } - } - /* verify GRand reliability, needed for reliable seeds */ if (1) { -- GitLab From bcee67e29c09e375eca6d6616f927493a6fe1dd2 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Fri, 4 Jan 2019 13:00:08 +0000 Subject: [PATCH 5/7] gtestutils: Test the combination of --tap and -p Signed-off-by: Simon McVittie --- glib/tests/testing.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index a2804cdd83..bd82ff7dd5 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -1091,6 +1091,36 @@ test_tap (void) g_free (output); g_ptr_array_unref (argv); + g_test_message ("-p"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/c/a"); + g_ptr_array_add (argv, "-p"); + g_ptr_array_add (argv, "/b"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "\nok 1 /b/a\n")); + g_assert_nonnull (strstr (output, "\nok 2 /b/b\n")); + g_assert_nonnull (strstr (output, "\nok 3 /c/a\n")); + g_assert_nonnull (strstr (output, "\nok 4 /c/a\n")); + g_assert_nonnull (strstr (output, "\n1..4\n")); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + g_test_message ("-s"); argv = g_ptr_array_new (); g_ptr_array_add (argv, (char *) testing_helper); -- GitLab From dfec3c67891745130c5babca1300a180b6f9d059 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 29 Apr 2019 16:02:09 +0100 Subject: [PATCH 6/7] Test what happens when GTestSkipCount is 0 or more than number of tests Using --GTestSkipCount 0 is the same as omitting it. A skip count greater than the number of tests is the same as equalling the number of tests: they are all skipped. Signed-off-by: Simon McVittie --- glib/tests/testing.c | 54 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index bd82ff7dd5..99be65a448 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -1091,6 +1091,60 @@ test_tap (void) g_free (output); g_ptr_array_unref (argv); + g_test_message ("--GTestSkipCount=0 is the same as omitting it"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "--GTestSkipCount"); + g_ptr_array_add (argv, "0"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "1..5\n")); + g_assert_nonnull (strstr (output, "\nok 1 /a\n")); + g_assert_nonnull (strstr (output, "\nok 2 /b/a\n")); + g_assert_nonnull (strstr (output, "\nok 3 /b/b\n")); + g_assert_nonnull (strstr (output, "\nok 4 /c/a\n")); + g_assert_nonnull (strstr (output, "\nok 5 /d/a\n")); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + + g_test_message ("--GTestSkipCount > number of tests skips all"); + argv = g_ptr_array_new (); + g_ptr_array_add (argv, (char *) testing_helper); + g_ptr_array_add (argv, "skip-options"); + g_ptr_array_add (argv, "--tap"); + g_ptr_array_add (argv, "--GTestSkipCount"); + g_ptr_array_add (argv, "6"); + g_ptr_array_add (argv, NULL); + + g_spawn_sync (NULL, (char **) argv->pdata, NULL, + G_SPAWN_STDERR_TO_DEV_NULL, + NULL, NULL, &output, NULL, &status, + &error); + g_assert_no_error (error); + g_assert_nonnull (strstr (output, "1..5\n")); + g_assert_nonnull (strstr (output, "\nok 1 /a # SKIP\n")); + g_assert_nonnull (strstr (output, "\nok 2 /b/a # SKIP\n")); + g_assert_nonnull (strstr (output, "\nok 3 /b/b # SKIP\n")); + g_assert_nonnull (strstr (output, "\nok 4 /c/a # SKIP\n")); + g_assert_nonnull (strstr (output, "\nok 5 /d/a # SKIP\n")); + + g_spawn_check_exit_status (status, &error); + g_assert_no_error (error); + + g_free (output); + g_ptr_array_unref (argv); + g_test_message ("-p"); argv = g_ptr_array_new (); g_ptr_array_add (argv, (char *) testing_helper); -- GitLab From 48cc3abede1a6a42d5242cee56a8f3bdae3e2419 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 29 Apr 2019 16:12:02 +0100 Subject: [PATCH 7/7] testing: Comment that test order with -p is reversed Signed-off-by: Simon McVittie --- glib/tests/testing.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glib/tests/testing.c b/glib/tests/testing.c index 99be65a448..28dfd86322 100644 --- a/glib/tests/testing.c +++ b/glib/tests/testing.c @@ -1163,6 +1163,8 @@ test_tap (void) NULL, NULL, &output, NULL, &status, &error); g_assert_no_error (error); + /* FIXME: Tests are run in an order that is the reverse of the order + * of -p options. */ g_assert_nonnull (strstr (output, "\nok 1 /b/a\n")); g_assert_nonnull (strstr (output, "\nok 2 /b/b\n")); g_assert_nonnull (strstr (output, "\nok 3 /c/a\n")); -- GitLab