=== added file 'tests/test_vars.in' --- tests/test_vars.in 1970-01-01 00:00:00 +0000 +++ tests/test_vars.in 2007-04-21 06:59:27 +0000 @@ -0,0 +1,2 @@ +# defined to 1 if subunit is enabled +export ENABLE_SUBUNIT=@ENABLE_SUBUNIT@ === modified file 'AUTHORS' --- AUTHORS 2006-12-12 19:11:09 +0000 +++ AUTHORS 2007-04-21 03:18:23 +0000 @@ -14,6 +14,7 @@ Robert Lemmen (gcov description in manual) Loic Martin (AM_PATH_CHECK patch) Ross Burton (pkg-config patch) + Robert Collins (subunit support) Anybody who has contributed code to Check or Check's build system is considered an author. Send patches to this file to === modified file 'NEWS' --- NEWS 2006-11-21 23:56:21 +0000 +++ NEWS 2007-04-21 03:18:23 +0000 @@ -1,3 +1,9 @@ +In development: + +* Added CK_SUBUNIT support for outputting test information in the subunit wire + protocol. See the check manual for more information. (Contributed by Robert + Collins). + Tue, Nov 21, 2006: Released Check 0.9.5 * Fixed code coverage support to work with gcc4 and buggy libtool. === modified file 'configure.ac' --- configure.ac 2006-12-12 19:11:09 +0000 +++ configure.ac 2007-04-21 06:59:27 +0000 @@ -65,6 +65,26 @@ AM_CONDITIONAL(NO_TIMEOUT_TESTS, test x"$enable_timeout_tests" = "xfalse") +AC_ARG_ENABLE(subunit, +AC_HELP_STRING([--enable-subunit], + [enable support for the subunit test protocol @<:@default=autodetect@:>@]), +[case "${enableval}" in + yes) + enable_subunit=true + echo "Enabled subunit support" + ;; + no) + enable_subunit=false + echoo "Disabled subunit support" + ;; + autodetect) + echo "Subunit support will enable automatically." + ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-subunit) ;; +esac], +[echo "Subunit support will enable automatically." + enable_subunit=autodetect]) + # Checks for programs. AC_PROG_AWK AC_PROG_CC @@ -85,6 +105,42 @@ AC_HEADER_SYS_WAIT AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h sys/time.h unistd.h]) +if test xfalse != x"$enable_subunit"; then +AC_CHECK_LIB(subunit, subunit_test_start, , +[case "$enable_subunit" in + autodetect) + enable_subunit=false + ;; + true) + AC_MSG_ERROR([libunit is required for subunit protocol support. The homepage for subunit is https://launchpad.net/subunit/]) + ;; + esac +]) +fi +if test xfalse != x"$enable_subunit"; then +AC_CHECK_HEADER([subunit/child.h], , +[case "$enable_subunit" in + autodetect) + enable_subunit=false + ;; + true) + AC_MSG_ERROR([The header subunit/child.h could not be succesfully included and is required for subunit protocol support. The homepage for subunit is https://launchpad.net/subunit/]) + ;; + esac +]) +fi +if test xfalse = x"$enable_subunit"; then +ENABLE_SUBUNIT="0" +else +ENABLE_SUBUNIT="1" +fi +AC_SUBST(ENABLE_SUBUNIT) +AC_DEFINE_UNQUOTED(ENABLE_SUBUNIT, $ENABLE_SUBUNIT, [Subunit protocol result output]) + +AM_CONDITIONAL(SUBUNIT, test x"$enable_subunit" != "xfalse") + + + # Checks for typedefs, structures, and compiler characteristics. AC_C_CONST AC_TYPE_PID_T @@ -112,6 +168,7 @@ doc/Makefile src/check.h src/Makefile - tests/Makefile]) + tests/Makefile + tests/test_vars]) AC_OUTPUT === modified file 'doc/check.texi' --- doc/check.texi 2007-01-16 21:57:52 +0000 +++ doc/check.texi 2007-04-21 03:18:23 +0000 @@ -39,6 +39,7 @@ @author Chris Pickett @author Fredrik Hugosson @author Robert Lemmen +@author Robert Collins @c The following two commands start the copyright page. @page @@ -98,6 +99,7 @@ * Test Timeouts:: * Determining Test Coverage:: * Test Logging:: +* Subunit Support:: Test Fixtures @@ -672,6 +674,11 @@ which can have the values "silent", "minimal", "normal", "verbose". If the variable is not found or the value is not recognized, the print mode is set to @code{CK_NORMAL}. + +@vindex CK_SUBUNIT +@item CK_SUBUNIT +Prints running progress through the @uref{https://launchpad.net/subunit/, +subunit} test runner protocol. See 'subunit support' under the Advanced Features section for more information. @end table With the @code{CK_NORMAL} flag specified in our @code{main()}, let's @@ -766,6 +773,7 @@ * Test Timeouts:: * Determining Test Coverage:: * Test Logging:: +* Subunit Support:: @end menu @node Running Multiple Cases, No Fork Mode, Advanced Features, Advanced Features @@ -1147,7 +1155,7 @@ you. For more information or help with other compilers, please refer to the relevant manuals. -@node Test Logging, , Determining Test Coverage, Advanced Features +@node Test Logging, Subunit Support, Determining Test Coverage, Advanced Features @section Test Logging @findex srunner_set_log() @@ -1250,6 +1258,50 @@ @end verbatim @end example +@node Subunit Support, , Test Logging, Advanced Features +@section Subunit Support + +Check supports running test suites with subunit output. This can be useful to +combine test results from multiple languages, or to perform programmatic +analysis on the results of multiple check test suites or otherise handle test +results in a programmatic manner. Using subunit with check is very straight +forward. There are two steps: +1) In your check test suite driver pass 'CK_SUBUNIT' as the output mode +for your srunner. +@example +@verbatim +SRunner *sr; +sr = srunner_create (make_s1_suite ()); +srunner_add_suite (sr, make_s2_suite ()); +srunner_run_all (sr, CK_SUBUNIT); +@end verbatim +@end example +2) Setup your main language test runner to run your check based test +executable. For instance using python: +@example +@verbatim + +import subunit + +class ShellTests(subunit.ExecTestCase): + """Run some tests from the C codebase.""" + + def test_group_one(self): + """./foo/check_driver""" + + def test_group_two(self): + """./foo/other_driver""" +@end verbatim +@end example + +In this example, running the test suite ShellTests in python (using any test +runner - unittest.py, tribunal, trial, nose or others) will run +./foo/check_driver and ./foo/other_driver and report on their result. + +Subunit is hosted on launchpad - the @uref{https://launchpad.net/subunit/, +subunit} project there contains bug tracker, future plans, and source code +control details. + @node Conclusion and References, AM_PATH_CHECK, Advanced Features, Top @chapter Conclusion and References The tutorial and description of advanced features has provided an === modified file 'src/check.h.in' --- src/check.h.in 2006-12-08 17:47:49 +0000 +++ src/check.h.in 2007-04-21 06:59:27 +0000 @@ -257,6 +257,9 @@ CK_NORMAL, /* All failed tests */ CK_VERBOSE, /* All tests */ CK_ENV, /* Look at environment var */ +#if @ENABLE_SUBUNIT@ + CK_SUBUNIT, /* Run as a subunit child process */ +#endif CK_LAST }; === modified file 'src/check_impl.h' --- src/check_impl.h 2006-10-13 00:24:56 +0000 +++ src/check_impl.h 2007-04-21 01:55:36 +0000 @@ -89,6 +89,7 @@ CLSTART_S, CLEND_SR, CLEND_S, + CLSTART_T, /* A test case is about to run */ CLEND_T }; === modified file 'src/check_log.c' --- src/check_log.c 2006-10-13 04:10:50 +0000 +++ src/check_log.c 2007-04-21 06:59:27 +0000 @@ -25,12 +25,16 @@ #include #include #include +#if HAVE_SUBUNIT_CHILD_H +#include +#endif #include "check_error.h" #include "check_list.h" #include "check_impl.h" #include "check_log.h" #include "check_print.h" +#include "check_str.h" static void srunner_send_evt (SRunner *sr, void *obj, enum cl_event evt); @@ -107,6 +111,13 @@ srunner_send_evt (sr, s, CLEND_S); } +void log_test_start (SRunner *sr, TCase * tc, TF * tfun) +{ + char buffer[100]; + snprintf(buffer, 99, "%s:%s", tc->name, tfun->name); + srunner_send_evt (sr, buffer, CLSTART_T); +} + void log_test_end (SRunner *sr, TestResult *tr) { srunner_send_evt (sr, tr, CLEND_T); @@ -128,7 +139,6 @@ void stdout_lfun (SRunner *sr, FILE *file, enum print_output printmode, void *obj, enum cl_event evt) { - TestResult *tr; Suite *s; if (printmode == CK_ENV) { @@ -163,8 +173,9 @@ case CLEND_S: s = obj; break; + case CLSTART_T: + break; case CLEND_T: - tr = obj; break; default: eprintf("Bad event type received in stdout_lfun", __FILE__, __LINE__); @@ -197,12 +208,14 @@ case CLEND_S: s = obj; break; + case CLSTART_T: + break; case CLEND_T: tr = obj; tr_fprint(file, tr, CK_VERBOSE); break; default: - eprintf("Bad event type received in stdout_lfun", __FILE__, __LINE__); + eprintf("Bad event type received in lfile_lfun", __FILE__, __LINE__); } @@ -250,6 +263,8 @@ fprintf(file, " \n"); s = obj; break; + case CLSTART_T: + break; case CLEND_T: tr = obj; tr_xmlprint(file, tr, CK_VERBOSE); @@ -260,6 +275,66 @@ } +#if ENABLE_SUBUNIT +void subunit_lfun (SRunner *sr, FILE *file, enum print_output printmode, + void *obj, enum cl_event evt) +{ + TestResult *tr; + Suite *s; + char const * name; + + /* assert(printmode == CK_SUBUNIT); */ + + switch (evt) { + case CLINITLOG_SR: + break; + case CLENDLOG_SR: + break; + case CLSTART_SR: + break; + case CLSTART_S: + s = obj; + break; + case CLEND_SR: + if (printmode > CK_SILENT) { + fprintf (file, "\n"); + srunner_fprint (file, sr, printmode); + } + break; + case CLEND_S: + s = obj; + break; + case CLSTART_T: + name = obj; + subunit_test_start(name); + break; + case CLEND_T: + tr = obj; + { + char *name = ck_strdup_printf ("%s:%s", tr->tcname, tr->tname); + char *msg = tr_short_str (tr); + switch (tr->rtype) { + case CK_PASS: + subunit_test_pass(name); + break; + case CK_FAILURE: + subunit_test_fail(name, msg); + break; + case CK_ERROR: + subunit_test_error(name, msg); + break; + default: + eprintf("Bad result type in subunit_lfun", __FILE__, __LINE__); + free(name); + free(msg); + } + } + break; + default: + eprintf("Bad event type received in subunit_lfun", __FILE__, __LINE__); + } +} +#endif FILE *srunner_open_lfile (SRunner *sr) { @@ -289,7 +364,14 @@ { FILE *f; sr->loglst = check_list_create(); - srunner_register_lfun (sr, stdout, 0, stdout_lfun, print_mode); +#if ENABLE_SUBUNIT + if (print_mode != CK_SUBUNIT) +#endif + srunner_register_lfun (sr, stdout, 0, stdout_lfun, print_mode); +#if ENABLE_SUBUNIT + else + srunner_register_lfun (sr, stdout, 0, subunit_lfun, print_mode); +#endif f = srunner_open_lfile (sr); if (f) { srunner_register_lfun (sr, f, 1, lfile_lfun, print_mode); === modified file 'src/check_log.h' --- src/check_log.h 2006-10-13 00:24:56 +0000 +++ src/check_log.h 2007-04-21 01:55:36 +0000 @@ -26,6 +26,7 @@ void log_suite_start (SRunner *sr, Suite *s); void log_suite_end (SRunner *sr, Suite *s); void log_test_end (SRunner *sr, TestResult *tr); +void log_test_start (SRunner *sr, TCase *tc, TF *tfun); void stdout_lfun (SRunner *sr, FILE *file, enum print_output, void *obj, enum cl_event evt); @@ -36,6 +37,9 @@ void xml_lfun (SRunner *sr, FILE *file, enum print_output, void *obj, enum cl_event evt); +void subunit_lfun (SRunner *sr, FILE *file, enum print_output, + void *obj, enum cl_event evt); + void srunner_register_lfun (SRunner *sr, FILE *lfile, int close, LFun lfun, enum print_output); === modified file 'src/check_print.c' --- src/check_print.c 2006-10-13 00:24:56 +0000 +++ src/check_print.c 2007-04-21 06:59:27 +0000 @@ -54,6 +54,11 @@ static void srunner_fprint_summary (FILE *file, SRunner *sr, enum print_output print_mode) { +#if ENABLE_SUBUNIT + if (print_mode == CK_SUBUNIT) + return; +#endif + if (print_mode >= CK_MINIMAL) { char *str; @@ -68,6 +73,11 @@ enum print_output print_mode) { List *resultlst; + +#if ENABLE_SUBUNIT + if (print_mode == CK_SUBUNIT) + return; +#endif resultlst = sr->resultlst; === modified file 'src/check_run.c' --- src/check_run.c 2006-11-18 01:02:13 +0000 +++ src/check_run.c 2007-04-21 01:55:36 +0000 @@ -188,6 +188,7 @@ for (i = tfun->loop_start; i < tfun->loop_end; i++) { + log_test_start (sr, tc, tfun); switch (srunner_fork_status(sr)) { case CK_FORK: tr = tcase_run_tfun_fork (sr, tc, tfun, i); === modified file 'src/check_str.c' --- src/check_str.c 2006-10-13 00:24:56 +0000 +++ src/check_str.c 2007-04-21 01:55:36 +0000 @@ -47,6 +47,20 @@ return rstr; } +char *tr_short_str (TestResult *tr) +{ + const char *exact_msg; + char *rstr; + + exact_msg = (tr->rtype == CK_ERROR) ? "(after this point) ": ""; + + rstr = ck_strdup_printf ("%s:%d: %s%s", + tr->file, tr->line, + exact_msg, tr->msg); + + return rstr; +} + char *sr_stat_str (SRunner *sr) { char *str; === modified file 'src/check_str.h' --- src/check_str.h 2006-10-13 00:24:56 +0000 +++ src/check_str.h 2007-04-21 01:55:36 +0000 @@ -25,6 +25,12 @@ value has been malloc'd, and must be freed by the caller */ char *tr_str (TestResult *tr); +/* Return a string representation of the given TestResult message + without the test id or result type. This is suitable for separate + formatting of the test and the message. Return value has been + malloc'd, and must be freed by the caller */ +char *tr_short_str (TestResult *tr); + /* Return a string representation of the given SRunner's run statistics (% passed, num run, passed, errors, failures). Return value has been malloc'd, and must be freed by the caller === modified file 'tests/Makefile.am' --- tests/Makefile.am 2006-11-17 08:56:43 +0000 +++ tests/Makefile.am 2007-04-21 06:59:27 +0000 @@ -15,7 +15,7 @@ ex_xml_output \ ex_log_output -EXTRA_DIST = test_output.sh test_log_output.sh test_xml_output.sh +EXTRA_DIST = test_output.sh test_log_output.sh test_vars.in test_xml_output.sh if NO_TIMEOUT_TESTS check_check_CFLAGS = -DTIMEOUT_TESTS_ENABLED=0 === modified file 'tests/check_check_log.c' --- tests/check_check_log.c 2006-10-13 00:24:56 +0000 +++ tests/check_check_log.c 2007-04-21 06:59:27 +0000 @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include #include "check_check.h" @@ -78,15 +81,40 @@ } END_TEST +#if ENABLE_SUBUNIT +START_TEST(test_init_logging_subunit) +{ + /* init_logging with CK_SUBUNIT sets stdout + * to a subunit function, not any log. + */ + Log * first_log = NULL; + Suite *s = suite_create("Suite"); + SRunner *sr = srunner_create(s); + srunner_init_logging(sr, CK_SUBUNIT); + list_front (sr->loglst); + fail_if (list_at_end(sr->loglst), "No entries in log list"); + first_log = list_val(sr->loglst); + fail_if (first_log == NULL, "log is NULL"); + list_advance(sr->loglst); + fail_unless(list_at_end(sr->loglst), "More than one entry in log list"); + fail_unless(first_log->lfun == subunit_lfun, + "Log function is not the subunit lfun."); + srunner_end_logging(sr); + srunner_free(sr); +} +END_TEST +#endif + Suite *make_log_suite(void) { Suite *s; - TCase *tc_core, *tc_core_xml; + TCase *tc_core, *tc_core_xml, *tc_core_subunit; s = suite_create("Log"); tc_core = tcase_create("Core"); tc_core_xml = tcase_create("Core XML"); + tc_core_subunit = tcase_create("Core SubUnit"); suite_add_tcase(s, tc_core); tcase_add_test(tc_core, test_set_log); @@ -98,6 +126,11 @@ tcase_add_test(tc_core_xml, test_no_set_xml); tcase_add_test(tc_core_xml, test_double_set_xml); +#if ENABLE_SUBUNIT + suite_add_tcase(s, tc_core_subunit); + tcase_add_test(tc_core_subunit, test_init_logging_subunit); +#endif + return s; } === modified file 'tests/ex_output.c' --- tests/ex_output.c 2006-10-13 00:24:56 +0000 +++ tests/ex_output.c 2007-04-21 06:59:27 +0000 @@ -2,6 +2,7 @@ #include #include #include +#include "config.h" START_TEST(test_pass) { @@ -47,11 +48,20 @@ srunner_free(sr); } +static void print_usage(void) +{ + printf ("Usage: ex_output (CK_SILENT | CK_MINIMAL | CK_NORMAL | CK_VERBOSE"); +#if ENABLE_SUBUNIT + printf (" | CK_SUBUNIT"); +#endif + printf (")\n"); +} + int main (int argc, char **argv) { if (argc != 2) { - printf ("Usage: ex_output (CK_SILENT | CK_MINIMAL | CK_NORMAL | CK_VERBOSE)\n"); + print_usage(); return EXIT_FAILURE; } @@ -63,8 +73,12 @@ run_tests(CK_NORMAL); else if (strcmp (argv[1], "CK_VERBOSE") == 0) run_tests(CK_VERBOSE); +#if ENABLE_SUBUNIT + else if (strcmp (argv[1], "CK_SUBUNIT") == 0) + run_tests(CK_SUBUNIT); +#endif else { - printf ("Usage: ex_output (CK_SILENT | CK_MINIMAL | CK_NORMAL | CK_VERBOSE)\n"); + print_usage(); return EXIT_FAILURE; } === modified file 'tests/test_output.sh' --- tests/test_output.sh 2006-10-13 00:24:56 +0000 +++ tests/test_output.sh 2007-04-21 06:59:27 +0000 @@ -1,5 +1,7 @@ #!/bin/sh +. "${srcdir}/"test_vars + if [ "${srcdir}" = "." ]; then lsrc="" else @@ -18,12 +20,24 @@ ${lsrc}ex_output.c:8:P:Core:test_pass:0: Passed ${lsrc}ex_output.c:14:F:Core:test_fail:0: Failure ${lsrc}ex_output.c:18:E:Core:test_exit:0: (after this point) Early exit with return value 1" +t4="xtest: Core:test_pass +success: Core:test_pass +test: Core:test_fail +failure: Core:test_fail [ +${lsrc}ex_output.c:14: Failure +] +test: Core:test_exit +error: Core:test_exit [ +${lsrc}ex_output.c:18: (after this point) Early exit with return value 1 +]" op0=`./ex_output CK_SILENT` op1=`./ex_output CK_MINIMAL` op2=`./ex_output CK_NORMAL` op3=`./ex_output CK_VERBOSE` - +if test 1 -eq $ENABLE_SUBUNIT; then +op4=`./ex_output CK_SUBUNIT` +fi test_output ( ) { if [ "${1}" != "${2}" ]; then @@ -41,4 +55,7 @@ test_output "$t1" x"$op1" "CK_MINIMAL"; test_output "$t2" x"$op2" "CK_NORMAL"; test_output "$t3" x"$op3" "CK_VERBOSE"; +if test 1 -eq $ENABLE_SUBUNIT; then +test_output "$t4" x"$op4" "CK_SUBUNIT"; +fi exit 0