Merge pull request #6459 from libgit2/ethomson/clar_update

tests: update clar test runner

4 files changed, 109 insertions, 37 deletions

diff --git a/tests/clar/clar.c b/tests/clar/clar.c
index ca508d073..c9c3fde38 100644
--- a/tests/clar/clar.c
+++ b/tests/clar/clar.c
@@ -12,6 +12,7 @@
 #include <math.h>
 #include <stdarg.h>
 #include <wchar.h>
+#include <time.h>
 
 /* required for sandboxing */
 #include <sys/types.h>
@@ -86,6 +87,8 @@
 	typedef struct stat STAT_T;
 #endif
 
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+
 #include "clar.h"
 
 static void fs_rm(const char *_source);
@@ -117,6 +120,8 @@ struct clar_report {
 	const char *suite;
 
 	enum cl_test_status status;
+	time_t start;
+	double elapsed;
 
 	struct clar_error *errors;
 	struct clar_error *last_error;
@@ -145,7 +150,7 @@ static struct {
 
 	int report_errors_only;
 	int exit_on_error;
-	int report_suite_names;
+	int verbosity;
 
 	int write_summary;
 	char *summary_filename;
@@ -186,7 +191,7 @@ struct clar_suite {
 static void clar_print_init(int test_count, int suite_count, const char *suite_names);
 static void clar_print_shutdown(int test_count, int suite_count, int error_count);
 static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error);
-static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status failed);
+static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed);
 static void clar_print_onsuite(const char *suite_name, int suite_index);
 static void clar_print_onabort(const char *msg, ...);
 
@@ -245,16 +250,53 @@ clar_report_all(void)
 	}
 }
 
+#ifdef WIN32
+# define clar_time DWORD
+
+static void clar_time_now(clar_time *out)
+{
+	*out = GetTickCount();
+}
+
+static double clar_time_diff(clar_time *start, clar_time *end)
+{
+	return ((double)*end - (double)*start) / 1000;
+}
+#else
+# include <sys/time.h>
+
+# define clar_time struct timeval
+
+static void clar_time_now(clar_time *out)
+{
+	struct timezone tz;
+
+	gettimeofday(out, &tz);
+}
+
+static double clar_time_diff(clar_time *start, clar_time *end)
+{
+	return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) -
+	       ((double)start->tv_sec + (double)start->tv_usec / 1.0E6);
+}
+#endif
+
 static void
 clar_run_test(
+	const struct clar_suite *suite,
 	const struct clar_func *test,
 	const struct clar_func *initialize,
 	const struct clar_func *cleanup)
 {
+	clar_time start, end;
+
 	_clar.trampoline_enabled = 1;
 
 	CL_TRACE(CL_TRACE__TEST__BEGIN);
 
+	_clar.last_report->start = time(NULL);
+	clar_time_now(&start);
+
 	if (setjmp(_clar.trampoline) == 0) {
 		if (initialize->ptr != NULL)
 			initialize->ptr();
@@ -264,11 +306,15 @@ clar_run_test(
 		CL_TRACE(CL_TRACE__TEST__RUN_END);
 	}
 
+	clar_time_now(&end);
+
 	_clar.trampoline_enabled = 0;
 
 	if (_clar.last_report->status == CL_TEST_NOTRUN)
 		_clar.last_report->status = CL_TEST_OK;
 
+	_clar.last_report->elapsed = clar_time_diff(&start, &end);
+
 	if (_clar.local_cleanup != NULL)
 		_clar.local_cleanup(_clar.local_cleanup_payload);
 
@@ -286,7 +332,7 @@ clar_run_test(
 	if (_clar.report_errors_only) {
 		clar_report_errors(_clar.last_report);
 	} else {
-		clar_print_ontest(test->name, _clar.tests_ran, _clar.last_report->status);
+		clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status);
 	}
 }
 
@@ -352,7 +398,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 
 		_clar.last_report = report;
 
-		clar_run_test(&test[i], &suite->initialize, &suite->cleanup);
+		clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup);
 
 		if (_clar.exit_on_error && _clar.total_errors)
 			return;
@@ -427,7 +473,7 @@ clar_parse_args(int argc, char **argv)
 					++found;
 
 					if (!exact)
-						_clar.report_suite_names = 1;
+						_clar.verbosity = MAX(_clar.verbosity, 1);
 
 					switch (action) {
 					case 's': {
@@ -486,13 +532,13 @@ clar_parse_args(int argc, char **argv)
 		}
 
 		case 'v':
-			_clar.report_suite_names = 1;
+			_clar.verbosity++;
 			break;
 
 		case 'r':
 			_clar.write_summary = 1;
 			free(_clar.summary_filename);
-			_clar.summary_filename = strdup(*(argument + 2) ? (argument + 2) : "summary.xml");
+			_clar.summary_filename = *(argument + 2) ? strdup(argument + 2) : NULL;
 			break;
 
 		default:
@@ -504,6 +550,8 @@ clar_parse_args(int argc, char **argv)
 void
 clar_test_init(int argc, char **argv)
 {
+	const char *summary_env;
+
 	if (argc > 1)
 		clar_parse_args(argc, argv);
 
@@ -513,11 +561,15 @@ clar_test_init(int argc, char **argv)
 		""
 	);
 
-	if ((_clar.summary_filename = getenv("CLAR_SUMMARY")) != NULL) {
+	if (!_clar.summary_filename &&
+	    (summary_env = getenv("CLAR_SUMMARY")) != NULL) {
 		_clar.write_summary = 1;
-		_clar.summary_filename = strdup(_clar.summary_filename);
+		_clar.summary_filename = strdup(summary_env);
 	}
 
+	if (_clar.write_summary && !_clar.summary_filename)
+		_clar.summary_filename = strdup("summary.xml");
+
 	if (_clar.write_summary &&
 	    !(_clar.summary = clar_summary_init(_clar.summary_filename))) {
 		clar_print_onabort("Failed to open the summary file\n");
diff --git a/tests/clar/clar.h b/tests/clar/clar.h
index 3f659c2f6..8c22382bd 100644
--- a/tests/clar/clar.h
+++ b/tests/clar/clar.h
@@ -13,12 +13,12 @@ enum cl_test_status {
 	CL_TEST_OK,
 	CL_TEST_FAILURE,
 	CL_TEST_SKIP,
-	CL_TEST_NOTRUN
+	CL_TEST_NOTRUN,
 };
 
 enum cl_output_format {
 	CL_OUTPUT_CLAP,
-	CL_OUTPUT_TAP
+	CL_OUTPUT_TAP,
 };
 
 /** Setup clar environment */
@@ -60,7 +60,7 @@ typedef enum cl_trace_event {
 	CL_TRACE__TEST__END,
 	CL_TRACE__TEST__RUN_BEGIN,
 	CL_TRACE__TEST__RUN_END,
-	CL_TRACE__TEST__LONGJMP
+	CL_TRACE__TEST__LONGJMP,
 } cl_trace_event;
 
 typedef void (cl_trace_cb)(
diff --git a/tests/clar/clar/print.h b/tests/clar/clar/print.h
index dbfd27655..c17e2f693 100644
--- a/tests/clar/clar/print.h
+++ b/tests/clar/clar/print.h
@@ -36,24 +36,35 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con
 	fflush(stdout);
 }
 
-static void clar_print_clap_ontest(const char *test_name, int test_number, enum cl_test_status status)
+static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
 {
 	(void)test_name;
 	(void)test_number;
 
-	switch(status) {
-	case CL_TEST_OK: printf("."); break;
-	case CL_TEST_FAILURE: printf("F"); break;
-	case CL_TEST_SKIP: printf("S"); break;
-	case CL_TEST_NOTRUN: printf("N"); break;
+	if (_clar.verbosity > 1) {
+		printf("%s::%s: ", suite_name, test_name);
+
+		switch (status) {
+		case CL_TEST_OK: printf("ok\n"); break;
+		case CL_TEST_FAILURE: printf("fail\n"); break;
+		case CL_TEST_SKIP: printf("skipped"); break;
+		case CL_TEST_NOTRUN: printf("notrun"); break;
+		}
+	} else {
+		switch (status) {
+		case CL_TEST_OK: printf("."); break;
+		case CL_TEST_FAILURE: printf("F"); break;
+		case CL_TEST_SKIP: printf("S"); break;
+		case CL_TEST_NOTRUN: printf("N"); break;
+		}
+
+		fflush(stdout);
 	}
-
-	fflush(stdout);
 }
 
 static void clar_print_clap_onsuite(const char *suite_name, int suite_index)
 {
-	if (_clar.report_suite_names)
+	if (_clar.verbosity == 1)
 		printf("\n%s", suite_name);
 
 	(void)suite_index;
@@ -102,7 +113,7 @@ static void print_escaped(const char *str)
 	printf("%s", str);
 }
 
-static void clar_print_tap_ontest(const char *test_name, int test_number, enum cl_test_status status)
+static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
 {
 	const struct clar_error *error = _clar.last_report->errors;
 
@@ -111,10 +122,10 @@ static void clar_print_tap_ontest(const char *test_name, int test_number, enum c
 
 	switch(status) {
 	case CL_TEST_OK:
-		printf("ok %d - %s::%s\n", test_number, _clar.active_suite, test_name);
+		printf("ok %d - %s::%s\n", test_number, suite_name, test_name);
 		break;
 	case CL_TEST_FAILURE:
-		printf("not ok %d - %s::%s\n", test_number, _clar.active_suite, test_name);
+		printf("not ok %d - %s::%s\n", test_number, suite_name, test_name);
 
 		printf("    ---\n");
 		printf("    reason: |\n");
@@ -132,7 +143,7 @@ static void clar_print_tap_ontest(const char *test_name, int test_number, enum c
 		break;
 	case CL_TEST_SKIP:
 	case CL_TEST_NOTRUN:
-		printf("ok %d - # SKIP %s::%s\n", test_number, _clar.active_suite, test_name);
+		printf("ok %d - # SKIP %s::%s\n", test_number, suite_name, test_name);
 		break;
 	}
 
@@ -181,9 +192,9 @@ static void clar_print_error(int num, const struct clar_report *report, const st
 	PRINT(error, num, report, error);
 }
 
-static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status status)
+static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status)
 {
-	PRINT(ontest, test_name, test_number, status);
+	PRINT(ontest, suite_name, test_name, test_number, status);
 }
 
 static void clar_print_onsuite(const char *suite_name, int suite_index)
diff --git a/tests/clar/clar/summary.h b/tests/clar/clar/summary.h
index 6279f5057..4dd352e28 100644
--- a/tests/clar/clar/summary.h
+++ b/tests/clar/clar/summary.h
@@ -20,8 +20,8 @@ static int clar_summary_testsuites(struct clar_summary *summary)
 }
 
 static int clar_summary_testsuite(struct clar_summary *summary,
-    int idn, const char *name, const char *pkg, time_t timestamp,
-    double elapsed, int test_count, int fail_count, int error_count)
+    int idn, const char *name, time_t timestamp,
+    int test_count, int fail_count, int error_count)
 {
 	struct tm *tm = localtime(&timestamp);
 	char iso_dt[20];
@@ -29,17 +29,15 @@ static int clar_summary_testsuite(struct clar_summary *summary,
 	if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", tm) == 0)
 		return -1;
 
-	return fprintf(summary->fp, "\t<testsuite "
+	return fprintf(summary->fp, "\t<testsuite"
 		       " id=\"%d\""
 		       " name=\"%s\""
-		       " package=\"%s\""
 		       " hostname=\"localhost\""
 		       " timestamp=\"%s\""
-		       " time=\"%.2f\""
 		       " tests=\"%d\""
 		       " failures=\"%d\""
 		       " errors=\"%d\">\n",
-		       idn, name, pkg, iso_dt, elapsed, test_count, fail_count, error_count);
+		       idn, name, iso_dt, test_count, fail_count, error_count);
 }
 
 static int clar_summary_testcase(struct clar_summary *summary,
@@ -58,15 +56,23 @@ static int clar_summary_failure(struct clar_summary *summary,
 	    type, message, desc);
 }
 
+static int clar_summary_skipped(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "\t\t\t<skipped />\n");
+}
+
 struct clar_summary *clar_summary_init(const char *filename)
 {
 	struct clar_summary *summary;
 	FILE *fp;
 
-	if ((fp = fopen(filename, "w")) == NULL)
+	if ((fp = fopen(filename, "w")) == NULL) {
+		perror("fopen");
 		return NULL;
+	}
 
 	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		perror("malloc");
 		fclose(fp);
 		return NULL;
 	}
@@ -90,14 +96,14 @@ int clar_summary_shutdown(struct clar_summary *summary)
 		struct clar_error *error = report->errors;
 
 		if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
-			if (clar_summary_testsuite(summary, 0, report->suite, "",
-			    time(NULL), 0, _clar.tests_ran, _clar.total_errors, 0) < 0)
+			if (clar_summary_testsuite(summary, 0, report->suite,
+			    report->start, _clar.tests_ran, _clar.total_errors, 0) < 0)
 				goto on_error;
 		}
 
 		last_suite = report->suite;
 
-		clar_summary_testcase(summary, report->test, "what", 0);
+		clar_summary_testcase(summary, report->test, report->suite, report->elapsed);
 
 		while (error != NULL) {
 			if (clar_summary_failure(summary, "assert",
@@ -107,6 +113,9 @@ int clar_summary_shutdown(struct clar_summary *summary)
 			error = error->next;
 		}
 
+		if (report->status == CL_TEST_SKIP)
+			clar_summary_skipped(summary);
+
 		if (clar_summary_close_tag(summary, "testcase", 2) < 0)
 			goto on_error;