Merge pull request #4778 from libgit2/ethomson/clar-xml

Clar XML output redux

12 files changed, 429 insertions, 189 deletions

diff --git a/.vsts-ci.yml b/.vsts-ci.yml
index c08a9b6ba..7506e3fb4 100644
--- a/.vsts-ci.yml
+++ b/.vsts-ci.yml
@@ -11,128 +11,50 @@ jobs:
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=gcc
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: linux_trusty_gcc_mbedtls
   displayName: 'Linux (Trusty; GCC; mbedTLS)'
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=gcc
        CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: 'LEAK_CHECK=valgrind'
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: linux_trusty_clang_openssl
   displayName: 'Linux (Trusty; Clang; OpenSSL)'
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-openssl:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=clang
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: linux_trusty_clang_mbedtls
   displayName: 'Linux (Trusty; Clang; mbedTLS)'
   pool:
     vmImage: 'Ubuntu 16.04'
   steps:
-  - task: Docker@0
-    displayName: Build
-    inputs:
-      action: 'Run an image'
+  - template: ci/vsts-docker.yml
+    parameters:
       imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: |
+      environmentVariables: |
        CC=clang
        CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS
        LEAK_CHECK=valgrind
-      workDir: '/build'
-      containerCommand: '/src/ci/build.sh'
-      detached: false
-  - task: Docker@0
-    displayName: Test
-    inputs:
-      action: 'Run an image'
-      imageName: 'libgit2/trusty-mbedtls:latest'
-      volumes: |
-       $(Build.SourcesDirectory):/src
-       $(Build.BinariesDirectory):/build
-      envVars: 'LEAK_CHECK=valgrind'
-      workDir: '/build'
-      containerCommand: '/src/ci/test.sh'
-      detached: false
 
 - job: macos
   displayName: 'macOS'
@@ -141,37 +63,30 @@ jobs:
   steps:
   - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh'
     displayName: Setup
-  - bash: . '$(Build.SourcesDirectory)/ci/build.sh'
-    displayName: Build
-    env:
-      PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
-  - bash: . '$(Build.SourcesDirectory)/ci/test.sh'
-    displayName: Test
-    env:
-      TMPDIR: $(Agent.TempDirectory)
-      LEAK_CHECK: leaks
+  - template: ci/vsts-bash.yml
+    parameters:
+      environmentVariables:
+        TMPDIR: $(Agent.TempDirectory)
+        PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
+        LEAK_CHECK: leaks
 
 - job: windows_vs_amd64
   displayName: 'Windows (Visual Studio; amd64)'
   pool: Hosted
   steps:
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64"
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64"
 
 - job: windows_vs_x86
   displayName: 'Windows (Visual Studio; x86)'
   pool: Hosted
   steps:
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013"
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013"
 
 - job: windows_mingw_amd64
   displayName: 'Windows (MinGW; amd64)'
@@ -182,13 +97,11 @@ jobs:
     env:
       TEMP: $(Agent.TempDirectory)
       ARCH: amd64
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -G"MinGW Makefiles"
-      PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -G"MinGW Makefiles"
+        PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
 
 - job: windows_mingw_x86
   displayName: 'Windows (MinGW; x86)'
@@ -196,13 +109,12 @@ jobs:
   steps:
   - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1'
     displayName: Setup
+    workingDirectory: '$(Build.BinariesDirectory)'
     env:
       TEMP: $(Agent.TempDirectory)
       ARCH: x86
-  - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
-    displayName: Build
-    env:
-      CMAKE_OPTIONS: -G"MinGW Makefiles"
-      PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
-  - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
-    displayName: Test
+  - template: ci/vsts-powershell.yml
+    parameters:
+      environmentVariables:
+        CMAKE_OPTIONS: -G"MinGW Makefiles"
+        PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d9c0a1fbf..a888c5a5f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -62,6 +62,8 @@ OPTION(USE_EXT_HTTP_PARSER		"Use system HTTP_Parser if available"			 ON)
 OPTION(DEBUG_POOL			"Enable debug pool allocator"				OFF)
 OPTION(ENABLE_WERROR			"Enable compilation with -Werror"			OFF)
 OPTION(USE_BUNDLED_ZLIB    		"Use the bundled version of zlib"			OFF)
+   SET(CLAR_XML 			"OFF"					       CACHE STRING
+       "Writes test results in XML format.  One of ON, OFF or the directory to write to; this does not affect the output executables, this only affects the behavior of the ctest command.")
 
 IF (UNIX AND NOT APPLE)
 	OPTION(ENABLE_REPRODUCIBLE_BUILDS "Enable reproducible builds"				OFF)
diff --git a/ci/build.ps1 b/ci/build.ps1
index 159c1dd1b..c74717d10 100644
--- a/ci/build.ps1
+++ b/ci/build.ps1
@@ -18,7 +18,9 @@ Write-Host "####################################################################
 Write-Host "## Configuring build environment"
 Write-Host "##############################################################################"
 
-Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS}"
+$TestOutputDirectory = $BuildDirectory -replace "\\", "/"
+
+Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON -DCLAR_XML=${TestOutputDirectory} ${Env:CMAKE_OPTIONS}"
 if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) }
 
 Write-Host ""
diff --git a/ci/build.sh b/ci/build.sh
index ddd9cb68a..b4f0a6f67 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -28,8 +28,8 @@ echo "##########################################################################
 echo "## Configuring build environment"
 echo "##############################################################################"
 
-echo cmake ${SOURCE_DIR} -DBUILD_EXAMPLES=ON ${CMAKE_OPTIONS}
-cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON ${CMAKE_OPTIONS}
+echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML=\"${BUILD_DIR}\" ${CMAKE_OPTIONS}
+cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML="${BUILD_DIR}" ${CMAKE_OPTIONS}
 
 echo ""
 echo "##############################################################################"
diff --git a/ci/vsts-bash.yml b/ci/vsts-bash.yml
new file mode 100644
index 000000000..d776a3649
--- /dev/null
+++ b/ci/vsts-bash.yml
@@ -0,0 +1,17 @@
+# These are the steps used for building on machines with bash.
+steps:
+- bash: . '$(Build.SourcesDirectory)/ci/build.sh'
+  displayName: Build
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- bash: . '$(Build.SourcesDirectory)/ci/test.sh'
+  displayName: Test
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- task: PublishTestResults@2
+  displayName: Publish Test Results
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: 'results_*.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    mergeTestResults: true
diff --git a/ci/vsts-docker.yml b/ci/vsts-docker.yml
new file mode 100644
index 000000000..e92510478
--- /dev/null
+++ b/ci/vsts-docker.yml
@@ -0,0 +1,33 @@
+# These are the steps used in a container-based build in VSTS.
+steps:
+- task: docker@0
+  displayName: Build
+  inputs:
+    action: 'Run an image'
+    imageName: ${{ parameters.imageName }}
+    volumes: |
+     $(Build.SourcesDirectory):/src
+     $(Build.BinariesDirectory):/build
+    envVars: ${{ parameters.environmentVariables }}
+    workDir: '/build'
+    containerCommand: '/src/ci/build.sh'
+    detached: false
+- task: docker@0
+  displayName: Test
+  inputs:
+    action: 'Run an image'
+    imageName: ${{ parameters.imageName }}
+    volumes: |
+     $(Build.SourcesDirectory):/src
+     $(Build.BinariesDirectory):/build
+    envVars: ${{ parameters.environmentVariables }}
+    workDir: '/build'
+    containerCommand: '/src/ci/test.sh'
+    detached: false
+- task: publishtestresults@2
+  displayName: Publish Test Results
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: 'results_*.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    mergeTestResults: true
diff --git a/ci/vsts-powershell.yml b/ci/vsts-powershell.yml
new file mode 100644
index 000000000..a2eb175d5
--- /dev/null
+++ b/ci/vsts-powershell.yml
@@ -0,0 +1,17 @@
+# These are the steps used for building on machines with PowerShell.
+steps:
+- powershell: . '$(Build.SourcesDirectory)\ci\build.ps1'
+  displayName: Build
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- powershell: . '$(Build.SourcesDirectory)\ci\test.ps1'
+  displayName: Test
+  workingDirectory: '$(Build.BinariesDirectory)'
+  env: ${{ parameters.environmentVariables }}
+- task: PublishTestResults@2
+  displayName: Publish Test Results
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: 'results_*.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    mergeTestResults: true
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 950250418..792e6b5ff 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -54,8 +54,22 @@ IF (MSVC_IDE)
 	SET_SOURCE_FILES_PROPERTIES("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h")
 ENDIF ()
 
-ADD_TEST(offline   "${libgit2_BINARY_DIR}/libgit2_clar" -v -xonline)
-ADD_TEST(online    "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline)
-ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push)
-ADD_TEST(ssh       "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths)
-ADD_TEST(proxy     "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request)
+IF (CLAR_XML)
+	IF (CLAR_XML STREQUAL "ON")
+		SET(XML_PATH "")
+	ELSE ()
+		SET(XML_PATH "${CLAR_XML}/")
+	ENDIF ()
+
+	SET(TESTS_OFFLINE_XML   "-r${XML_PATH}results_offline.xml")
+	SET(TESTS_ONLINE_XML    "-r${XML_PATH}results_online.xml")
+	SET(TESTS_GITDAEMON_XML "-r${XML_PATH}results_gitdaemon.xml")
+	SET(TESTS_SSH_XML       "-r${XML_PATH}results_ssh.xml")
+	SET(TESTS_PROXY_XML     "-r${XML_PATH}results_proxy.xml")
+ENDIF ()
+
+ADD_TEST(offline   "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_OFFLINE_XML} -xonline)
+ADD_TEST(online    "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_ONLINE_XML} -sonline)
+ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_GITDAEMON_XML} -sonline::push)
+ADD_TEST(ssh       "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_SSH_XML} -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths)
+ADD_TEST(proxy     "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_PROXY_XML} -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request)
diff --git a/tests/clar.c b/tests/clar.c
index d5212d1ca..27d35e1c7 100644
--- a/tests/clar.c
+++ b/tests/clar.c
@@ -95,9 +95,6 @@ static const char *
 fixture_path(const char *base, const char *fixture_name);
 
 struct clar_error {
-	const char *test;
-	int test_number;
-	const char *suite;
 	const char *file;
 	int line_number;
 	const char *error_msg;
@@ -106,11 +103,34 @@ struct clar_error {
 	struct clar_error *next;
 };
 
-static struct {
-	int argc;
-	char **argv;
+struct clar_explicit {
+	size_t suite_idx;
+	const char *filter;
+
+	struct clar_explicit *next;
+};
+
+struct clar_report {
+	const char *test;
+	int test_number;
+	const char *suite;
+
+	enum cl_test_status status;
+
+	struct clar_error *errors;
+	struct clar_error *last_error;
+
+	struct clar_report *next;
+};
 
+struct clar_summary {
+	const char *filename;
+	FILE *fp;
+};
+
+static struct {
 	enum cl_test_status test_status;
+
 	const char *active_test;
 	const char *active_suite;
 
@@ -124,8 +144,15 @@ static struct {
 	int exit_on_error;
 	int report_suite_names;
 
-	struct clar_error *errors;
-	struct clar_error *last_error;
+	int write_summary;
+	const char *summary_filename;
+	struct clar_summary *summary;
+
+	struct clar_explicit *explicit;
+	struct clar_explicit *last_explicit;
+
+	struct clar_report *reports;
+	struct clar_report *last_report;
 
 	void (*local_cleanup)(void *);
 	void *local_cleanup_payload;
@@ -155,7 +182,7 @@ struct clar_suite {
 /* From clar_print_*.c */
 static void clar_print_init(int test_count, int suite_count, const char *suite_names);
 static void clar_print_shutdown(int test_count, int suite_count, int error_count);
-static void clar_print_error(int num, const struct clar_error *error);
+static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error);
 static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status failed);
 static void clar_print_onsuite(const char *suite_name, int suite_index);
 static void clar_print_onabort(const char *msg, ...);
@@ -164,6 +191,10 @@ static void clar_print_onabort(const char *msg, ...);
 static void clar_unsandbox(void);
 static int clar_sandbox(void);
 
+/* From summary.h */
+static struct clar_summary *clar_summary_init(const char *filename);
+static int clar_summary_shutdown(struct clar_summary *fp);
+
 /* Load the declarations for the test suite */
 #include "clar.suite"
 
@@ -186,21 +217,29 @@ void cl_trace_register(cl_trace_cb *cb, void *payload)
 
 /* Core test functions */
 static void
-clar_report_errors(void)
+clar_report_errors(struct clar_report *report)
 {
+	struct clar_error *error;
 	int i = 1;
-	struct clar_error *error, *next;
-
-	error = _clar.errors;
-	while (error != NULL) {
-		next = error->next;
-		clar_print_error(i++, error);
-		free(error->description);
-		free(error);
-		error = next;
-	}
 
-	_clar.errors = _clar.last_error = NULL;
+	for (error = report->errors; error; error = error->next)
+		clar_print_error(i++, _clar.last_report, error);
+}
+
+static void
+clar_report_all(void)
+{
+	struct clar_report *report;
+	struct clar_error *error;
+	int i = 1;
+
+	for (report = _clar.reports; report; report = report->next) {
+		if (report->status != CL_TEST_FAILURE)
+			continue;
+
+		for (error = report->errors; error; error = error->next)
+			clar_print_error(i++, report, error);
+	}
 }
 
 static void
@@ -209,7 +248,6 @@ clar_run_test(
 	const struct clar_func *initialize,
 	const struct clar_func *cleanup)
 {
-	_clar.test_status = CL_TEST_OK;
 	_clar.trampoline_enabled = 1;
 
 	CL_TRACE(CL_TRACE__TEST__BEGIN);
@@ -225,6 +263,9 @@ clar_run_test(
 
 	_clar.trampoline_enabled = 0;
 
+	if (_clar.last_report->status == CL_TEST_NOTRUN)
+		_clar.last_report->status = CL_TEST_OK;
+
 	if (_clar.local_cleanup != NULL)
 		_clar.local_cleanup(_clar.local_cleanup_payload);
 
@@ -240,9 +281,9 @@ clar_run_test(
 	_clar.local_cleanup_payload = NULL;
 
 	if (_clar.report_errors_only) {
-		clar_report_errors();
+		clar_report_errors(_clar.last_report);
 	} else {
-		clar_print_ontest(test->name, _clar.tests_ran, _clar.test_status);
+		clar_print_ontest(test->name, _clar.tests_ran, _clar.last_report->status);
 	}
 }
 
@@ -251,6 +292,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 {
 	const struct clar_func *test = suite->tests;
 	size_t i, matchlen;
+	struct clar_report *report;
 
 	if (!suite->enabled)
 		return;
@@ -283,6 +325,21 @@ clar_run_suite(const struct clar_suite *suite, const char *filter)
 			continue;
 
 		_clar.active_test = test[i].name;
+
+		report = calloc(1, sizeof(struct clar_report));
+		report->suite = _clar.active_suite;
+		report->test = _clar.active_test;
+		report->test_number = _clar.tests_ran;
+		report->status = CL_TEST_NOTRUN;
+
+		if (_clar.reports == NULL)
+			_clar.reports = report;
+
+		if (_clar.last_report != NULL)
+			_clar.last_report->next = report;
+
+		_clar.last_report = report;
+
 		clar_run_test(&test[i], &suite->initialize, &suite->cleanup);
 
 		if (_clar.exit_on_error && _clar.total_errors)
@@ -298,13 +355,14 @@ clar_usage(const char *arg)
 {
 	printf("Usage: %s [options]\n\n", arg);
 	printf("Options:\n");
-	printf("  -sname\tRun only the suite with `name` (can go to individual test name)\n");
-	printf("  -iname\tInclude the suite with `name`\n");
-	printf("  -xname\tExclude the suite with `name`\n");
-	printf("  -v    \tIncrease verbosity (show suite names)\n");
-	printf("  -q    \tOnly report tests that had an error\n");
-	printf("  -Q    \tQuit as soon as a test fails\n");
-	printf("  -l    \tPrint suite names\n");
+	printf("  -sname        Run only the suite with `name` (can go to individual test name)\n");
+	printf("  -iname        Include the suite with `name`\n");
+	printf("  -xname        Exclude the suite with `name`\n");
+	printf("  -v            Increase verbosity (show suite names)\n");
+	printf("  -q            Only report tests that had an error\n");
+	printf("  -Q            Quit as soon as a test fails\n");
+	printf("  -l            Print suite names\n");
+	printf("  -r[filename]  Write summary file (to the optional filename)\n");
 	exit(-1);
 }
 
@@ -318,7 +376,7 @@ clar_parse_args(int argc, char **argv)
 		char *argument = argv[i];
 
 		if (argument[0] != '-' || argument[1] == '\0'
-		    || strchr("sixvqQl", argument[1]) == NULL) {
+		    || strchr("sixvqQlr", argument[1]) == NULL) {
 			clar_usage(argv[0]);
 		}
 	}
@@ -359,7 +417,24 @@ clar_parse_args(int argc, char **argv)
 						_clar.report_suite_names = 1;
 
 					switch (action) {
-					case 's': _clar_suites[j].enabled = 1; clar_run_suite(&_clar_suites[j], argument); break;
+					case 's': {
+						struct clar_explicit *explicit =
+							calloc(1, sizeof(struct clar_explicit));
+						assert(explicit);
+
+						explicit->suite_idx = j;
+						explicit->filter = argument;
+
+						if (_clar.explicit == NULL)
+							_clar.explicit = explicit;
+
+						if (_clar.last_explicit != NULL)
+							_clar.last_explicit->next = explicit;
+
+						_clar_suites[j].enabled = 1;
+						_clar.last_explicit = explicit;
+						break;
+					}
 					case 'i': _clar_suites[j].enabled = 1; break;
 					case 'x': _clar_suites[j].enabled = 0; break;
 					}
@@ -397,6 +472,12 @@ clar_parse_args(int argc, char **argv)
 			_clar.report_suite_names = 1;
 			break;
 
+		case 'r':
+			_clar.write_summary = 1;
+			_clar.summary_filename = *(argument + 2) ? (argument + 2) :
+			    "summary.xml";
+			break;
+
 		default:
 			assert(!"Unexpected commandline argument!");
 		}
@@ -412,23 +493,31 @@ clar_test_init(int argc, char **argv)
 		""
 	);
 
+	if (argc > 1)
+		clar_parse_args(argc, argv);
+
+	if (_clar.write_summary &&
+	    !(_clar.summary = clar_summary_init(_clar.summary_filename))) {
+		clar_print_onabort("Failed to open the summary file\n");
+		exit(-1);
+	}
+
 	if (clar_sandbox() < 0) {
 		clar_print_onabort("Failed to sandbox the test runner.\n");
 		exit(-1);
 	}
-
-	_clar.argc = argc;
-	_clar.argv = argv;
 }
 
 int
 clar_test_run(void)
 {
-	if (_clar.argc > 1)
-		clar_parse_args(_clar.argc, _clar.argv);
+	size_t i;
+	struct clar_explicit *explicit;
 
-	if (!_clar.suites_ran) {
-		size_t i;
+	if (_clar.explicit) {
+		for (explicit = _clar.explicit; explicit; explicit = explicit->next)
+			clar_run_suite(&_clar_suites[explicit->suite_idx], explicit->filter);
+	} else {
 		for (i = 0; i < _clar_suite_count; ++i)
 			clar_run_suite(&_clar_suites[i], NULL);
 	}
@@ -439,6 +528,9 @@ clar_test_run(void)
 void
 clar_test_shutdown(void)
 {
+	struct clar_explicit *explicit, *explicit_next;
+	struct clar_report *report, *report_next;
+
 	clar_print_shutdown(
 		_clar.tests_ran,
 		(int)_clar_suite_count,
@@ -446,6 +538,21 @@ clar_test_shutdown(void)
 	);
 
 	clar_unsandbox();
+
+	if (_clar.write_summary && clar_summary_shutdown(_clar.summary) < 0) {
+		clar_print_onabort("Failed to write the summary file\n");
+		exit(-1);
+	}
+
+	for (explicit = _clar.explicit; explicit; explicit = explicit_next) {
+		explicit_next = explicit->next;
+		free(explicit);
+	}
+
+	for (report = _clar.reports; report; report = report_next) {
+		report_next = report->next;
+		free(report);
+	}
 }
 
 int
@@ -465,7 +572,7 @@ static void abort_test(void)
 	if (!_clar.trampoline_enabled) {
 		clar_print_onabort(
 				"Fatal error: a cleanup method raised an exception.");
-		clar_report_errors();
+		clar_report_errors(_clar.last_report);
 		exit(-1);
 	}
 
@@ -475,7 +582,7 @@ static void abort_test(void)
 
 void clar__skip(void)
 {
-	_clar.test_status = CL_TEST_SKIP;
+	_clar.last_report->status = CL_TEST_SKIP;
 	_clar.total_skipped++;
 	abort_test();
 }
@@ -489,17 +596,14 @@ void clar__fail(
 {
 	struct clar_error *error = calloc(1, sizeof(struct clar_error));
 
-	if (_clar.errors == NULL)
-		_clar.errors = error;
+	if (_clar.last_report->errors == NULL)
+		_clar.last_report->errors = error;
 
-	if (_clar.last_error != NULL)
-		_clar.last_error->next = error;
+	if (_clar.last_report->last_error != NULL)
+		_clar.last_report->last_error->next = error;
 
-	_clar.last_error = error;
+	_clar.last_report->last_error = error;
 
-	error->test = _clar.active_test;
-	error->test_number = _clar.tests_ran;
-	error->suite = _clar.active_suite;
 	error->file = file;
 	error->line_number = line;
 	error->error_msg = error_msg;
@@ -508,7 +612,7 @@ void clar__fail(
 		error->description = strdup(description);
 
 	_clar.total_errors++;
-	_clar.test_status = CL_TEST_FAILURE;
+	_clar.last_report->status = CL_TEST_FAILURE;
 
 	if (should_abort)
 		abort_test();
@@ -653,3 +757,4 @@ void cl_set_cleanup(void (*cleanup)(void *), void *opaque)
 #include "clar/fixtures.h"
 #include "clar/fs.h"
 #include "clar/print.h"
+#include "clar/summary.h"
diff --git a/tests/clar.h b/tests/clar.h
index dfb88d76d..2f9f96b3f 100644
--- a/tests/clar.h
+++ b/tests/clar.h
@@ -12,13 +12,16 @@
 enum cl_test_status {
 	CL_TEST_OK,
 	CL_TEST_FAILURE,
-	CL_TEST_SKIP
+	CL_TEST_SKIP,
+	CL_TEST_NOTRUN,
 };
 
+/** Setup clar environment */
 void clar_test_init(int argc, char *argv[]);
 int clar_test_run(void);
 void clar_test_shutdown(void);
 
+/** One shot setup & run */
 int clar_test(int argc, char *argv[]);
 
 const char *clar_sandbox_path(void);
diff --git a/tests/clar/print.h b/tests/clar/print.h
index 916d807c1..73c4a8953 100644
--- a/tests/clar/print.h
+++ b/tests/clar/print.h
@@ -13,16 +13,16 @@ static void clar_print_shutdown(int test_count, int suite_count, int error_count
 	(void)error_count;
 
 	printf("\n\n");
-	clar_report_errors();
+	clar_report_all();
 }
 
-static void clar_print_error(int num, const struct clar_error *error)
+static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error)
 {
 	printf("  %d) Failure:\n", num);
 
 	printf("%s::%s [%s:%d]\n",
-		error->suite,
-		error->test,
+		report->suite,
+		report->test,
 		error->file,
 		error->line_number);
 
@@ -44,6 +44,7 @@ static void clar_print_ontest(const char *test_name, int test_number, enum cl_te
 	case CL_TEST_OK: printf("."); break;
 	case CL_TEST_FAILURE: printf("F"); break;
 	case CL_TEST_SKIP: printf("S"); break;
+	case CL_TEST_NOTRUN: printf("N"); break;
 	}
 
 	fflush(stdout);
diff --git a/tests/clar/summary.h b/tests/clar/summary.h
new file mode 100644
index 000000000..1af110efa
--- /dev/null
+++ b/tests/clar/summary.h
@@ -0,0 +1,134 @@
+
+#include <stdio.h>
+#include <time.h>
+
+int clar_summary_close_tag(
+    struct clar_summary *summary, const char *tag, int indent)
+{
+	const char *indt;
+
+	if (indent == 0) indt = "";
+	else if (indent == 1) indt = "\t";
+	else indt = "\t\t";
+
+	return fprintf(summary->fp, "%s</%s>\n", indt, tag);
+}
+
+int clar_summary_testsuites(struct clar_summary *summary)
+{
+	return fprintf(summary->fp, "<testsuites>\n");
+}
+
+int clar_summary_testsuite(struct clar_summary *summary,
+    int idn, const char *name, const char *pkg, time_t timestamp,
+    double elapsed, int test_count, int fail_count, int error_count)
+{
+	struct tm *tm = localtime(&timestamp);
+	char iso_dt[20];
+
+	if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", tm) == 0)
+		return -1;
+
+	return fprintf(summary->fp, "\t<testsuite "
+		       " id=\"%d\""
+		       " name=\"%s\""
+		       " package=\"%s\""
+		       " hostname=\"localhost\""
+		       " timestamp=\"%s\""
+		       " time=\"%.2f\""
+		       " tests=\"%d\""
+		       " failures=\"%d\""
+		       " errors=\"%d\">\n",
+		       idn, name, pkg, iso_dt, elapsed, test_count, fail_count, error_count);
+}
+
+int clar_summary_testcase(struct clar_summary *summary,
+    const char *name, const char *classname, double elapsed)
+{
+	return fprintf(summary->fp,
+	    "\t\t<testcase name=\"%s\" classname=\"%s\" time=\"%.2f\">\n",
+		name, classname, elapsed);
+}
+
+int clar_summary_failure(struct clar_summary *summary,
+    const char *type, const char *message, const char *desc)
+{
+	return fprintf(summary->fp,
+	    "\t\t\t<failure type=\"%s\"><![CDATA[%s\n%s]]></failure>\n",
+	    type, message, desc);
+}
+
+struct clar_summary *clar_summary_init(const char *filename)
+{
+	struct clar_summary *summary;
+	FILE *fp;
+
+	if ((fp = fopen(filename, "w")) == NULL)
+		return NULL;
+
+	if ((summary = malloc(sizeof(struct clar_summary))) == NULL) {
+		fclose(fp);
+		return NULL;
+	}
+
+	summary->filename = filename;
+	summary->fp = fp;
+
+	return summary;
+}
+
+int clar_summary_shutdown(struct clar_summary *summary)
+{
+	struct clar_report *report;
+	const char *last_suite = NULL;
+
+	if (clar_summary_testsuites(summary) < 0)
+		goto on_error;
+
+	report = _clar.reports;
+	while (report != NULL) {
+		struct clar_error *error = report->errors;
+
+		if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_testsuite(summary, 0, report->suite, "",
+			    time(NULL), 0, _clar.tests_ran, _clar.total_errors, 0) < 0)
+				goto on_error;
+		}
+
+		last_suite = report->suite;
+
+		clar_summary_testcase(summary, report->test, "what", 0);
+
+		while (error != NULL) {
+			if (clar_summary_failure(summary, "assert",
+			    error->error_msg, error->description) < 0)
+				goto on_error;
+
+			error = error->next;
+		}
+
+		if (clar_summary_close_tag(summary, "testcase", 2) < 0)
+			goto on_error;
+
+		report = report->next;
+
+		if (!report || strcmp(last_suite, report->suite) != 0) {
+			if (clar_summary_close_tag(summary, "testsuite", 1) < 0)
+				goto on_error;
+		}
+	}
+
+	if (clar_summary_close_tag(summary, "testsuites", 0) < 0 ||
+	    fclose(summary->fp) != 0)
+		goto on_error;
+
+	printf("written summary file to %s\n", summary->filename);
+
+	free(summary);
+	return 0;
+
+on_error:
+	fclose(summary->fp);
+	free(summary);
+	return -1;
+}