diff options
-rw-r--r-- | .vsts-ci.yml | 162 | ||||
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rw-r--r-- | ci/build.ps1 | 4 | ||||
-rwxr-xr-x | ci/build.sh | 4 | ||||
-rw-r--r-- | ci/vsts-bash.yml | 17 | ||||
-rw-r--r-- | ci/vsts-docker.yml | 33 | ||||
-rw-r--r-- | ci/vsts-powershell.yml | 17 | ||||
-rw-r--r-- | tests/CMakeLists.txt | 24 | ||||
-rw-r--r-- | tests/clar.c | 207 | ||||
-rw-r--r-- | tests/clar.h | 5 | ||||
-rw-r--r-- | tests/clar/print.h | 9 | ||||
-rw-r--r-- | tests/clar/summary.h | 134 |
12 files changed, 429 insertions, 189 deletions
diff --git a/.vsts-ci.yml b/.vsts-ci.yml index c08a9b6ba..7506e3fb4 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -11,128 +11,50 @@ jobs: pool: vmImage: 'Ubuntu 16.04' steps: - - task: Docker@0 - displayName: Build - inputs: - action: 'Run an image' + - template: ci/vsts-docker.yml + parameters: imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - workDir: '/build' - containerCommand: '/src/ci/build.sh' - detached: false - - task: Docker@0 - displayName: Test - inputs: - action: 'Run an image' - imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: | + environmentVariables: | CC=gcc LEAK_CHECK=valgrind - workDir: '/build' - containerCommand: '/src/ci/test.sh' - detached: false - job: linux_trusty_gcc_mbedtls displayName: 'Linux (Trusty; GCC; mbedTLS)' pool: vmImage: 'Ubuntu 16.04' steps: - - task: Docker@0 - displayName: Build - inputs: - action: 'Run an image' + - template: ci/vsts-docker.yml + parameters: imageName: 'libgit2/trusty-mbedtls:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: | + environmentVariables: | CC=gcc CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS LEAK_CHECK=valgrind - workDir: '/build' - containerCommand: '/src/ci/build.sh' - detached: false - - task: Docker@0 - displayName: Test - inputs: - action: 'Run an image' - imageName: 'libgit2/trusty-mbedtls:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: 'LEAK_CHECK=valgrind' - workDir: '/build' - containerCommand: '/src/ci/test.sh' - detached: false - job: linux_trusty_clang_openssl displayName: 'Linux (Trusty; Clang; OpenSSL)' pool: vmImage: 'Ubuntu 16.04' steps: - - task: Docker@0 - displayName: Build - inputs: - action: 'Run an image' - imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - workDir: '/build' - containerCommand: '/src/ci/build.sh' - detached: false - - task: Docker@0 - displayName: Test - inputs: - action: 'Run an image' + - template: ci/vsts-docker.yml + parameters: imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: | + environmentVariables: | CC=clang LEAK_CHECK=valgrind - workDir: '/build' - containerCommand: '/src/ci/test.sh' - detached: false - job: linux_trusty_clang_mbedtls displayName: 'Linux (Trusty; Clang; mbedTLS)' pool: vmImage: 'Ubuntu 16.04' steps: - - task: Docker@0 - displayName: Build - inputs: - action: 'Run an image' + - template: ci/vsts-docker.yml + parameters: imageName: 'libgit2/trusty-mbedtls:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: | + environmentVariables: | CC=clang CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS LEAK_CHECK=valgrind - workDir: '/build' - containerCommand: '/src/ci/build.sh' - detached: false - - task: Docker@0 - displayName: Test - inputs: - action: 'Run an image' - imageName: 'libgit2/trusty-mbedtls:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: 'LEAK_CHECK=valgrind' - workDir: '/build' - containerCommand: '/src/ci/test.sh' - detached: false - job: macos displayName: 'macOS' @@ -141,37 +63,30 @@ jobs: steps: - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh' displayName: Setup - - bash: . '$(Build.SourcesDirectory)/ci/build.sh' - displayName: Build - env: - PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig - - bash: . '$(Build.SourcesDirectory)/ci/test.sh' - displayName: Test - env: - TMPDIR: $(Agent.TempDirectory) - LEAK_CHECK: leaks + - template: ci/vsts-bash.yml + parameters: + environmentVariables: + TMPDIR: $(Agent.TempDirectory) + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + LEAK_CHECK: leaks - job: windows_vs_amd64 displayName: 'Windows (Visual Studio; amd64)' pool: Hosted steps: - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - env: - CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" - job: windows_vs_x86 displayName: 'Windows (Visual Studio; x86)' pool: Hosted steps: - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - env: - CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" - job: windows_mingw_amd64 displayName: 'Windows (MinGW; amd64)' @@ -182,13 +97,11 @@ jobs: env: TEMP: $(Agent.TempDirectory) ARCH: amd64 - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - env: - CMAKE_OPTIONS: -G"MinGW Makefiles" - PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -G"MinGW Makefiles" + PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - job: windows_mingw_x86 displayName: 'Windows (MinGW; x86)' @@ -196,13 +109,12 @@ jobs: steps: - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1' displayName: Setup + workingDirectory: '$(Build.BinariesDirectory)' env: TEMP: $(Agent.TempDirectory) ARCH: x86 - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - env: - CMAKE_OPTIONS: -G"MinGW Makefiles" - PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -G"MinGW Makefiles" + PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin diff --git a/CMakeLists.txt b/CMakeLists.txt index d9c0a1fbf..a888c5a5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,8 @@ OPTION(USE_EXT_HTTP_PARSER "Use system HTTP_Parser if available" ON) OPTION(DEBUG_POOL "Enable debug pool allocator" OFF) OPTION(ENABLE_WERROR "Enable compilation with -Werror" OFF) OPTION(USE_BUNDLED_ZLIB "Use the bundled version of zlib" OFF) + SET(CLAR_XML "OFF" CACHE STRING + "Writes test results in XML format. One of ON, OFF or the directory to write to; this does not affect the output executables, this only affects the behavior of the ctest command.") IF (UNIX AND NOT APPLE) OPTION(ENABLE_REPRODUCIBLE_BUILDS "Enable reproducible builds" OFF) diff --git a/ci/build.ps1 b/ci/build.ps1 index 159c1dd1b..c74717d10 100644 --- a/ci/build.ps1 +++ b/ci/build.ps1 @@ -18,7 +18,9 @@ Write-Host "#################################################################### Write-Host "## Configuring build environment" Write-Host "##############################################################################" -Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS}" +$TestOutputDirectory = $BuildDirectory -replace "\\", "/" + +Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON -DCLAR_XML=${TestOutputDirectory} ${Env:CMAKE_OPTIONS}" if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } Write-Host "" diff --git a/ci/build.sh b/ci/build.sh index ddd9cb68a..b4f0a6f67 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -28,8 +28,8 @@ echo "########################################################################## echo "## Configuring build environment" echo "##############################################################################" -echo cmake ${SOURCE_DIR} -DBUILD_EXAMPLES=ON ${CMAKE_OPTIONS} -cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON ${CMAKE_OPTIONS} +echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML=\"${BUILD_DIR}\" ${CMAKE_OPTIONS} +cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -DCLAR_XML="${BUILD_DIR}" ${CMAKE_OPTIONS} echo "" echo "##############################################################################" diff --git a/ci/vsts-bash.yml b/ci/vsts-bash.yml new file mode 100644 index 000000000..d776a3649 --- /dev/null +++ b/ci/vsts-bash.yml @@ -0,0 +1,17 @@ +# These are the steps used for building on machines with bash. +steps: +- bash: . '$(Build.SourcesDirectory)/ci/build.sh' + displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- bash: . '$(Build.SourcesDirectory)/ci/test.sh' + displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFiles: 'results_*.xml' + searchFolder: '$(Build.BinariesDirectory)' + mergeTestResults: true diff --git a/ci/vsts-docker.yml b/ci/vsts-docker.yml new file mode 100644 index 000000000..e92510478 --- /dev/null +++ b/ci/vsts-docker.yml @@ -0,0 +1,33 @@ +# These are the steps used in a container-based build in VSTS. +steps: +- task: docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: ${{ parameters.imageName }} + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + envVars: ${{ parameters.environmentVariables }} + workDir: '/build' + containerCommand: '/src/ci/build.sh' + detached: false +- task: docker@0 + displayName: Test + inputs: + action: 'Run an image' + imageName: ${{ parameters.imageName }} + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + envVars: ${{ parameters.environmentVariables }} + workDir: '/build' + containerCommand: '/src/ci/test.sh' + detached: false +- task: publishtestresults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFiles: 'results_*.xml' + searchFolder: '$(Build.BinariesDirectory)' + mergeTestResults: true diff --git a/ci/vsts-powershell.yml b/ci/vsts-powershell.yml new file mode 100644 index 000000000..a2eb175d5 --- /dev/null +++ b/ci/vsts-powershell.yml @@ -0,0 +1,17 @@ +# These are the steps used for building on machines with PowerShell. +steps: +- powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' + displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' + displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFiles: 'results_*.xml' + searchFolder: '$(Build.BinariesDirectory)' + mergeTestResults: true diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 950250418..792e6b5ff 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -54,8 +54,22 @@ IF (MSVC_IDE) SET_SOURCE_FILES_PROPERTIES("precompiled.c" COMPILE_FLAGS "/Ycprecompiled.h") ENDIF () -ADD_TEST(offline "${libgit2_BINARY_DIR}/libgit2_clar" -v -xonline) -ADD_TEST(online "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline) -ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push) -ADD_TEST(ssh "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths) -ADD_TEST(proxy "${libgit2_BINARY_DIR}/libgit2_clar" -v -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request) +IF (CLAR_XML) + IF (CLAR_XML STREQUAL "ON") + SET(XML_PATH "") + ELSE () + SET(XML_PATH "${CLAR_XML}/") + ENDIF () + + SET(TESTS_OFFLINE_XML "-r${XML_PATH}results_offline.xml") + SET(TESTS_ONLINE_XML "-r${XML_PATH}results_online.xml") + SET(TESTS_GITDAEMON_XML "-r${XML_PATH}results_gitdaemon.xml") + SET(TESTS_SSH_XML "-r${XML_PATH}results_ssh.xml") + SET(TESTS_PROXY_XML "-r${XML_PATH}results_proxy.xml") +ENDIF () + +ADD_TEST(offline "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_OFFLINE_XML} -xonline) +ADD_TEST(online "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_ONLINE_XML} -sonline) +ADD_TEST(gitdaemon "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_GITDAEMON_XML} -sonline::push) +ADD_TEST(ssh "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_SSH_XML} -sonline::push -sonline::clone::ssh_cert -sonline::clone::ssh_with_paths) +ADD_TEST(proxy "${libgit2_BINARY_DIR}/libgit2_clar" -v ${TESTS_PROXY_XML} -sonline::clone::proxy_credentials_in_url -sonline::clone::proxy_credentials_request) diff --git a/tests/clar.c b/tests/clar.c index d5212d1ca..27d35e1c7 100644 --- a/tests/clar.c +++ b/tests/clar.c @@ -95,9 +95,6 @@ static const char * fixture_path(const char *base, const char *fixture_name); struct clar_error { - const char *test; - int test_number; - const char *suite; const char *file; int line_number; const char *error_msg; @@ -106,11 +103,34 @@ struct clar_error { struct clar_error *next; }; -static struct { - int argc; - char **argv; +struct clar_explicit { + size_t suite_idx; + const char *filter; + + struct clar_explicit *next; +}; + +struct clar_report { + const char *test; + int test_number; + const char *suite; + + enum cl_test_status status; + + struct clar_error *errors; + struct clar_error *last_error; + + struct clar_report *next; +}; +struct clar_summary { + const char *filename; + FILE *fp; +}; + +static struct { enum cl_test_status test_status; + const char *active_test; const char *active_suite; @@ -124,8 +144,15 @@ static struct { int exit_on_error; int report_suite_names; - struct clar_error *errors; - struct clar_error *last_error; + int write_summary; + const char *summary_filename; + struct clar_summary *summary; + + struct clar_explicit *explicit; + struct clar_explicit *last_explicit; + + struct clar_report *reports; + struct clar_report *last_report; void (*local_cleanup)(void *); void *local_cleanup_payload; @@ -155,7 +182,7 @@ struct clar_suite { /* From clar_print_*.c */ static void clar_print_init(int test_count, int suite_count, const char *suite_names); static void clar_print_shutdown(int test_count, int suite_count, int error_count); -static void clar_print_error(int num, const struct clar_error *error); +static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); static void clar_print_ontest(const char *test_name, int test_number, enum cl_test_status failed); static void clar_print_onsuite(const char *suite_name, int suite_index); static void clar_print_onabort(const char *msg, ...); @@ -164,6 +191,10 @@ static void clar_print_onabort(const char *msg, ...); static void clar_unsandbox(void); static int clar_sandbox(void); +/* From summary.h */ +static struct clar_summary *clar_summary_init(const char *filename); +static int clar_summary_shutdown(struct clar_summary *fp); + /* Load the declarations for the test suite */ #include "clar.suite" @@ -186,21 +217,29 @@ void cl_trace_register(cl_trace_cb *cb, void *payload) /* Core test functions */ static void -clar_report_errors(void) +clar_report_errors(struct clar_report *report) { + struct clar_error *error; int i = 1; - struct clar_error *error, *next; - - error = _clar.errors; - while (error != NULL) { - next = error->next; - clar_print_error(i++, error); - free(error->description); - free(error); - error = next; - } - _clar.errors = _clar.last_error = NULL; + for (error = report->errors; error; error = error->next) + clar_print_error(i++, _clar.last_report, error); +} + +static void +clar_report_all(void) +{ + struct clar_report *report; + struct clar_error *error; + int i = 1; + + for (report = _clar.reports; report; report = report->next) { + if (report->status != CL_TEST_FAILURE) + continue; + + for (error = report->errors; error; error = error->next) + clar_print_error(i++, report, error); + } } static void @@ -209,7 +248,6 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - _clar.test_status = CL_TEST_OK; _clar.trampoline_enabled = 1; CL_TRACE(CL_TRACE__TEST__BEGIN); @@ -225,6 +263,9 @@ clar_run_test( _clar.trampoline_enabled = 0; + if (_clar.last_report->status == CL_TEST_NOTRUN) + _clar.last_report->status = CL_TEST_OK; + if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -240,9 +281,9 @@ clar_run_test( _clar.local_cleanup_payload = NULL; if (_clar.report_errors_only) { - clar_report_errors(); + clar_report_errors(_clar.last_report); } else { - clar_print_ontest(test->name, _clar.tests_ran, _clar.test_status); + clar_print_ontest(test->name, _clar.tests_ran, _clar.last_report->status); } } @@ -251,6 +292,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) { const struct clar_func *test = suite->tests; size_t i, matchlen; + struct clar_report *report; if (!suite->enabled) return; @@ -283,6 +325,21 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) continue; _clar.active_test = test[i].name; + + report = calloc(1, sizeof(struct clar_report)); + report->suite = _clar.active_suite; + report->test = _clar.active_test; + report->test_number = _clar.tests_ran; + report->status = CL_TEST_NOTRUN; + + if (_clar.reports == NULL) + _clar.reports = report; + + if (_clar.last_report != NULL) + _clar.last_report->next = report; + + _clar.last_report = report; + clar_run_test(&test[i], &suite->initialize, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) @@ -298,13 +355,14 @@ clar_usage(const char *arg) { printf("Usage: %s [options]\n\n", arg); printf("Options:\n"); - printf(" -sname\tRun only the suite with `name` (can go to individual test name)\n"); - printf(" -iname\tInclude the suite with `name`\n"); - printf(" -xname\tExclude the suite with `name`\n"); - printf(" -v \tIncrease verbosity (show suite names)\n"); - printf(" -q \tOnly report tests that had an error\n"); - printf(" -Q \tQuit as soon as a test fails\n"); - printf(" -l \tPrint suite names\n"); + printf(" -sname Run only the suite with `name` (can go to individual test name)\n"); + printf(" -iname Include the suite with `name`\n"); + printf(" -xname Exclude the suite with `name`\n"); + printf(" -v Increase verbosity (show suite names)\n"); + printf(" -q Only report tests that had an error\n"); + printf(" -Q Quit as soon as a test fails\n"); + printf(" -l Print suite names\n"); + printf(" -r[filename] Write summary file (to the optional filename)\n"); exit(-1); } @@ -318,7 +376,7 @@ clar_parse_args(int argc, char **argv) char *argument = argv[i]; if (argument[0] != '-' || argument[1] == '\0' - || strchr("sixvqQl", argument[1]) == NULL) { + || strchr("sixvqQlr", argument[1]) == NULL) { clar_usage(argv[0]); } } @@ -359,7 +417,24 @@ clar_parse_args(int argc, char **argv) _clar.report_suite_names = 1; switch (action) { - case 's': _clar_suites[j].enabled = 1; clar_run_suite(&_clar_suites[j], argument); break; + case 's': { + struct clar_explicit *explicit = + calloc(1, sizeof(struct clar_explicit)); + assert(explicit); + + explicit->suite_idx = j; + explicit->filter = argument; + + if (_clar.explicit == NULL) + _clar.explicit = explicit; + + if (_clar.last_explicit != NULL) + _clar.last_explicit->next = explicit; + + _clar_suites[j].enabled = 1; + _clar.last_explicit = explicit; + break; + } case 'i': _clar_suites[j].enabled = 1; break; case 'x': _clar_suites[j].enabled = 0; break; } @@ -397,6 +472,12 @@ clar_parse_args(int argc, char **argv) _clar.report_suite_names = 1; break; + case 'r': + _clar.write_summary = 1; + _clar.summary_filename = *(argument + 2) ? (argument + 2) : + "summary.xml"; + break; + default: assert(!"Unexpected commandline argument!"); } @@ -412,23 +493,31 @@ clar_test_init(int argc, char **argv) "" ); + if (argc > 1) + clar_parse_args(argc, argv); + + if (_clar.write_summary && + !(_clar.summary = clar_summary_init(_clar.summary_filename))) { + clar_print_onabort("Failed to open the summary file\n"); + exit(-1); + } + if (clar_sandbox() < 0) { clar_print_onabort("Failed to sandbox the test runner.\n"); exit(-1); } - - _clar.argc = argc; - _clar.argv = argv; } int clar_test_run(void) { - if (_clar.argc > 1) - clar_parse_args(_clar.argc, _clar.argv); + size_t i; + struct clar_explicit *explicit; - if (!_clar.suites_ran) { - size_t i; + if (_clar.explicit) { + for (explicit = _clar.explicit; explicit; explicit = explicit->next) + clar_run_suite(&_clar_suites[explicit->suite_idx], explicit->filter); + } else { for (i = 0; i < _clar_suite_count; ++i) clar_run_suite(&_clar_suites[i], NULL); } @@ -439,6 +528,9 @@ clar_test_run(void) void clar_test_shutdown(void) { + struct clar_explicit *explicit, *explicit_next; + struct clar_report *report, *report_next; + clar_print_shutdown( _clar.tests_ran, (int)_clar_suite_count, @@ -446,6 +538,21 @@ clar_test_shutdown(void) ); clar_unsandbox(); + + if (_clar.write_summary && clar_summary_shutdown(_clar.summary) < 0) { + clar_print_onabort("Failed to write the summary file\n"); + exit(-1); + } + + for (explicit = _clar.explicit; explicit; explicit = explicit_next) { + explicit_next = explicit->next; + free(explicit); + } + + for (report = _clar.reports; report; report = report_next) { + report_next = report->next; + free(report); + } } int @@ -465,7 +572,7 @@ static void abort_test(void) if (!_clar.trampoline_enabled) { clar_print_onabort( "Fatal error: a cleanup method raised an exception."); - clar_report_errors(); + clar_report_errors(_clar.last_report); exit(-1); } @@ -475,7 +582,7 @@ static void abort_test(void) void clar__skip(void) { - _clar.test_status = CL_TEST_SKIP; + _clar.last_report->status = CL_TEST_SKIP; _clar.total_skipped++; abort_test(); } @@ -489,17 +596,14 @@ void clar__fail( { struct clar_error *error = calloc(1, sizeof(struct clar_error)); - if (_clar.errors == NULL) - _clar.errors = error; + if (_clar.last_report->errors == NULL) + _clar.last_report->errors = error; - if (_clar.last_error != NULL) - _clar.last_error->next = error; + if (_clar.last_report->last_error != NULL) + _clar.last_report->last_error->next = error; - _clar.last_error = error; + _clar.last_report->last_error = error; - error->test = _clar.active_test; - error->test_number = _clar.tests_ran; - error->suite = _clar.active_suite; error->file = file; error->line_number = line; error->error_msg = error_msg; @@ -508,7 +612,7 @@ void clar__fail( error->description = strdup(description); _clar.total_errors++; - _clar.test_status = CL_TEST_FAILURE; + _clar.last_report->status = CL_TEST_FAILURE; if (should_abort) abort_test(); @@ -653,3 +757,4 @@ void cl_set_cleanup(void (*cleanup)(void *), void *opaque) #include "clar/fixtures.h" #include "clar/fs.h" #include "clar/print.h" +#include "clar/summary.h" diff --git a/tests/clar.h b/tests/clar.h index dfb88d76d..2f9f96b3f 100644 --- a/tests/clar.h +++ b/tests/clar.h @@ -12,13 +12,16 @@ enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, - CL_TEST_SKIP + CL_TEST_SKIP, + CL_TEST_NOTRUN, }; +/** Setup clar environment */ void clar_test_init(int argc, char *argv[]); int clar_test_run(void); void clar_test_shutdown(void); +/** One shot setup & run */ int clar_test(int argc, char *argv[]); const char *clar_sandbox_path(void); diff --git a/tests/clar/print.h b/tests/clar/print.h index 916d807c1..73c4a8953 100644 --- a/tests/clar/print.h +++ b/tests/clar/print.h @@ -13,16 +13,16 @@ static void clar_print_shutdown(int test_count, int suite_count, int error_count (void)error_count; printf("\n\n"); - clar_report_errors(); + clar_report_all(); } -static void clar_print_error(int num, const struct clar_error *error) +static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error) { printf(" %d) Failure:\n", num); printf("%s::%s [%s:%d]\n", - error->suite, - error->test, + report->suite, + report->test, error->file, error->line_number); @@ -44,6 +44,7 @@ static void clar_print_ontest(const char *test_name, int test_number, enum cl_te case CL_TEST_OK: printf("."); break; case CL_TEST_FAILURE: printf("F"); break; case CL_TEST_SKIP: printf("S"); break; + case CL_TEST_NOTRUN: printf("N"); break; } fflush(stdout); diff --git a/tests/clar/summary.h b/tests/clar/summary.h new file mode 100644 index 000000000..1af110efa --- /dev/null +++ b/tests/clar/summary.h @@ -0,0 +1,134 @@ + +#include <stdio.h> +#include <time.h> + +int clar_summary_close_tag( + struct clar_summary *summary, const char *tag, int indent) +{ + const char *indt; + + if (indent == 0) indt = ""; + else if (indent == 1) indt = "\t"; + else indt = "\t\t"; + + return fprintf(summary->fp, "%s</%s>\n", indt, tag); +} + +int clar_summary_testsuites(struct clar_summary *summary) +{ + return fprintf(summary->fp, "<testsuites>\n"); +} + +int clar_summary_testsuite(struct clar_summary *summary, + int idn, const char *name, const char *pkg, time_t timestamp, + double elapsed, int test_count, int fail_count, int error_count) +{ + struct tm *tm = localtime(×tamp); + char iso_dt[20]; + + if (strftime(iso_dt, sizeof(iso_dt), "%Y-%m-%dT%H:%M:%S", tm) == 0) + return -1; + + return fprintf(summary->fp, "\t<testsuite " + " id=\"%d\"" + " name=\"%s\"" + " package=\"%s\"" + " hostname=\"localhost\"" + " timestamp=\"%s\"" + " time=\"%.2f\"" + " tests=\"%d\"" + " failures=\"%d\"" + " errors=\"%d\">\n", + idn, name, pkg, iso_dt, elapsed, test_count, fail_count, error_count); +} + +int clar_summary_testcase(struct clar_summary *summary, + const char *name, const char *classname, double elapsed) +{ + return fprintf(summary->fp, + "\t\t<testcase name=\"%s\" classname=\"%s\" time=\"%.2f\">\n", + name, classname, elapsed); +} + +int clar_summary_failure(struct clar_summary *summary, + const char *type, const char *message, const char *desc) +{ + return fprintf(summary->fp, + "\t\t\t<failure type=\"%s\"><![CDATA[%s\n%s]]></failure>\n", + type, message, desc); +} + +struct clar_summary *clar_summary_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) + return NULL; + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +int clar_summary_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + const char *last_suite = NULL; + + if (clar_summary_testsuites(summary) < 0) + goto on_error; + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { + if (clar_summary_testsuite(summary, 0, report->suite, "", + time(NULL), 0, _clar.tests_ran, _clar.total_errors, 0) < 0) + goto on_error; + } + + last_suite = report->suite; + + clar_summary_testcase(summary, report->test, "what", 0); + + while (error != NULL) { + if (clar_summary_failure(summary, "assert", + error->error_msg, error->description) < 0) + goto on_error; + + error = error->next; + } + + if (clar_summary_close_tag(summary, "testcase", 2) < 0) + goto on_error; + + report = report->next; + + if (!report || strcmp(last_suite, report->suite) != 0) { + if (clar_summary_close_tag(summary, "testsuite", 1) < 0) + goto on_error; + } + } + + if (clar_summary_close_tag(summary, "testsuites", 0) < 0 || + fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} |