regexp: implement new regular expression API

We currently support a set of different regular expression backends with
PCRE, PCRE2, regcomp(3P) and regcomp_l(3). The current implementation of
this is done via a simple POSIX wrapper that either directly uses
supplied functions or that is a very small wrapper.

To support PCRE and PCRE2, we use their provided <pcreposix.h> and
<pcre2posix.h> wrappers. These wrappers are implemented in such a way
that the accompanying libraries pcre-posix and pcre2-posix provide the
same symbols as the libc ones, namely regcomp(3P) et al. This works out
on some systems just fine, most importantly on glibc-based ones, where
the regular expression functions are implemented as weak aliases and
thus get overridden by linking in the pcre{,2}-posix library. On other
systems we depend on the linking order of libc and pcre library, and as
libc always comes first we will end up with the functions of the libc
implementation. As a result, we may use the structures `regex_t` and
`regmatch_t` declared by <pcre{,2}posix.h>, but use functions defined by
the libc, leading to segfaults.

The issue is not easily solvable. Somed distributions like Debian have
resolved this by patching PCRE and PCRE2 to carry custom prefixes to all
the POSIX function wrappers. But this is not supported by upstream and
thus inherently unportable between distributions. We could instead try
to modify linking order, but this starts becoming fragile and will not
work e.g. when libgit2 is loaded via dlopen(3P) or similar ways. In the
end, this means that we simply cannot use the POSIX wrappers provided by
the PCRE libraries at all.

Thus, this commit introduces a new regular expression API. The new API
is on a tad higher level than the previous POSIX abstraction layer, as
it tries to abstract away any non-portable flags like e.g. REG_EXTENDED,
which has no equivalents in all of our supported backends. As there are
no users of POSIX regular expressions that do _not_ reguest REG_EXTENDED
this is fine to be abstracted away, though. Due to the API being
higher-level than before, it should generally be a tad easier to use
than the previous one.

Note: ideally, the new API would've been called `git_regex_foobar` with
a file "regex.h" and "regex.c". Unfortunately, this is currently
impossible to implement due to naming clashes between the then-existing
"regex.h" and <regex.h> provided by the libc. As we add the source
directory of libgit2 to the header search path, an include of <regex.h>
would always find our own "regex.h". Thus, we have to take the bitter
pill of adding one more character to all the functions to disambiguate
the includes.

To improve guarantees around cross-backend compatibility, this commit
also brings along an improved regular expression test suite
core::regexp.

3 files changed, 531 insertions, 0 deletions

diff --git a/src/regexp.c b/src/regexp.c
new file mode 100644
index 000000000..05ed1907c
--- /dev/null
+++ b/src/regexp.c
@@ -0,0 +1,221 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "regexp.h"
+
+#if defined(GIT_REGEX_BUILTIN) || defined(GIT_REGEX_PCRE)
+
+int git_regexp_compile(git_regexp *r, const char *pattern, int flags)
+{
+	int erroffset, cflags = 0;
+	const char *error;
+
+	if (flags & GIT_REGEXP_ICASE)
+		cflags |= PCRE_CASELESS;
+
+	if ((*r = pcre_compile(pattern, cflags, &error, &erroffset, NULL)) == NULL) {
+		git_error_set_str(GIT_ERROR_REGEX, error);
+		return GIT_EINVALIDSPEC;
+	}
+
+	return 0;
+}
+
+void git_regexp_dispose(git_regexp *r)
+{
+	pcre_free(*r);
+	*r = NULL;
+}
+
+int git_regexp_match(const git_regexp *r, const char *string)
+{
+	int error;
+	if ((error = pcre_exec(*r, NULL, string, (int) strlen(string), 0, 0, NULL, 0)) < 0)
+		return (error == PCRE_ERROR_NOMATCH) ? GIT_ENOTFOUND : GIT_EINVALIDSPEC;
+	return 0;
+}
+
+int git_regexp_search(const git_regexp *r, const char *string, size_t nmatches, git_regmatch *matches)
+{
+	int static_ovec[9], *ovec;
+	int error;
+	size_t i;
+
+	/* The ovec array always needs to be a mutiple of three */
+	if (nmatches <= ARRAY_SIZE(static_ovec) / 3)
+		ovec = static_ovec;
+	else
+		ovec = git__calloc(nmatches * 3, sizeof(*ovec));
+	GIT_ERROR_CHECK_ALLOC(ovec);
+
+	if ((error = pcre_exec(*r, NULL, string, (int) strlen(string), 0, 0, ovec, (int) nmatches * 3)) < 0)
+		goto out;
+
+	if (error == 0)
+		error = (int) nmatches;
+
+	for (i = 0; i < (unsigned int) error; i++) {
+		matches[i].start = (ovec[i * 2] < 0) ? -1 : ovec[i * 2];
+		matches[i].end = (ovec[i * 2 + 1] < 0) ? -1 : ovec[i * 2 + 1];
+	}
+	for (i = (unsigned int) error; i < nmatches; i++)
+		matches[i].start = matches[i].end = -1;
+
+out:
+	if (nmatches > ARRAY_SIZE(static_ovec) / 3)
+		git__free(ovec);
+	if (error < 0)
+		return (error == PCRE_ERROR_NOMATCH) ? GIT_ENOTFOUND : GIT_EINVALIDSPEC;
+	return 0;
+}
+
+#elif defined(GIT_REGEX_PCRE2)
+
+int git_regexp_compile(git_regexp *r, const char *pattern, int flags)
+{
+	unsigned char errmsg[1024];
+	unsigned long erroff;
+	int error, cflags = 0;
+
+	if (flags & GIT_REGEXP_ICASE)
+		cflags |= PCRE2_CASELESS;
+
+	if ((*r = pcre2_compile((const unsigned char *) pattern, PCRE2_ZERO_TERMINATED,
+				cflags, &error, &erroff, NULL)) == NULL) {
+		pcre2_get_error_message(error, errmsg, sizeof(errmsg));
+		git_error_set_str(GIT_ERROR_REGEX, (char *) errmsg);
+		return GIT_EINVALIDSPEC;
+	}
+
+	return 0;
+}
+
+void git_regexp_dispose(git_regexp *r)
+{
+	pcre2_code_free(*r);
+	*r = NULL;
+}
+
+int git_regexp_match(const git_regexp *r, const char *string)
+{
+	pcre2_match_data *data;
+	int error;
+
+	data = pcre2_match_data_create(1, NULL);
+	GIT_ERROR_CHECK_ALLOC(data);
+
+	if ((error = pcre2_match(*r, (const unsigned char *) string, strlen(string),
+			 0, 0, data, NULL)) < 0)
+		return (error == PCRE2_ERROR_NOMATCH) ? GIT_ENOTFOUND : GIT_EINVALIDSPEC;
+
+	pcre2_match_data_free(data);
+	return 0;
+}
+
+int git_regexp_search(const git_regexp *r, const char *string, size_t nmatches, git_regmatch *matches)
+{
+	pcre2_match_data *data = NULL;
+	PCRE2_SIZE *ovec;
+	int error;
+	size_t i;
+
+	if ((data = pcre2_match_data_create(nmatches, NULL)) == NULL) {
+		git_error_set_oom();
+		goto out;
+	}
+
+	if ((error = pcre2_match(*r, (const unsigned char *) string, strlen(string),
+			     0, 0, data, NULL)) < 0)
+		goto out;
+
+	if (error == 0 || (unsigned int) error  > nmatches)
+		error = nmatches;
+	ovec = pcre2_get_ovector_pointer(data);
+
+	for (i = 0; i < (unsigned int) error; i++) {
+		matches[i].start = (ovec[i * 2] == PCRE2_UNSET) ? -1 : (ssize_t) ovec[i * 2];
+		matches[i].end = (ovec[i * 2 + 1] == PCRE2_UNSET) ? -1 :  (ssize_t) ovec[i * 2 + 1];
+	}
+	for (i = (unsigned int) error; i < nmatches; i++)
+		matches[i].start = matches[i].end = -1;
+
+out:
+	pcre2_match_data_free(data);
+	if (error < 0)
+		return (error == PCRE2_ERROR_NOMATCH) ? GIT_ENOTFOUND : GIT_EINVALIDSPEC;
+	return 0;
+}
+
+#elif defined(GIT_REGEX_REGCOMP) || defined(GIT_REGEX_REGCOMP_L)
+
+#if defined(GIT_REGEX_REGCOMP_L)
+# include <xlocale.h>
+#endif
+
+int git_regexp_compile(git_regexp *r, const char *pattern, int flags)
+{
+	int cflags = REG_EXTENDED, error;
+	char errmsg[1024];
+
+	if (flags & GIT_REGEXP_ICASE)
+		cflags |= REG_ICASE;
+
+# if defined(GIT_REGEX_REGCOMP)
+	if ((error = regcomp(r, pattern, cflags)) != 0)
+# else
+	if ((error = regcomp_l(r, pattern, cflags, (locale_t) 0)) != 0)
+# endif
+	{
+		regerror(error, r, errmsg, sizeof(errmsg));
+		git_error_set_str(GIT_ERROR_REGEX, errmsg);
+		return GIT_EINVALIDSPEC;
+	}
+
+	return 0;
+}
+
+void git_regexp_dispose(git_regexp *r)
+{
+	regfree(r);
+}
+
+int git_regexp_match(const git_regexp *r, const char *string)
+{
+	int error;
+	if ((error = regexec(r, string, 0, NULL, 0)) != 0)
+		return (error == REG_NOMATCH) ? GIT_ENOTFOUND : GIT_EINVALIDSPEC;
+	return 0;
+}
+
+int git_regexp_search(const git_regexp *r, const char *string, size_t nmatches, git_regmatch *matches)
+{
+	regmatch_t static_m[3], *m;
+	int error;
+	size_t i;
+
+	if (nmatches <= ARRAY_SIZE(static_m))
+		m = static_m;
+	else
+		m = git__calloc(nmatches, sizeof(*m));
+
+	if ((error = regexec(r, string, nmatches, m, 0)) != 0)
+		goto out;
+
+	for (i = 0; i < nmatches; i++) {
+		matches[i].start = (m[i].rm_so < 0) ? -1 : m[i].rm_so;
+		matches[i].end = (m[i].rm_eo < 0) ? -1 : m[i].rm_eo;
+	}
+
+out:
+	if (nmatches > ARRAY_SIZE(static_m))
+		git__free(m);
+	if (error)
+		return (error == REG_NOMATCH) ? GIT_ENOTFOUND : GIT_EINVALIDSPEC;
+	return 0;
+}
+
+#endif
diff --git a/src/regexp.h b/src/regexp.h
new file mode 100644
index 000000000..2592ef383
--- /dev/null
+++ b/src/regexp.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_regexp_h__
+#define INCLUDE_regexp_h__
+
+#include "common.h"
+
+#if defined(GIT_REGEX_BUILTIN) || defined(GIT_REGEX_PCRE)
+# include "pcre.h"
+typedef pcre *git_regexp;
+# define GIT_REGEX_INIT NULL
+#elif defined(GIT_REGEX_PCRE2)
+# define PCRE2_CODE_UNIT_WIDTH 8
+# include <pcre2.h>
+typedef pcre2_code *git_regexp;
+# define GIT_REGEX_INIT NULL
+#elif defined(GIT_REGEX_REGCOMP) || defined(GIT_REGEX_REGCOMP_L)
+# include <regex.h>
+typedef regex_t git_regexp;
+# define GIT_REGEX_INIT { 0 }
+#else
+# error "No regex backend"
+#endif
+
+/** Options supported by @git_regexp_compile. */
+typedef enum {
+	/** Enable case-insensitive matching */
+	GIT_REGEXP_ICASE = (1 << 0)
+} git_regexp_flags_t;
+
+/** Structure containing information about regular expression matching groups */
+typedef struct {
+	/** Start of the given match. -1 if the group didn't match anything */
+	ssize_t start;
+	/** End of the given match. -1 if the group didn't match anything */
+	ssize_t end;
+} git_regmatch;
+
+/**
+ * Compile a regular expression. The compiled expression needs to
+ * be cleaned up afterwards with `git_regexp_dispose`.
+ *
+ * @param r Pointer to the storage where to initialize the regular expression.
+ * @param pattern The pattern that shall be compiled.
+ * @param flags Flags to alter how the pattern shall be handled.
+ *              0 for defaults, otherwise see @git_regexp_flags_t.
+ * @return 0 on success, otherwise a negative return value.
+ */
+int git_regexp_compile(git_regexp *r, const char *pattern, int flags);
+
+/**
+ * Free memory associated with the regular expression
+ *
+ * @param r The regular expression structure to dispose.
+ */
+void git_regexp_dispose(git_regexp *r);
+
+/**
+ * Test whether a given string matches a compiled regular
+ * expression.
+ *
+ * @param r Compiled regular expression.
+ * @param string String to match against the regular expression.
+ * @return 0 if the string matches, a negative error code
+ *         otherwise. GIT_ENOTFOUND if no match was found,
+ *         GIT_EINVALIDSPEC if the regular expression matching
+ *         was invalid.
+ */
+int git_regexp_match(const git_regexp *r, const char *string);
+
+/**
+ * Search for matches inside of a given string.
+ *
+ * Given a regular expression with capturing groups, this
+ * function will populate provided @git_regmatch structures with
+ * offsets for each of the given matches. Non-matching groups
+ * will have start and end values of the respective @git_regmatch
+ * structure set to -1.
+ *
+ * @param r Compiled regular expression.
+ * @param string String to match against the regular expression.
+ * @param nmatches Number of @git_regmatch structures provided by
+ *                 the user.
+ * @param matches Pointer to an array of @git_regmatch structures.
+ * @return 0 if the string matches, a negative error code
+ *         otherwise. GIT_ENOTFOUND if no match was found,
+ *         GIT_EINVALIDSPEC if the regular expression matching
+ *         was invalid.
+ */
+int git_regexp_search(const git_regexp *r, const char *string, size_t nmatches, git_regmatch *matches);
+
+#endif
diff --git a/tests/core/regexp.c b/tests/core/regexp.c
new file mode 100644
index 000000000..8db5641e5
--- /dev/null
+++ b/tests/core/regexp.c
@@ -0,0 +1,213 @@
+#include "clar_libgit2.h"
+
+#include <locale.h>
+
+#include "regexp.h"
+#include "userdiff.h"
+
+#if LC_ALL > 0
+static const char *old_locales[LC_ALL];
+#endif
+
+static git_regexp regex;
+
+void test_core_regexp__initialize(void)
+{
+#if LC_ALL > 0
+	memset(&old_locales, 0, sizeof(old_locales));
+#endif
+}
+
+void test_core_regexp__cleanup(void)
+{
+	git_regexp_dispose(&regex);
+}
+
+static void try_set_locale(int category)
+{
+#if LC_ALL > 0
+	old_locales[category] = setlocale(category, NULL);
+#endif
+
+	if (!setlocale(category, "UTF-8") &&
+	    !setlocale(category, "c.utf8") &&
+	    !setlocale(category, "en_US.UTF-8"))
+		cl_skip();
+
+	if (MB_CUR_MAX == 1)
+		cl_fail("Expected locale to be switched to multibyte");
+}
+
+
+void test_core_regexp__compile_ignores_global_locale_ctype(void)
+{
+	try_set_locale(LC_CTYPE);
+	cl_git_pass(git_regexp_compile(&regex, "[\xc0-\xff][\x80-\xbf]", 0));
+}
+
+void test_core_regexp__compile_ignores_global_locale_collate(void)
+{
+#ifdef GIT_WIN32
+	cl_skip();
+#endif
+
+	try_set_locale(LC_COLLATE);
+	cl_git_pass(git_regexp_compile(&regex, "[\xc0-\xff][\x80-\xbf]", 0));
+}
+
+void test_core_regexp__regex_matches_digits_with_locale(void)
+{
+	char c, str[2];
+
+#ifdef GIT_WIN32
+	cl_skip();
+#endif
+
+	try_set_locale(LC_COLLATE);
+	try_set_locale(LC_CTYPE);
+
+	cl_git_pass(git_regexp_compile(&regex, "[[:digit:]]", 0));
+
+	str[1] = '\0';
+	for (c = '0'; c <= '9'; c++) {
+	    str[0] = c;
+	    cl_git_pass(git_regexp_match(&regex, str));
+	}
+}
+
+void test_core_regexp__regex_matches_alphabet_with_locale(void)
+{
+	char c, str[2];
+
+#ifdef GIT_WIN32
+	cl_skip();
+#endif
+
+	try_set_locale(LC_COLLATE);
+	try_set_locale(LC_CTYPE);
+
+	cl_git_pass(git_regexp_compile(&regex, "[[:alpha:]]", 0));
+
+	str[1] = '\0';
+	for (c = 'a'; c <= 'z'; c++) {
+	    str[0] = c;
+	    cl_git_pass(git_regexp_match(&regex, str));
+	}
+	for (c = 'A'; c <= 'Z'; c++) {
+	    str[0] = c;
+	    cl_git_pass(git_regexp_match(&regex, str));
+	}
+}
+
+void test_core_regexp__compile_userdiff_regexps(void)
+{
+	size_t idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(builtin_defs); ++idx) {
+		git_diff_driver_definition ddef = builtin_defs[idx];
+
+		cl_git_pass(git_regexp_compile(&regex, ddef.fns, ddef.flags));
+		git_regexp_dispose(&regex);
+
+		cl_git_pass(git_regexp_compile(&regex, ddef.words, 0));
+		git_regexp_dispose(&regex);
+	}
+}
+
+void test_core_regexp__simple_search_matches(void)
+{
+	cl_git_pass(git_regexp_compile(&regex, "a", 0));
+	cl_git_pass(git_regexp_search(&regex, "a", 0, NULL));
+}
+
+void test_core_regexp__case_insensitive_search_matches(void)
+{
+	cl_git_pass(git_regexp_compile(&regex, "a", GIT_REGEXP_ICASE));
+	cl_git_pass(git_regexp_search(&regex, "A", 0, NULL));
+}
+
+void test_core_regexp__nonmatching_search_returns_error(void)
+{
+	cl_git_pass(git_regexp_compile(&regex, "a", 0));
+	cl_git_fail(git_regexp_search(&regex, "b", 0, NULL));
+}
+
+void test_core_regexp__search_finds_complete_match(void)
+{
+	git_regmatch matches[1];
+
+	cl_git_pass(git_regexp_compile(&regex, "abc", 0));
+	cl_git_pass(git_regexp_search(&regex, "abc", 1, matches));
+	cl_assert_equal_i(matches[0].start, 0);
+	cl_assert_equal_i(matches[0].end, 3);
+}
+
+void test_core_regexp__search_finds_correct_offsets(void)
+{
+	git_regmatch matches[3];
+
+	cl_git_pass(git_regexp_compile(&regex, "(a*)(b*)", 0));
+	cl_git_pass(git_regexp_search(&regex, "ab", 3, matches));
+	cl_assert_equal_i(matches[0].start, 0);
+	cl_assert_equal_i(matches[0].end, 2);
+	cl_assert_equal_i(matches[1].start, 0);
+	cl_assert_equal_i(matches[1].end, 1);
+	cl_assert_equal_i(matches[2].start, 1);
+	cl_assert_equal_i(matches[2].end, 2);
+}
+
+void test_core_regexp__search_finds_empty_group(void)
+{
+	git_regmatch matches[3];
+
+	cl_git_pass(git_regexp_compile(&regex, "(a*)(b*)c", 0));
+	cl_git_pass(git_regexp_search(&regex, "ac", 3, matches));
+	cl_assert_equal_i(matches[0].start, 0);
+	cl_assert_equal_i(matches[0].end, 2);
+	cl_assert_equal_i(matches[1].start, 0);
+	cl_assert_equal_i(matches[1].end, 1);
+	cl_assert_equal_i(matches[2].start, 1);
+	cl_assert_equal_i(matches[2].end, 1);
+}
+
+void test_core_regexp__search_fills_matches_with_first_matching_groups(void)
+{
+	git_regmatch matches[2];
+
+	cl_git_pass(git_regexp_compile(&regex, "(a)(b)(c)", 0));
+	cl_git_pass(git_regexp_search(&regex, "abc", 2, matches));
+	cl_assert_equal_i(matches[0].start, 0);
+	cl_assert_equal_i(matches[0].end, 3);
+	cl_assert_equal_i(matches[1].start, 0);
+	cl_assert_equal_i(matches[1].end, 1);
+}
+
+void test_core_regexp__search_skips_nonmatching_group(void)
+{
+	git_regmatch matches[4];
+
+	cl_git_pass(git_regexp_compile(&regex, "(a)(b)?(c)", 0));
+	cl_git_pass(git_regexp_search(&regex, "ac", 4, matches));
+	cl_assert_equal_i(matches[0].start, 0);
+	cl_assert_equal_i(matches[0].end, 2);
+	cl_assert_equal_i(matches[1].start, 0);
+	cl_assert_equal_i(matches[1].end, 1);
+	cl_assert_equal_i(matches[2].start, -1);
+	cl_assert_equal_i(matches[2].end, -1);
+	cl_assert_equal_i(matches[3].start, 1);
+	cl_assert_equal_i(matches[3].end, 2);
+}
+
+void test_core_regexp__search_initializes_trailing_nonmatching_groups(void)
+{
+	git_regmatch matches[3];
+
+	cl_git_pass(git_regexp_compile(&regex, "(a)bc", 0));
+	cl_git_pass(git_regexp_search(&regex, "abc", 3, matches));
+	cl_assert_equal_i(matches[0].start, 0);
+	cl_assert_equal_i(matches[0].end, 3);
+	cl_assert_equal_i(matches[1].start, 0);
+	cl_assert_equal_i(matches[1].end, 1);
+	cl_assert_equal_i(matches[2].start, -1);
+	cl_assert_equal_i(matches[2].end, -1);
+}