util: detect all possible qsort_r and qsort_s variants

As reported in https://bugs.freebsd.org/271234, recent versions of
FreeBSD have adjusted the prototype for qsort_r() to match the POSIX
interface. This causes libgit2's CMake configuration check to fail to
detect qsort_r(), making it fall back to qsort_s(), which in libgit2
also has an incompatible interface. With recent versions of clang this
results in a "incompatible function pointer types" compile error.

Summarizing, there are four variations of 'qsort-with-context':
* old style BSD qsort_r(), used in FreeBSD 13 and earlier, where the
  comparison function has the context parameter first
* GNU or POSIX qsort_r(), also used in FreeBSD 14 and later, where the
  comparison function has the context parameter last
* C11 qsort_s(), where the comparison function has the context parameter
  last
* Microsoft qsort_s(), where the comparison function has the context
  parameter first

Add explicit detections for all these variants, so they get detected as
(in the same order as above):

* `GIT_QSORT_R_BSD`
* `GIT_QSORT_R_GNU`
* `GIT_QSORT_S_C11`
* `GIT_QSORT_S_MSC`

An additional complication is that on FreeBSD 14 and later, <stdlib.h>
uses the C11 _Generic() macro mechanism to automatically select the
correct qsort_r() prototype, depending on the caller's comparison
function argument. This breaks CMake's check_prototype_definition()
functionality, since it tries to redefine the function, and _Generic
macro is expanded inline causing a compile error.

Work around that problem by putting the function names in parentheses,
to prevent the preprocessor from using a macro to replace the function
name.

Also, in `git__qsort_r()`, change the `#if` order so the variants that
do not have to use glue are preferred.

3 files changed, 29 insertions, 11 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index cc0a0d4dc..de591e4e4 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -58,15 +58,29 @@ add_feature_info(futimens GIT_USE_FUTIMENS "futimens support")
 
 # qsort
 
+# old-style FreeBSD qsort_r() has the 'context' parameter as the first argument
+# of the comparison function:
 check_prototype_definition(qsort_r
-	"void qsort_r(void *base, size_t nmemb, size_t size, void *thunk, int (*compar)(void *, const void *, const void *))"
+	"void (qsort_r)(void *base, size_t nmemb, size_t size, void *context, int (*compar)(void *, const void *, const void *))"
 	"" "stdlib.h" GIT_QSORT_R_BSD)
 
+# GNU or POSIX qsort_r() has the 'context' parameter as the last argument of the
+# comparison function:
 check_prototype_definition(qsort_r
-	"void qsort_r(void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *, void *), void *arg)"
+	"void (qsort_r)(void *base, size_t nmemb, size_t size, int (*compar)(const void *, const void *, void *), void *context)"
 	"" "stdlib.h" GIT_QSORT_R_GNU)
 
-check_function_exists(qsort_s GIT_QSORT_S)
+# C11 qsort_s() has the 'context' parameter as the last argument of the
+# comparison function, and returns an error status:
+check_prototype_definition(qsort_s
+	"errno_t (qsort_s)(void *base, rsize_t nmemb, rsize_t size, int (*compar)(const void *, const void *, void *), void *context)"
+	"0" "stdlib.h" GIT_QSORT_S_C11)
+
+# MSC qsort_s() has the 'context' parameter as the first argument of the
+# comparison function, and as the last argument of qsort_s():
+check_prototype_definition(qsort_s
+	"void (qsort_s)(void *base, size_t num, size_t width, int (*compare )(void *, const void *, const void *), void *context)"
+	"" "stdlib.h" GIT_QSORT_S_MSC)
 
 # random / entropy data
 
diff --git a/src/util/git2_features.h.in b/src/util/git2_features.h.in
index 1575be641..5f606df84 100644
--- a/src/util/git2_features.h.in
+++ b/src/util/git2_features.h.in
@@ -26,7 +26,8 @@
 
 #cmakedefine GIT_QSORT_R_BSD
 #cmakedefine GIT_QSORT_R_GNU
-#cmakedefine GIT_QSORT_S
+#cmakedefine GIT_QSORT_S_C11
+#cmakedefine GIT_QSORT_S_MSC
 
 #cmakedefine GIT_SSH 1
 #cmakedefine GIT_SSH_MEMORY_CREDENTIALS 1
diff --git a/src/util/util.c b/src/util/util.c
index 9c9f2c040..f4eaf5835 100644
--- a/src/util/util.c
+++ b/src/util/util.c
@@ -18,7 +18,7 @@
 # endif
 # include <windows.h>
 
-# ifdef GIT_QSORT_S
+# ifdef GIT_QSORT_S_MSC
 #  include <search.h>
 # endif
 #endif
@@ -673,7 +673,7 @@ size_t git__unescape(char *str)
 	return (pos - str);
 }
 
-#if defined(GIT_QSORT_S) || defined(GIT_QSORT_R_BSD)
+#if defined(GIT_QSORT_S_MSC) || defined(GIT_QSORT_R_BSD)
 typedef struct {
 	git__sort_r_cmp cmp;
 	void *payload;
@@ -690,7 +690,8 @@ static int GIT_LIBGIT2_CALL git__qsort_r_glue_cmp(
 
 #if !defined(GIT_QSORT_R_BSD) && \
 	!defined(GIT_QSORT_R_GNU) && \
-	!defined(GIT_QSORT_S)
+	!defined(GIT_QSORT_S_C11) && \
+	!defined(GIT_QSORT_S_MSC)
 static void swap(uint8_t *a, uint8_t *b, size_t elsize)
 {
 	char tmp[256];
@@ -721,12 +722,14 @@ static void insertsort(
 void git__qsort_r(
 	void *els, size_t nel, size_t elsize, git__sort_r_cmp cmp, void *payload)
 {
-#if defined(GIT_QSORT_R_BSD)
+#if defined(GIT_QSORT_R_GNU)
+	qsort_r(els, nel, elsize, cmp, payload);
+#elif defined(GIT_QSORT_S_C11)
+	qsort_s(els, nel, elsize, cmp, payload);
+#elif defined(GIT_QSORT_R_BSD)
 	git__qsort_r_glue glue = { cmp, payload };
 	qsort_r(els, nel, elsize, &glue, git__qsort_r_glue_cmp);
-#elif defined(GIT_QSORT_R_GNU)
-	qsort_r(els, nel, elsize, cmp, payload);
-#elif defined(GIT_QSORT_S)
+#elif defined(GIT_QSORT_S_MSC)
 	git__qsort_r_glue glue = { cmp, payload };
 	qsort_s(els, nel, elsize, git__qsort_r_glue_cmp, &glue);
 #else