MDEV-23926: Fix warnings generated during compilation of plugin/auth_pam/mapper/pam_user_map.c on MacOS

Compiler warnings like one listed below are generated during server build on MacOS:

[88%] Building C object plugin/auth_pam/CMakeFiles/pam_user_map.dir/mapper/pam_user_map.c.o
mariadb/server-10.2/plugin/auth_pam/mapper/pam_user_map.c:87:41: error: passing
'gid_t *' (aka 'unsigned int *') to parameter of type 'int *' converts between pointers to integer types
with different sign [-Werror,-Wpointer-sign]
  if (getgrouplist(user, user_group_id, loc_groups, &ng) < 0)
     ^~~~~~~~~~
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk/usr/include/unistd.h:650:43: note:
  passing argument to parameter here
  int getgrouplist(const char *, int, int *, int *);
      ^

In case MariaDB server is build with -DCMAKE_BUILD_TYPE=Debug it results in
build error.

The reason of compiler warnings is that declaration of the Posix C API function
getgrouplist() on MacOS differs from declaration of getgrouplist() proposed
by Posix.

To suppress this compiler warning cmake configure was adapted to detect what
kind of getgrouplist() function is declared on the build platform and
set the macros HAVE_POSIX_GETGROUPLIST in case the building platform supports
Posix compatible interface for the getgrouplist() function. Depending on
whether this macros is set the compatible type of arguments is used to pass
parameter values to the function.

2 files changed, 45 insertions, 11 deletions

diff --git a/plugin/auth_pam/CMakeLists.txt b/plugin/auth_pam/CMakeLists.txt
index ac598e4ffa6..fce9b8cf8a6 100644
--- a/plugin/auth_pam/CMakeLists.txt
+++ b/plugin/auth_pam/CMakeLists.txt
@@ -5,6 +5,30 @@ CHECK_INCLUDE_FILES (security/pam_ext.h  HAVE_PAM_EXT_H)
 CHECK_INCLUDE_FILES (security/pam_appl.h HAVE_PAM_APPL_H)
 CHECK_FUNCTION_EXISTS (strndup HAVE_STRNDUP)
 
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
+
+# Check whether getgrouplist uses git_t for second and third arguments.
+SET(CMAKE_REQUIRED_FLAGS -Werror)
+CHECK_C_SOURCE_COMPILES(
+"
+#ifdef HAVE_GRP_H
+#include <grp.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+int main() {
+  char *arg_1;
+  gid_t arg_2, arg_3;
+  int arg_4;
+  (void)getgrouplist(arg_1,arg_2,&arg_3,arg_4);
+  return 0;
+}
+"
+HAVE_POSIX_GETGROUPLIST
+)
+SET(CMAKE_REQUIRED_FLAGS)
+
 SET(CMAKE_REQUIRED_LIBRARIES pam)
 CHECK_FUNCTION_EXISTS(pam_syslog HAVE_PAM_SYSLOG)
 SET(CMAKE_REQUIRED_LIBRARIES)
@@ -36,3 +60,5 @@ IF(HAVE_PAM_APPL_H)
   ENDIF()
 ENDIF(HAVE_PAM_APPL_H)
 
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake
+               ${CMAKE_CURRENT_BINARY_DIR}/config_auth_pam.h)
diff --git a/plugin/auth_pam/mapper/pam_user_map.c b/plugin/auth_pam/mapper/pam_user_map.c
index 9d7ed53f8b1..fa8d9ae08c1 100644
--- a/plugin/auth_pam/mapper/pam_user_map.c
+++ b/plugin/auth_pam/mapper/pam_user_map.c
@@ -31,6 +31,7 @@ These comments are written to the syslog as 'authpriv.debug'
 and usually end up in /var/log/secure file.
 */
 
+#include <config_auth_pam.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <ctype.h>
@@ -70,10 +71,16 @@ pam_syslog (const pam_handle_t *pamh, int priority,
 #define GROUP_BUFFER_SIZE 100
 static const char debug_keyword[]= "debug";
 
-static int populate_user_groups(const char *user, gid_t **groups)
+#ifdef HAVE_POSIX_GETGROUPLIST
+typedef gid_t my_gid_t;
+#else
+typedef int my_gid_t;
+#endif
+
+static int populate_user_groups(const char *user, my_gid_t **groups)
 {
-  gid_t user_group_id;
-  gid_t *loc_groups= *groups;
+  my_gid_t user_group_id;
+  my_gid_t *loc_groups= *groups;
   int ng;
 
   {
@@ -88,22 +95,23 @@ static int populate_user_groups(const char *user, gid_t **groups)
   {
     /* The rare case when the user is present in more than */
     /* GROUP_BUFFER_SIZE groups.                           */
-    loc_groups= (gid_t *) malloc(ng * sizeof (gid_t));
+    loc_groups= (my_gid_t *) malloc(ng * sizeof (my_gid_t));
+
     if (!loc_groups)
       return 0;
 
     (void) getgrouplist(user, user_group_id, loc_groups, &ng);
-    *groups= loc_groups;
+    *groups= (my_gid_t*)loc_groups;
   }
 
   return ng;
 }
 
 
-static int user_in_group(const gid_t *user_groups, int ng,const char *group)
+static int user_in_group(const my_gid_t *user_groups, int ng,const char *group)
 {
-  gid_t group_id;
-  const gid_t *groups_end = user_groups + ng;
+  my_gid_t group_id;
+  const my_gid_t *groups_end = user_groups + ng;
 
   {
     struct group *g= getgrnam(group);
@@ -122,7 +130,7 @@ static int user_in_group(const gid_t *user_groups, int ng,const char *group)
 }
 
 
-static void print_groups(pam_handle_t *pamh, const gid_t *user_groups, int ng)
+static void print_groups(pam_handle_t *pamh, const my_gid_t *user_groups, int ng)
 {
   char buf[256];
   char *c_buf= buf, *buf_end= buf+sizeof(buf)-2;
@@ -158,8 +166,8 @@ int pam_sm_authenticate(pam_handle_t *pamh, int flags,
   const char *username;
   char buf[256];
   FILE *f;
-  gid_t group_buffer[GROUP_BUFFER_SIZE];
-  gid_t *groups= group_buffer;
+  my_gid_t group_buffer[GROUP_BUFFER_SIZE];
+  my_gid_t *groups= group_buffer;
   int n_groups= -1;
 
   for (; argc > 0; argc--)