MDEV-25195: AIX pam fixesbb-10.4-danielblack-MDEV-25195-aix-pam

AIX doesn't have getgrouplist so use getgroupsbyname instead.

Minor API differences also covered by compile time defined(_AIX).

4 files changed, 35 insertions, 5 deletions

diff --git a/plugin/auth_pam/CMakeLists.txt b/plugin/auth_pam/CMakeLists.txt
index f7d8e019751..cc9d4071e2c 100644
--- a/plugin/auth_pam/CMakeLists.txt
+++ b/plugin/auth_pam/CMakeLists.txt
@@ -4,10 +4,12 @@ INCLUDE (CheckFunctionExists)
 CHECK_INCLUDE_FILES (security/pam_ext.h  HAVE_PAM_EXT_H)
 CHECK_INCLUDE_FILES (security/pam_appl.h HAVE_PAM_APPL_H)
 CHECK_FUNCTION_EXISTS (strndup HAVE_STRNDUP)
+CHECK_FUNCTION_EXISTS (getgrouplist HAVE_GETGROUPLIST)
+CHECK_FUNCTION_EXISTS (getgroupsbyname HAVE_GETGROUPSBYNAME)
 
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
 
-# Check whether getgrouplist uses git_t for second and third arguments.
+# Check whether getgrouplist uses gtid_t for second and third arguments.
 SET(CMAKE_REQUIRED_FLAGS -Werror)
 CHECK_C_SOURCE_COMPILES(
 "
@@ -29,7 +31,7 @@ SET(CMAKE_REQUIRED_LIBRARIES pam)
 CHECK_FUNCTION_EXISTS(pam_syslog HAVE_PAM_SYSLOG)
 SET(CMAKE_REQUIRED_LIBRARIES)
 
-IF(HAVE_PAM_APPL_H)
+IF(HAVE_PAM_APPL_H AND (HAVE_GETGROUPLIST OR HAVE_GETGROUPSBYNAME))
   FIND_LIBRARY(PAM_LIBRARY pam) # for srpm build-depends detection
   ADD_DEFINITIONS(-D_GNU_SOURCE)
   MYSQL_ADD_PLUGIN(auth_pam_v1 auth_pam_v1.c LINK_LIBRARIES pam MODULE_ONLY)
@@ -54,7 +56,7 @@ IF(HAVE_PAM_APPL_H)
       SET(CPACK_RPM_server_USER_FILELIST ${CPACK_RPM_server_USER_FILELIST} "%config(noreplace) ${INSTALL_PAMDATADIR}/*" PARENT_SCOPE)
     ENDIF()
   ENDIF()
-ENDIF(HAVE_PAM_APPL_H)
+ENDIF()
 
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake
                ${CMAKE_CURRENT_BINARY_DIR}/config_auth_pam.h)
diff --git a/plugin/auth_pam/auth_pam_base.c b/plugin/auth_pam/auth_pam_base.c
index 1e8f4a08def..d22a6077a16 100644
--- a/plugin/auth_pam/auth_pam_base.c
+++ b/plugin/auth_pam/auth_pam_base.c
@@ -63,7 +63,12 @@ static char pam_debug = 0;
 
 static char winbind_hack = 0;
 
-static int conv(int n, const struct pam_message **msg,
+static int conv(int n,
+#ifdef _AIX
+                struct pam_message **msg,
+#else
+                const struct pam_message **msg,
+#endif
                 struct pam_response **resp, void *data)
 {
   struct param *param = (struct param *)data;
@@ -128,7 +133,7 @@ static int conv(int n, const struct pam_message **msg,
 
 #define DO(X) if ((status = (X)) != PAM_SUCCESS) goto end
 
-#if defined(SOLARIS) || defined(__sun)
+#if defined(SOLARIS) || defined(__sun) || defined (_AIX)
 typedef void** pam_get_item_3_arg;
 #else
 typedef const void** pam_get_item_3_arg;
diff --git a/plugin/auth_pam/config.h.cmake b/plugin/auth_pam/config.h.cmake
index 2a60e99d52c..8b8033faaab 100644
--- a/plugin/auth_pam/config.h.cmake
+++ b/plugin/auth_pam/config.h.cmake
@@ -3,3 +3,5 @@
 #cmakedefine HAVE_PAM_EXT_H 1
 #cmakedefine HAVE_PAM_APPL_H 1
 #cmakedefine HAVE_STRNDUP 1
+#cmakedefine HAVE_GETGROUPLIST 1
+#cmakedefine HAVE_GETGROUPSBYNAME 1
diff --git a/plugin/auth_pam/mapper/pam_user_map.c b/plugin/auth_pam/mapper/pam_user_map.c
index fa8d9ae08c1..14710deb93c 100644
--- a/plugin/auth_pam/mapper/pam_user_map.c
+++ b/plugin/auth_pam/mapper/pam_user_map.c
@@ -40,6 +40,10 @@ and usually end up in /var/log/secure file.
 #include <grp.h>
 #include <pwd.h>
 
+#if defined(HAVE_GETGROUPSBYNAME)
+#include <unistd.h>
+#endif
+
 #ifdef HAVE_PAM_EXT_H
 #include <security/pam_ext.h>
 #endif
@@ -91,16 +95,33 @@ static int populate_user_groups(const char *user, my_gid_t **groups)
   }
 
   ng= GROUP_BUFFER_SIZE;
+#if defined(HAVE_GETGROUPSBYNAME)
+  loc_group[0]= user_group_id;
+  if (getgroupsbyname(user, ng - 1, loc_group + 1) < 0)
+#elif defined(HAVE_GETGROUPLIST)
   if (getgrouplist(user, user_group_id, loc_groups, &ng) < 0)
+#else
+#error missing implementation to retreive groups
+#endif
   {
     /* The rare case when the user is present in more than */
     /* GROUP_BUFFER_SIZE groups.                           */
+#ifdef HAVE_GETGROUPLIST
     loc_groups= (my_gid_t *) malloc(ng * sizeof (my_gid_t));
+#else
+    loc_groups= (my_gid_t *) malloc((getgroupsbyname(user, 0, groups) + 1) *
+                                    sizeof (my_gid_t));
+#endif
 
     if (!loc_groups)
       return 0;
 
+#ifdef HAVE_GETGROUPLIST
     (void) getgrouplist(user, user_group_id, loc_groups, &ng);
+#else
+    (void) getgroupsbyname(user, ng, loc_group);
+    loc_group[ng++]= user_group_id;
+#endif
     *groups= (my_gid_t*)loc_groups;
   }