file-has-acl: port to Fedora 39

Fedora 39 getxattr with XATTR_NAME_POSIX_ACL_ACCESS either
succeeds or fails with ENODATA, so it is no longer possible to
detect from its failure that the filesystem might support NFSv4 ACLs.
Problem reported by Ondrej Valousek in:
https://lists.gnu.org/r/bug-gnulib/2023-04/msg00228.html
Instead, use listxattr to determine whether NFSv4 ACLs are in play.
This typically saves syscalls anyway.
* lib/file-has-acl.c: In #if, use (HAVE_LINUX_XATTR_H &&
HAVE_LISTXATTR) instead of GETXATTR_WITH_POSIX_ACLS.
The following changes apply when (USE_ACL && HAVE_LINUX_XATTR_H &&
HAVE_LISTXATTR):
Include minmax.h.
(have_xattr): New function.
(file_has_acl): Try listxattr first; typically this means we need
to do no other syscall.  Call getxattr only if there are NFSv4
ACLs but not POSIX ACLs.
* m4/acl.m4 (gl_FILE_HAS_ACL): Simplify by merely testing for
linux/xattr.h and listxattr.  All uses changed.

3 files changed, 105 insertions, 57 deletions

diff --git a/ChangeLog b/ChangeLog
index 7d99b6f4cc..169645e055 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,25 @@
+2023-05-12  Paul Eggert  <eggert@cs.ucla.edu>
+
+	file-has-acl: port to Fedora 39
+	Fedora 39 getxattr with XATTR_NAME_POSIX_ACL_ACCESS either
+	succeeds or fails with ENODATA, so it is no longer possible to
+	detect from its failure that the filesystem might support NFSv4 ACLs.
+	Problem reported by Ondrej Valousek in:
+	https://lists.gnu.org/r/bug-gnulib/2023-04/msg00228.html
+	Instead, use listxattr to determine whether NFSv4 ACLs are in play.
+	This typically saves syscalls anyway.
+	* lib/file-has-acl.c: In #if, use (HAVE_LINUX_XATTR_H &&
+	HAVE_LISTXATTR) instead of GETXATTR_WITH_POSIX_ACLS.
+	The following changes apply when (USE_ACL && HAVE_LINUX_XATTR_H &&
+	HAVE_LISTXATTR):
+	Include minmax.h.
+	(have_xattr): New function.
+	(file_has_acl): Try listxattr first; typically this means we need
+	to do no other syscall.  Call getxattr only if there are NFSv4
+	ACLs but not POSIX ACLs.
+	* m4/acl.m4 (gl_FILE_HAS_ACL): Simplify by merely testing for
+	linux/xattr.h and listxattr.  All uses changed.
+
 2023-05-10  Josh Soref  <jsoref@gmail.com>
 
 	bootstrap: spelling/grammar fix in comment
diff --git a/lib/file-has-acl.c b/lib/file-has-acl.c
index b31a2ea252..1edcd2cbd6 100644
--- a/lib/file-has-acl.c
+++ b/lib/file-has-acl.c
@@ -29,7 +29,7 @@
 
 #include "acl-internal.h"
 
-#if USE_ACL && GETXATTR_WITH_POSIX_ACLS
+#if USE_ACL && HAVE_LINUX_XATTR_H && HAVE_LISTXATTR
 # include <string.h>
 # include <arpa/inet.h>
 # include <sys/xattr.h>
@@ -44,6 +44,20 @@ enum {
   ACE4_IDENTIFIER_GROUP        = 0x00000040
 };
 
+/* Return true if ATTR is in the set represented by the NUL-terminated
+   strings in LISTBUF, which is of size LISTSIZE.  */
+
+static bool
+have_xattr (char const *attr, char const *listbuf, ssize_t listsize)
+{
+  char const *blim = listbuf + listsize;
+  for (char const *b = listbuf; b < blim; b += strlen (b) + 1)
+    for (char const *a = attr; *a == *b; a++, b++)
+      if (!*a)
+        return true;
+  return false;
+}
+
 /* Return 1 if given ACL in XDR format is non-trivial, 0 if it is trivial.
    -1 upon failure to determine it.  Possibly change errno.  Assume that
    the ACL is valid, except avoid undefined behavior even if invalid.
@@ -137,37 +151,63 @@ file_has_acl (char const *name, struct stat const *sb)
   if (! S_ISLNK (sb->st_mode))
     {
 
-# if GETXATTR_WITH_POSIX_ACLS
-
-      ssize_t ret;
+# if HAVE_LINUX_XATTR_H && HAVE_LISTXATTR
       int initial_errno = errno;
 
-      ret = getxattr (name, XATTR_NAME_POSIX_ACL_ACCESS, NULL, 0);
-      if (ret < 0 && errno == ENODATA)
-        ret = 0;
-      else if (ret > 0)
-        return 1;
-
-      if (ret == 0 && S_ISDIR (sb->st_mode))
+      /* The max length of a trivial NFSv4 ACL is 6 words for owner,
+         6 for group, 7 for everyone, all times 2 because there are
+         both allow and deny ACEs.  There are 6 words for owner
+         because of type, flag, mask, wholen, "OWNER@"+pad and
+         similarly for group; everyone is another word to hold
+         "EVERYONE@".  */
+      typedef uint32_t trivial_NFSv4_xattr_buf[2 * (6 + 6 + 7)];
+
+      /* A buffer large enough to hold any trivial NFSv4 ACL,
+         and also useful as a small array of char.  */
+      union {
+        trivial_NFSv4_xattr_buf xattr;
+        char ch[sizeof (trivial_NFSv4_xattr_buf)];
+      } stackbuf;
+
+      char *listbuf = stackbuf.ch;
+      ssize_t listbufsize = sizeof stackbuf.ch;
+      char *heapbuf = NULL;
+      ssize_t listsize;
+
+      /* Use listxattr first, as this means just one syscall in the
+         typical case where the file lacks an ACL.  Try stackbuf
+         first, falling back on malloc if stackbuf is too small.  */
+      while ((listsize = listxattr (name, listbuf, listbufsize)) < 0
+             && errno == ERANGE)
         {
-          ret = getxattr (name, XATTR_NAME_POSIX_ACL_DEFAULT, NULL, 0);
-          if (ret < 0 && errno == ENODATA)
-            ret = 0;
-          else if (ret > 0)
-            return 1;
+          free (heapbuf);
+          listbufsize = listxattr (name, NULL, 0);
+          if (listbufsize < 0)
+            return -1;
+          if (SIZE_MAX < listbufsize)
+            {
+              errno = ENOMEM;
+              return -1;
+            }
+          listbuf = heapbuf = malloc (listbufsize);
+          if (!listbuf)
+            return -1;
         }
 
-      if (ret < 0)
+      int ret
+        = (listsize < 0 ? -1
+           : (have_xattr (XATTR_NAME_POSIX_ACL_ACCESS, listbuf, listsize)
+              || (S_ISDIR (sb->st_mode)
+                  && have_xattr (XATTR_NAME_POSIX_ACL_DEFAULT,
+                                 listbuf, listsize))));
+      free (heapbuf);
+
+      /* If there is an NFSv4 ACL but no POSIX ACL, follow up with a
+         getxattr syscall to see whether the NFSv4 ACL is nontrivial.  */
+      if (ret == 0 && have_xattr (XATTR_NAME_NFSV4_ACL, listbuf, listsize))
         {
-          /* Check for NFSv4 ACLs.  The max length of a trivial
-             ACL is 6 words for owner, 6 for group, 7 for everyone,
-             all times 2 because there are both allow and deny ACEs.
-             There are 6 words for owner because of type, flag, mask,
-             wholen, "OWNER@"+pad and similarly for group; everyone is
-             another word to hold "EVERYONE@".  */
-          uint32_t xattr[2 * (6 + 6 + 7)];
-
-          ret = getxattr (name, XATTR_NAME_NFSV4_ACL, xattr, sizeof xattr);
+          ret = getxattr (name, XATTR_NAME_NFSV4_ACL,
+                          stackbuf.xattr, sizeof stackbuf.xattr);
           if (ret < 0)
             switch (errno)
               {
@@ -177,7 +217,7 @@ file_has_acl (char const *name, struct stat const *sb)
           else
             {
               /* It looks like a trivial ACL, but investigate further.  */
-              ret = acl_nfs4_nontrivial (xattr, ret);
+              ret = acl_nfs4_nontrivial (stackbuf.xattr, ret);
               if (ret < 0)
                 {
                   errno = EINVAL;
diff --git a/m4/acl.m4 b/m4/acl.m4
index c2f3e2c995..38b1dc6621 100644
--- a/m4/acl.m4
+++ b/m4/acl.m4
@@ -1,5 +1,5 @@
 # acl.m4 - check for access control list (ACL) primitives
-# serial 28
+# serial 29
 
 # Copyright (C) 2002, 2004-2023 Free Software Foundation, Inc.
 # This file is free software; the Free Software Foundation
@@ -177,37 +177,23 @@ AC_DEFUN([gl_ACL_GET_FILE],
   AS_IF([test "$gl_cv_func_working_acl_get_file" != no], [$1], [$2])
 ])
 
-# On GNU/Linux, testing if a file has an acl can be done with the getxattr
-# syscall which doesn't require linking against additional libraries.
+# On GNU/Linux, testing if a file has an acl can be done with the
+# listxattr and getxattr syscalls, which don't require linking
+# against additional libraries.  Assume this works if linux/attr.h
+# and listxattr are present.
 AC_DEFUN([gl_FILE_HAS_ACL],
 [
   AC_REQUIRE([gl_FUNC_ACL_ARG])
-  if test "$enable_acl" != no; then
-    AC_CACHE_CHECK([for getxattr with XATTR_NAME_POSIX_ACL macros],
-      [gl_cv_getxattr_with_posix_acls],
-      [gl_cv_getxattr_with_posix_acls=no
-       AC_LINK_IFELSE(
-         [AC_LANG_PROGRAM(
-            [[#include <sys/types.h>
-              #include <sys/xattr.h>
-              #include <linux/xattr.h>
-            ]],
-            [[ssize_t a = getxattr (".", XATTR_NAME_POSIX_ACL_ACCESS, 0, 0);
-              ssize_t b = getxattr (".", XATTR_NAME_POSIX_ACL_DEFAULT, 0, 0);
-              return a < 0 || b < 0;
-            ]])],
-         [gl_cv_getxattr_with_posix_acls=yes])])
-  fi
-  if test "$gl_cv_getxattr_with_posix_acls" = yes; then
-    FILE_HAS_ACL_LIB=
-    AC_DEFINE([GETXATTR_WITH_POSIX_ACLS], 1,
-      [Define to 1 if getxattr works with XATTR_NAME_POSIX_ACL_ACCESS
-       and XATTR_NAME_POSIX_ACL_DEFAULT.])
-  else
-    dnl Set gl_need_lib_has_acl to a nonempty value, so that any
-    dnl later gl_FUNC_ACL call will set FILE_HAS_ACL_LIB=$LIB_ACL.
-    gl_need_lib_has_acl=1
-    FILE_HAS_ACL_LIB=$LIB_ACL
-  fi
+  AC_CHECK_HEADERS_ONCE([linux/xattr.h])
+  AC_CHECK_FUNCS_ONCE([listxattr])
+  FILE_HAS_ACL_LIB=
+  AS_CASE([$enable_acl,$ac_cv_header_linux_xattr_h,$ac_cv_func_listxattr],
+    [no,*,*], [],
+    [*,yes,yes], [],
+    [*],
+      [dnl Set gl_need_lib_has_acl to a nonempty value, so that any
+       dnl later gl_FUNC_ACL call will set FILE_HAS_ACL_LIB=$LIB_ACL.
+       gl_need_lib_has_acl=1
+       FILE_HAS_ACL_LIB=$LIB_ACL])
   AC_SUBST([FILE_HAS_ACL_LIB])
 ])