file-has-acl: port to Fedora 39

Fedora 39 getxattr with XATTR_NAME_POSIX_ACL_ACCESS either
succeeds or fails with ENODATA, so it is no longer possible to
detect from its failure that the filesystem might support NFSv4 ACLs.
Problem reported by Ondrej Valousek in:
https://lists.gnu.org/r/bug-gnulib/2023-04/msg00228.html
Instead, use listxattr to determine whether NFSv4 ACLs are in play.
This typically saves syscalls anyway.
* lib/file-has-acl.c: In #if, use (HAVE_LINUX_XATTR_H &&
HAVE_LISTXATTR) instead of GETXATTR_WITH_POSIX_ACLS.
The following changes apply when (USE_ACL && HAVE_LINUX_XATTR_H &&
HAVE_LISTXATTR):
Include minmax.h.
(have_xattr): New function.
(file_has_acl): Try listxattr first; typically this means we need
to do no other syscall.  Call getxattr only if there are NFSv4
ACLs but not POSIX ACLs.
* m4/acl.m4 (gl_FILE_HAS_ACL): Simplify by merely testing for
linux/xattr.h and listxattr.  All uses changed.

1 files changed, 67 insertions, 27 deletions

diff --git a/lib/file-has-acl.c b/lib/file-has-acl.c
index b31a2ea252..1edcd2cbd6 100644
--- a/lib/file-has-acl.c
+++ b/lib/file-has-acl.c
@@ -29,7 +29,7 @@
 
 #include "acl-internal.h"
 
-#if USE_ACL && GETXATTR_WITH_POSIX_ACLS
+#if USE_ACL && HAVE_LINUX_XATTR_H && HAVE_LISTXATTR
 # include <string.h>
 # include <arpa/inet.h>
 # include <sys/xattr.h>
@@ -44,6 +44,20 @@ enum {
   ACE4_IDENTIFIER_GROUP        = 0x00000040
 };
 
+/* Return true if ATTR is in the set represented by the NUL-terminated
+   strings in LISTBUF, which is of size LISTSIZE.  */
+
+static bool
+have_xattr (char const *attr, char const *listbuf, ssize_t listsize)
+{
+  char const *blim = listbuf + listsize;
+  for (char const *b = listbuf; b < blim; b += strlen (b) + 1)
+    for (char const *a = attr; *a == *b; a++, b++)
+      if (!*a)
+        return true;
+  return false;
+}
+
 /* Return 1 if given ACL in XDR format is non-trivial, 0 if it is trivial.
    -1 upon failure to determine it.  Possibly change errno.  Assume that
    the ACL is valid, except avoid undefined behavior even if invalid.
@@ -137,37 +151,63 @@ file_has_acl (char const *name, struct stat const *sb)
   if (! S_ISLNK (sb->st_mode))
     {
 
-# if GETXATTR_WITH_POSIX_ACLS
-
-      ssize_t ret;
+# if HAVE_LINUX_XATTR_H && HAVE_LISTXATTR
       int initial_errno = errno;
 
-      ret = getxattr (name, XATTR_NAME_POSIX_ACL_ACCESS, NULL, 0);
-      if (ret < 0 && errno == ENODATA)
-        ret = 0;
-      else if (ret > 0)
-        return 1;
-
-      if (ret == 0 && S_ISDIR (sb->st_mode))
+      /* The max length of a trivial NFSv4 ACL is 6 words for owner,
+         6 for group, 7 for everyone, all times 2 because there are
+         both allow and deny ACEs.  There are 6 words for owner
+         because of type, flag, mask, wholen, "OWNER@"+pad and
+         similarly for group; everyone is another word to hold
+         "EVERYONE@".  */
+      typedef uint32_t trivial_NFSv4_xattr_buf[2 * (6 + 6 + 7)];
+
+      /* A buffer large enough to hold any trivial NFSv4 ACL,
+         and also useful as a small array of char.  */
+      union {
+        trivial_NFSv4_xattr_buf xattr;
+        char ch[sizeof (trivial_NFSv4_xattr_buf)];
+      } stackbuf;
+
+      char *listbuf = stackbuf.ch;
+      ssize_t listbufsize = sizeof stackbuf.ch;
+      char *heapbuf = NULL;
+      ssize_t listsize;
+
+      /* Use listxattr first, as this means just one syscall in the
+         typical case where the file lacks an ACL.  Try stackbuf
+         first, falling back on malloc if stackbuf is too small.  */
+      while ((listsize = listxattr (name, listbuf, listbufsize)) < 0
+             && errno == ERANGE)
         {
-          ret = getxattr (name, XATTR_NAME_POSIX_ACL_DEFAULT, NULL, 0);
-          if (ret < 0 && errno == ENODATA)
-            ret = 0;
-          else if (ret > 0)
-            return 1;
+          free (heapbuf);
+          listbufsize = listxattr (name, NULL, 0);
+          if (listbufsize < 0)
+            return -1;
+          if (SIZE_MAX < listbufsize)
+            {
+              errno = ENOMEM;
+              return -1;
+            }
+          listbuf = heapbuf = malloc (listbufsize);
+          if (!listbuf)
+            return -1;
         }
 
-      if (ret < 0)
+      int ret
+        = (listsize < 0 ? -1
+           : (have_xattr (XATTR_NAME_POSIX_ACL_ACCESS, listbuf, listsize)
+              || (S_ISDIR (sb->st_mode)
+                  && have_xattr (XATTR_NAME_POSIX_ACL_DEFAULT,
+                                 listbuf, listsize))));
+      free (heapbuf);
+
+      /* If there is an NFSv4 ACL but no POSIX ACL, follow up with a
+         getxattr syscall to see whether the NFSv4 ACL is nontrivial.  */
+      if (ret == 0 && have_xattr (XATTR_NAME_NFSV4_ACL, listbuf, listsize))
         {
-          /* Check for NFSv4 ACLs.  The max length of a trivial
-             ACL is 6 words for owner, 6 for group, 7 for everyone,
-             all times 2 because there are both allow and deny ACEs.
-             There are 6 words for owner because of type, flag, mask,
-             wholen, "OWNER@"+pad and similarly for group; everyone is
-             another word to hold "EVERYONE@".  */
-          uint32_t xattr[2 * (6 + 6 + 7)];
-
-          ret = getxattr (name, XATTR_NAME_NFSV4_ACL, xattr, sizeof xattr);
+          ret = getxattr (name, XATTR_NAME_NFSV4_ACL,
+                          stackbuf.xattr, sizeof stackbuf.xattr);
           if (ret < 0)
             switch (errno)
               {
@@ -177,7 +217,7 @@ file_has_acl (char const *name, struct stat const *sb)
           else
             {
               /* It looks like a trivial ACL, but investigate further.  */
-              ret = acl_nfs4_nontrivial (xattr, ret);
+              ret = acl_nfs4_nontrivial (stackbuf.xattr, ret);
               if (ret < 0)
                 {
                   errno = EINVAL;