cp,mv,install: improve EACCES targetdir messages

This improves on the fix for --target-directory diagnostics bugs on
Solaris 11.  Problem reported by Bruno Haible and Pádraig Brady; see:
https://lists.gnu.org/r/coreutils/2022-04/msg00044.html
Also, omit some unnecessary stat calls.
* gl/lib/targetdir.c (target_directory_operand): If !O_DIRECTORY,
do not bother calling open if stat failed with errno != EOVERFLOW.
Rename is_a_dir to try_to_open since that’s closer to what it means.
If the open failed with EACCES and we used O_SEARCH, look at stat
results to see whether errno should be ENOTDIR for better diagnostics.
Treat EOVERFLOW as an “I don’t know whether it’s a directory and
there’s no easy way to find out” rather than as an error.

1 files changed, 34 insertions, 16 deletions

diff --git a/gl/lib/targetdir.c b/gl/lib/targetdir.c
index a966e1ea1..76e67dc00 100644
--- a/gl/lib/targetdir.c
+++ b/gl/lib/targetdir.c
@@ -64,31 +64,49 @@ target_directory_operand (char const *file, struct stat *st)
     return AT_FDCWD;
 
   int fd = -1;
-  int maybe_dir = -1;
-
-  /* On old systems without O_DIRECTORY, like Solaris 10,
-     check with stat first lest we try to open a fifo for example and hang.
-     Also check on systems with O_PATHSEARCH == O_SEARCH, like Solaris 11,
-     where open() was seen to return EACCES for non executable non dirs.
-     */
-  if ((!O_DIRECTORY || (O_PATHSEARCH == O_SEARCH))
-      && stat (file, st) == 0)
+  int try_to_open = 1;
+  int stat_result;
+
+  /* On old systems without O_DIRECTORY, like Solaris 10, check with
+     stat first lest we try to open a fifo for example and hang.  */
+  if (!O_DIRECTORY)
     {
-      maybe_dir = S_ISDIR (st->st_mode);
-      if (! maybe_dir)
-        errno = ENOTDIR;
+      stat_result = stat (file, st);
+      if (stat_result == 0)
+        {
+          try_to_open = S_ISDIR (st->st_mode);
+          errno = ENOTDIR;
+        }
+      else
+        {
+          /* On EOVERFLOW failure, give up on checking, as there is no
+             easy way to check.  This should be rare.  */
+          try_to_open = errno == EOVERFLOW;
+        }
     }
 
-  if (maybe_dir)
-    fd = open (file, O_PATHSEARCH | O_DIRECTORY);
+  if (try_to_open)
+    {
+      fd = open (file, O_PATHSEARCH | O_DIRECTORY);
+
+      /* On platforms lacking O_PATH, using O_SEARCH | O_DIRECTORY to
+         open an overly-protected non-directory can fail with either
+         EACCES or ENOTDIR.  Prefer ENOTDIR as it makes for better
+         diagnostics.  */
+      if (O_PATHSEARCH == O_SEARCH && fd < 0 && errno == EACCES)
+        errno = (((O_DIRECTORY ? stat (file, st) : stat_result) == 0
+                  && !S_ISDIR (st->st_mode))
+                 ? ENOTDIR : EACCES);
+    }
 
   if (!O_DIRECTORY && 0 <= fd)
     {
       /* On old systems like Solaris 10 double check type,
          to ensure we've opened a directory.  */
       int err;
-      if (fstat (fd, st) != 0 ? (err = errno, true)
-          : !S_ISDIR (st->st_mode) && (err = ENOTDIR, true))
+      if (fstat (fd, st) == 0
+          ? !S_ISDIR (st->st_mode) && (err = ENOTDIR, true)
+          : (err = errno) != EOVERFLOW)
         {
           close (fd);
           errno = err;