openat: fix openat bugs on Solaris 9

openat(fd,"file/",O_RDONLY) mistakenly succeeded.

* lib/openat.c (rpl_openat): Work around Solaris 9 bug.
* m4/openat.m4 (gl_FUNC_OPENAT): Also replace openat on Solaris.
* modules/openat (Depends-on): Add open.
* m4/fcntl_h.m4 (gl_FCNTL_H_DEFAULTS): Provide new default.
* modules/fcntl-h (Makefile.am): Substitute it.
* lib/fcntl.in.h (openat): Declare replacement.
* doc/posix-functions/openat.texi (openat): Document this.

Signed-off-by: Eric Blake <ebb9@byu.net>

2 files changed, 100 insertions, 1 deletions

diff --git a/lib/fcntl.in.h b/lib/fcntl.in.h
index cadb6a1572..0ae8213e47 100644
--- a/lib/fcntl.in.h
+++ b/lib/fcntl.in.h
@@ -62,9 +62,11 @@ extern int open (const char *filename, int flags, ...);
 #endif
 
 #if @GNULIB_OPENAT@
-# if !@HAVE_OPENAT@
+# if @REPLACE_OPENAT@
 #  undef openat
 #  define openat rpl_openat
+# endif
+# if !@HAVE_OPENAT@ || @REPLACE_OPENAT@
 int openat (int fd, char const *file, int flags, /* mode_t mode */ ...);
 # endif
 #elif defined GNULIB_POSIXCHECK
diff --git a/lib/openat.c b/lib/openat.c
index 2a194e8856..7e46a2672b 100644
--- a/lib/openat.c
+++ b/lib/openat.c
@@ -22,12 +22,107 @@
 
 #include <stdarg.h>
 #include <stddef.h>
+#include <string.h>
 #include <sys/stat.h>
 
 #include "dirname.h" /* solely for definition of IS_ABSOLUTE_FILE_NAME */
 #include "openat-priv.h"
 #include "save-cwd.h"
 
+#if HAVE_OPENAT
+
+# undef openat
+
+/* Like openat, but work around Solaris 9 bugs with trailing slash.  */
+int
+rpl_openat (int dfd, char const *filename, int flags, ...)
+{
+  mode_t mode;
+  int fd;
+
+  mode = 0;
+  if (flags & O_CREAT)
+    {
+      va_list arg;
+      va_start (arg, flags);
+
+      /* We have to use PROMOTED_MODE_T instead of mode_t, otherwise GCC 4
+	 creates crashing code when 'mode_t' is smaller than 'int'.  */
+      mode = va_arg (arg, PROMOTED_MODE_T);
+
+      va_end (arg);
+    }
+
+#if OPEN_TRAILING_SLASH_BUG
+  /* If the filename ends in a slash and one of O_CREAT, O_WRONLY, O_RDWR
+     is specified, then fail.
+     Rationale: POSIX <http://www.opengroup.org/susv3/basedefs/xbd_chap04.html>
+     says that
+       "A pathname that contains at least one non-slash character and that
+        ends with one or more trailing slashes shall be resolved as if a
+        single dot character ( '.' ) were appended to the pathname."
+     and
+       "The special filename dot shall refer to the directory specified by
+        its predecessor."
+     If the named file already exists as a directory, then
+       - if O_CREAT is specified, open() must fail because of the semantics
+         of O_CREAT,
+       - if O_WRONLY or O_RDWR is specified, open() must fail because POSIX
+         <http://www.opengroup.org/susv3/functions/open.html> says that it
+         fails with errno = EISDIR in this case.
+     If the named file does not exist or does not name a directory, then
+       - if O_CREAT is specified, open() must fail since open() cannot create
+         directories,
+       - if O_WRONLY or O_RDWR is specified, open() must fail because the
+         file does not contain a '.' directory.  */
+  if (flags & (O_CREAT | O_WRONLY | O_RDWR))
+    {
+      size_t len = strlen (filename);
+      if (len > 0 && filename[len - 1] == '/')
+	{
+	  errno = EISDIR;
+	  return -1;
+	}
+    }
+#endif
+
+  fd = openat (dfd, filename, flags, mode);
+
+#if OPEN_TRAILING_SLASH_BUG
+  /* If the filename ends in a slash and fd does not refer to a directory,
+     then fail.
+     Rationale: POSIX <http://www.opengroup.org/susv3/basedefs/xbd_chap04.html>
+     says that
+       "A pathname that contains at least one non-slash character and that
+        ends with one or more trailing slashes shall be resolved as if a
+        single dot character ( '.' ) were appended to the pathname."
+     and
+       "The special filename dot shall refer to the directory specified by
+        its predecessor."
+     If the named file without the slash is not a directory, open() must fail
+     with ENOTDIR.  */
+  if (fd >= 0)
+    {
+      size_t len = strlen (filename);
+      if (len > 0 && filename[len - 1] == '/')
+	{
+	  struct stat statbuf;
+
+	  if (fstat (fd, &statbuf) >= 0 && !S_ISDIR (statbuf.st_mode))
+	    {
+	      close (fd);
+	      errno = ENOTDIR;
+	      return -1;
+	    }
+	}
+    }
+#endif
+
+  return fd;
+}
+
+#else /* !HAVE_OPENAT */
+
 /* Replacement for Solaris' openat function.
    <http://www.google.com/search?q=openat+site:docs.sun.com>
    First, try to simulate it via open ("/proc/self/fd/FD/FILE").
@@ -156,3 +251,5 @@ openat_needs_fchdir (void)
 
   return needs_fchdir;
 }
+
+#endif /* !HAVE_OPENAT */