ptsname_r: Fix test failures on IRIX, Solaris.

* m4/ptsname_r.m4 (gl_PREREQ_PTSNAME_R): Test whether isatty sets
errno when it fails. Define ISATTY_FAILS_WITHOUT_SETTING_ERRNO
accordingly.
* lib/ptsname_r.c: Include <fcntl.h>.
(__ptsname_r): When isatty returned false, then on IRIX, Solaris
set errno if fd is invalid.
* tests/test-isatty.c (main): Update comments.

1 files changed, 11 insertions, 2 deletions

diff --git a/lib/ptsname_r.c b/lib/ptsname_r.c
index f2e34100c1..aa3ba38cb0 100644
--- a/lib/ptsname_r.c
+++ b/lib/ptsname_r.c
@@ -19,6 +19,7 @@
 #include <stdlib.h>
 
 #include <errno.h>
+#include <fcntl.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -59,8 +60,16 @@ __ptsname_r (int fd, char *buf, size_t buflen)
     }
 
   if (!__isatty (fd))
-    /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
-    return errno;
+    {
+#if ISATTY_FAILS_WITHOUT_SETTING_ERRNO && defined F_GETFL /* IRIX, Solaris */
+      /* Set errno.  */
+      if (fcntl (fd, F_GETFL) != -1)
+        errno = ENOTTY;
+#else
+      /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
+#endif
+      return errno;
+    }
 
   if (buflen < strlen (_PATH_TTY) + 3)
     {