binary-io: Attempted use of O_BINARY on consoles no longer fails.

Reported by KO Myung-Hun <komh78@gmail.com> in
<https://lists.gnu.org/archive/html/bug-gnulib/2019-05/msg00124.html>.

* lib/binary-io.h (__gl_setmode_check): Remove function.
(set_binary_mode): Declare as notinline on DJGPP and EMX.
* lib/binary-io.c (__gl_setmode_check): Remove function.
(set_binary_mode): Define here on DJGPP and EMX. Inline
__gl_setmode_check. In case of a tty, don't return an error code.

2 files changed, 14 insertions, 16 deletions

diff --git a/lib/binary-io.c b/lib/binary-io.c
index 01e0bf6476..77490e68e3 100644
--- a/lib/binary-io.c
+++ b/lib/binary-io.c
@@ -20,18 +20,20 @@
 #include "binary-io.h"
 
 #if defined __DJGPP__ || defined __EMX__
-# include <errno.h>
 # include <unistd.h>
 
 int
-__gl_setmode_check (int fd)
+set_binary_mode (int fd, int mode)
 {
   if (isatty (fd))
-    {
-      errno = EINVAL;
-      return -1;
-    }
+    /* If FD refers to a console (not a pipe, not a regular file),
+       O_TEXT is the only reasonable mode, both on input and on output.
+       Silently ignore the request.  If we were to return -1 here,
+       all programs that use xset_binary_mode would fail when run
+       with console input or console output.  */
+    return O_TEXT;
   else
-    return 0;
+    return __gl_setmode (fd, mode);
 }
+
 #endif
diff --git a/lib/binary-io.h b/lib/binary-io.h
index 720b08c755..8d4133b0f3 100644
--- a/lib/binary-io.h
+++ b/lib/binary-io.h
@@ -53,25 +53,21 @@ __gl_setmode (int fd _GL_UNUSED, int mode _GL_UNUSED)
 }
 #endif
 
-#if defined __DJGPP__ || defined __EMX__
-extern int __gl_setmode_check (int);
-#else
-BINARY_IO_INLINE int
-__gl_setmode_check (int fd _GL_UNUSED) { return 0; }
-#endif
-
 /* Set FD's mode to MODE, which should be either O_TEXT or O_BINARY.
    Return the old mode if successful, -1 (setting errno) on failure.
    Ordinarily this function would be called 'setmode', since that is
    its name on MS-Windows, but it is called 'set_binary_mode' here
    to avoid colliding with a BSD function of another name.  */
 
+#if defined __DJGPP__ || defined __EMX__
+extern int set_binary_mode (int fd, int mode);
+#else
 BINARY_IO_INLINE int
 set_binary_mode (int fd, int mode)
 {
-  int r = __gl_setmode_check (fd);
-  return r != 0 ? r : __gl_setmode (fd, mode);
+  return __gl_setmode (fd, mode);
 }
+#endif
 
 /* This macro is obsolescent.  */
 #define SET_BINARY(fd) ((void) set_binary_mode (fd, O_BINARY))