xsetmode: new module

This is to fix a problem noted by Eric Blake.
Code was using xfreopen to change files to binary mode, but this
fails for stdout when in append mode.  Such code should use
xsetmode instead.
* NEWS: Document incompatible changes to binary-io module.
* lib/binary-io.c (__gl_setmode_check) [__DJGPP__ || __EMX__]:
New function.
* lib/binary-io.h (__gl_setmode): Rename from set_binary_mode.
(set_binary_mode): New function, which also checks for tty.
* lib/xsetmode.c, lib/xsetmode.h, modules/xsetmode: New files.

1 files changed, 16 insertions, 14 deletions

diff --git a/lib/binary-io.h b/lib/binary-io.h
index f766439e2f..ae7690de28 100644
--- a/lib/binary-io.h
+++ b/lib/binary-io.h
@@ -33,15 +33,12 @@ _GL_INLINE_HEADER_BEGIN
 # define BINARY_IO_INLINE _GL_INLINE
 #endif
 
-/* set_binary_mode (fd, mode)
-   sets the binary/text I/O mode of file descriptor fd to the given mode
-   (must be O_BINARY or O_TEXT) and returns the previous mode.  */
 #if O_BINARY
 # if defined __EMX__ || defined __DJGPP__ || defined __CYGWIN__
 #  include <io.h> /* declares setmode() */
-#  define set_binary_mode setmode
+#  define __gl_setmode setmode
 # else
-#  define set_binary_mode _setmode
+#  define __gl_setmode _setmode
 #  undef fileno
 #  define fileno _fileno
 # endif
@@ -50,7 +47,7 @@ _GL_INLINE_HEADER_BEGIN
   /* Use a function rather than a macro, to avoid gcc warnings
      "warning: statement with no effect".  */
 BINARY_IO_INLINE int
-set_binary_mode (int fd, int mode)
+__gl_setmode (int fd, int mode)
 {
   (void) fd;
   (void) mode;
@@ -58,18 +55,23 @@ set_binary_mode (int fd, int mode)
 }
 #endif
 
-/* SET_BINARY (fd);
-   changes the file descriptor fd to perform binary I/O.  */
+/* Set FD's mode to MODE.  Return the old mode if successful, -1
+   (setting errno) on failure.  */
+
 #if defined __DJGPP__ || defined __EMX__
-# include <unistd.h> /* declares isatty() */
-  /* Avoid putting stdin/stdout in binary mode if it is connected to
-     the console, because that would make it impossible for the user
-     to interrupt the program through Ctrl-C or Ctrl-Break.  */
-# define SET_BINARY(fd) ((void) (!isatty (fd) ? (set_binary_mode (fd, O_BINARY), 0) : 0))
+extern int __gl_setmode_check (int);
 #else
-# define SET_BINARY(fd) ((void) set_binary_mode (fd, O_BINARY))
+BINARY_IO_INLINE int
+__gl_setmode_check (int fd) { return 0; }
 #endif
 
+BINARY_IO_INLINE int
+set_binary_mode (int fd, int mode)
+{
+  int r = __gl_setmode_check (fd);
+  return r != 0 ? r : __gl_setmode (fd, mode);
+}
+
 _GL_INLINE_HEADER_END
 
 #endif /* _BINARY_H */