Kill a use of %n format specifier

This format has been used as a security exploit vector for decades
now.  Some operating systems (OpenBSD, Android, MSVC). It is targeted
for removal in C2X standard:
http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2834.htm

This requires extending the debug message function to return the
number of bytes written (like printf(3)), to permit %n format
specifier in one in one invocation of statsPrintf() in
report_summary().

Implemented by Matthias Kilian (kili<AT>outback.escape.de)

1 files changed, 7 insertions, 6 deletions

diff --git a/rts/RtsMessages.c b/rts/RtsMessages.c
index 9b165faea7..33be410001 100644
--- a/rts/RtsMessages.c
+++ b/rts/RtsMessages.c
@@ -36,7 +36,7 @@
 
 // Default to the stdio implementation of these hooks.
 RtsMsgFunction *fatalInternalErrorFn = rtsFatalInternalErrorFn;
-RtsMsgFunction *debugMsgFn           = rtsDebugMsgFn;
+RtsMsgFunctionRetLen *debugMsgFn           = rtsDebugMsgFn;
 RtsMsgFunction *errorMsgFn           = rtsErrorMsgFn;
 RtsMsgFunction *sysErrorMsgFn        = rtsSysErrorMsgFn;
 
@@ -102,10 +102,10 @@ debugBelch(const char*s, ...)
   va_end(ap);
 }
 
-void
+int
 vdebugBelch(const char*s, va_list ap)
 {
-  (*debugMsgFn)(s,ap);
+  return (*debugMsgFn)(s,ap);
 }
 
 /* -----------------------------------------------------------------------------
@@ -285,16 +285,16 @@ rtsSysErrorMsgFn(const char *s, va_list ap)
 #endif
 }
 
-void
+int
 rtsDebugMsgFn(const char *s, va_list ap)
 {
+  int r;
 #if defined(mingw32_HOST_OS)
   /* Ensure we're in text mode so newlines get encoded properly.  */
   int mode = _setmode (_fileno(stderr), _O_TEXT);
   if (isGUIApp())
   {
      char buf[BUFSIZE];
-         int r;
 
          r = vsnprintf(buf, BUFSIZE, s, ap);
          if (r > 0 && r < BUFSIZE) {
@@ -305,12 +305,13 @@ rtsDebugMsgFn(const char *s, va_list ap)
 #endif
   {
      /* don't fflush(stdout); WORKAROUND bug in Linux glibc */
-     vfprintf(stderr, s, ap);
+     r = vfprintf(stderr, s, ap);
      fflush(stderr);
   }
 #if defined(mingw32_HOST_OS)
   _setmode (_fileno(stderr), mode);
 #endif
+  return r;
 }