[src/vasprintf.c] Added overflow detection at the end (due to the switch

to mpfr_intmax_t for the string length in the string_buffer structure,
it was no longer detected; however, this bug was not normally visible in
the tests because here, a negative value was returned thanks to wrapping
in a type conversion and the negative value was changed to the expected
-1 value in printf.c).

git-svn-id: svn://scm.gforge.inria.fr/svn/mpfr/trunk@11531 280ebfd0-de03-0410-8827-d642c229c3f4

1 files changed, 29 insertions, 23 deletions

diff --git a/src/vasprintf.c b/src/vasprintf.c
index 51f0db6b4..a5c0c3e49 100644
--- a/src/vasprintf.c
+++ b/src/vasprintf.c
@@ -1868,7 +1868,7 @@ partition_number (struct number_parts *np, mpfr_srcptr p,
 
    return the size of the string (not counting the terminating '\0')
    return -1 if the built string is too long (i.e. has more than
-   INT_MAX characters).
+   INT_MAX or MPFR_INTMAX_MAX characters).
 
    If spec.size is 0, we only want the size of the string.
 */
@@ -1968,7 +1968,7 @@ mpfr_vasnprintf_aux (char **ptr, char *Buf, size_t size, const char *fmt,
                      va_list ap)
 {
   struct string_buffer buf;
-  size_t nbchar;
+  int nbchar;
 
   /* informations on the conversion specification filled by the parser */
   struct printf_spec spec;
@@ -2237,34 +2237,40 @@ mpfr_vasnprintf_aux (char **ptr, char *Buf, size_t size, const char *fmt,
     FLUSH (xgmp_fmt_flag, start, fmt, ap2, &buf);
 
   va_end (ap2);
-  MPFR_ASSERTD (buf.len >= 0);  /* overflow already detected */
-  nbchar = buf.len;
 
-  if (ptr != NULL)  /* implement mpfr_vasprintf */
-    {
-      MPFR_ASSERTD (nbchar == strlen (buf.start));
-      *ptr = (char *)
-        (*__gmp_reallocate_func) (buf.start, buf.size, nbchar + 1);
-    }
-  else if (size > 0)  /* implement mpfr_vsnprintf */
+  if (buf.len > INT_MAX)  /* overflow */
+    buf.len = -1;
+
+  if (buf.len != -1)
     {
-      if (nbchar < size)
+      nbchar = buf.len;
+
+      if (ptr != NULL)  /* implement mpfr_vasprintf */
         {
-          strncpy (Buf, buf.start, nbchar);
-          Buf[nbchar] = '\0';
+          MPFR_ASSERTD (nbchar == strlen (buf.start));
+          *ptr = (char *)
+            (*__gmp_reallocate_func) (buf.start, buf.size, nbchar + 1);
         }
-      else
+      else if (size > 0)  /* implement mpfr_vsnprintf */
         {
-          strncpy (Buf, buf.start, size - 1);
-          Buf[size-1] = '\0';
+          if (nbchar < size)
+            {
+              strncpy (Buf, buf.start, nbchar);
+              Buf[nbchar] = '\0';
+            }
+          else
+            {
+              strncpy (Buf, buf.start, size - 1);
+              Buf[size-1] = '\0';
+            }
+          (*__gmp_free_func) (buf.start, buf.size);
         }
-      (*__gmp_free_func) (buf.start, buf.size);
-    }
 
-  MPFR_SAVE_EXPO_FREE (expo);
-  return nbchar; /* return the number of characters that would have been
-                    written had 'size' be sufficiently large, not counting
-                    the terminating null character */
+      MPFR_SAVE_EXPO_FREE (expo);
+      return nbchar; /* return the number of characters that would have
+                        been written had 'size' be sufficiently large,
+                        not counting the terminating null character */
+    }
 
  error:
   if (buf.len == -1)  /* overflow */