[src/vasprintf.c] Fixed overflow checking in partition_number().

In details:
  * The computation of the number of characters to be written could
    be incorrect by 1 when the thousands separator was not empty.
    However, in the 3.1 branch (without additional patches), this
    value is currently only used for overflow checking, so that this
    bug could have only very little effect (contrary to the trunk).
  * When the int and long types have the same size, the code assumed
    "wrapping behavior in two's complement". But this is actually
    undefined behavior in ISO C; in practice, a smart compiler may
    have ignored "total < 0" tests, because total is computed with
    sums of non-negative integers.
(merged changesets r11513,11515-11516 from the trunk)

git-svn-id: svn://scm.gforge.inria.fr/svn/mpfr/branches/3.1@11538 280ebfd0-de03-0410-8827-d642c229c3f4

1 files changed, 24 insertions, 24 deletions

diff --git a/src/vasprintf.c b/src/vasprintf.c
index e4c4ba612..1aebd6950 100644
--- a/src/vasprintf.c
+++ b/src/vasprintf.c
@@ -1452,7 +1452,7 @@ partition_number (struct number_parts *np, mpfr_srcptr p,
                   struct printf_spec spec)
 {
   char *str;
-  long total;
+  unsigned int total;  /* can hold the sum of two non-negative int's + 1 */
   int uppercase;
 
   /* WARNING: left justification means right space padding */
@@ -1645,43 +1645,43 @@ partition_number (struct number_parts *np, mpfr_srcptr p,
 
   /* compute the number of characters to be written verifying it is not too
      much */
+
+#define INCR_TOTAL(V)                           \
+  do {                                          \
+    MPFR_ASSERTD ((V) >= 0);                    \
+    if (MPFR_UNLIKELY ((V) > INT_MAX))          \
+      goto error;                               \
+    total += (V);                               \
+    if (MPFR_UNLIKELY (total > INT_MAX))        \
+      goto error;                               \
+  } while (0)
+
   total = np->sign ? 1 : 0;
-  total += np->prefix_size;
-  total += np->ip_size;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
-  total += np->ip_trailing_zeros;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
+  INCR_TOTAL (np->prefix_size);
+  INCR_TOTAL (np->ip_size);
+  INCR_TOTAL (np->ip_trailing_zeros);
+  MPFR_ASSERTD (np->ip_size + np->ip_trailing_zeros >= 1);
   if (np->thousands_sep)
     /* ' flag, style f and the thousands separator in current locale is not
        reduced to the null character */
-    total += (np->ip_size + np->ip_trailing_zeros) / 3;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
+    INCR_TOTAL ((np->ip_size + np->ip_trailing_zeros - 1) / 3);
   if (np->point)
     ++total;
-  total += np->fp_leading_zeros;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
-  total += np->fp_size;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
-  total += np->fp_trailing_zeros;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
-  total += np->exp_size;
-  if (MPFR_UNLIKELY (total < 0 || total > INT_MAX))
-    goto error;
+  INCR_TOTAL (np->fp_leading_zeros);
+  INCR_TOTAL (np->fp_size);
+  INCR_TOTAL (np->fp_trailing_zeros);
+  INCR_TOTAL (np->exp_size);
 
   if (spec.width > total)
     /* pad with spaces or zeros depending on np->pad_type */
     {
       np->pad_size = spec.width - total;
       total += np->pad_size; /* here total == spec.width,
-                                so 0 < total < INT_MAX */
+                                so 0 < total <= INT_MAX */
+      MPFR_ASSERTD (total == spec.width);
     }
 
+  MPFR_ASSERTD (total > 0 && total <= INT_MAX);
   return total;
 
  error: