BUG: dragon4 fractional output mode adds too many trailing zeros

Fixes #10713

2 files changed, 38 insertions, 27 deletions

diff --git a/numpy/core/src/multiarray/dragon4.c b/numpy/core/src/multiarray/dragon4.c
index 8606adf99..e005234a0 100644
--- a/numpy/core/src/multiarray/dragon4.c
+++ b/numpy/core/src/multiarray/dragon4.c
@@ -896,7 +896,7 @@ BigInt_ShiftLeft(BigInt *result, npy_uint32 shift)
     if (shiftBits == 0) {
         npy_uint32 i;
 
-        /* copy blcoks from high to low */
+        /* copy blocks from high to low */
         for (pInCur = result->blocks + result->length,
                  pOutCur = pInCur + shiftBlocks;
                  pInCur >= pInBlocks;
@@ -1002,7 +1002,7 @@ BigInt_ShiftLeft(BigInt *result, npy_uint32 shift)
  *   * exponent - value exponent in base 2
  *   * mantissaBit - index of the highest set mantissa bit
  *   * hasUnequalMargins - is the high margin twice as large as the low margin
- *   * cutoffMode - how to intepret cutoffNumber: fractional or total digits?
+ *   * cutoffMode - how to interpret cutoffNumber: fractional or total digits?
  *   * cutoffNumber - cut off printing after this many digits. -1 for no cutoff
  *   * pOutBuffer - buffer to output into
  *   * bufferSize - maximum characters that can be printed to pOutBuffer
@@ -1381,7 +1381,7 @@ Dragon4(const npy_uint64 mantissa, const npy_int32 exponent,
 
         /*
          * if we are directly in the middle, round towards the even digit (i.e.
-         * IEEE rouding rules)
+         * IEEE rounding rules)
          */
         if (compare == 0) {
             roundDown = (outputDigit & 1) == 0;
@@ -1588,12 +1588,12 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
                  npy_int32 digits_right)
 {
     npy_int32 printExponent;
-    npy_int32 numDigits, numWholeDigits, has_sign=0;
+    npy_int32 numDigits, numWholeDigits=0, has_sign=0;
 
-    npy_int32 maxPrintLen = bufferSize - 1, pos = 0;
+    npy_int32 maxPrintLen = (npy_int32)bufferSize - 1, pos = 0;
 
     /* track the # of digits past the decimal point that have been printed */
-    npy_int32 numFractionDigits = 0;
+    npy_int32 numFractionDigits = 0, desiredFractionalDigits;
 
     DEBUG_ASSERT(bufferSize > 0);
 
@@ -1637,11 +1637,11 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
             }
         }
         /* insert the decimal point prior to the fraction */
-        else if (numDigits > (npy_uint32)numWholeDigits) {
-            npy_uint32 maxFractionDigits;
+        else if (numDigits > numWholeDigits) {
+            npy_int32 maxFractionDigits;
 
             numFractionDigits = numDigits - numWholeDigits;
-            maxFractionDigits = maxPrintLen - numWholeDigits -1-pos;
+            maxFractionDigits = maxPrintLen - numWholeDigits - 1 - pos;
             if (numFractionDigits > maxFractionDigits) {
                 numFractionDigits = maxFractionDigits;
             }
@@ -1656,19 +1656,20 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
     }
     else {
         /* shift out the fraction to make room for the leading zeros */
-        npy_uint32 numFractionZeros = 0;
+        npy_int32 numFractionZeros = 0;
         if (pos + 2 < maxPrintLen) {
-            npy_uint32 maxFractionZeros, digitsStartIdx, maxFractionDigits, i;
+            npy_int32 maxFractionZeros, digitsStartIdx, maxFractionDigits, i;
 
             maxFractionZeros = maxPrintLen - 2 - pos;
-            numFractionZeros = (npy_uint32)-printExponent - 1;
+            numFractionZeros = -(printExponent + 1);
             if (numFractionZeros > maxFractionZeros) {
                 numFractionZeros = maxFractionZeros;
             }
 
             digitsStartIdx = 2 + numFractionZeros;
 
-            /* shift the significant digits right such that there is room for
+            /*
+             * shift the significant digits right such that there is room for
              * leading zeros
              */
             numFractionDigits = numDigits;
@@ -1710,6 +1711,11 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
         buffer[pos++] = '.';
     }
 
+    desiredFractionalDigits = precision;
+    if (cutoff_mode == CutoffMode_TotalLength && precision >= 0) {
+        desiredFractionalDigits = precision - numWholeDigits;
+    }
+
     if (trim_mode == TrimMode_LeaveOneZero) {
         /* if we didn't print any fractional digits, add a trailing 0 */
         if (numFractionDigits == 0 && pos < maxPrintLen) {
@@ -1718,11 +1724,12 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
         }
     }
     else if (trim_mode == TrimMode_None &&
-            digit_mode != DigitMode_Unique &&
-            precision > (npy_int32)numFractionDigits && pos < maxPrintLen) {
+             digit_mode != DigitMode_Unique &&
+             desiredFractionalDigits > numFractionDigits &&
+             pos < maxPrintLen) {
         /* add trailing zeros up to precision length */
         /* compute the number of trailing zeros needed */
-        npy_uint32 count = precision - numFractionDigits;
+        npy_int32 count = desiredFractionalDigits - numFractionDigits;
         if (pos + count > maxPrintLen) {
             count = maxPrintLen - pos;
         }
@@ -1751,7 +1758,7 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
 
     /* add any whitespace padding to right side */
     if (digits_right >= numFractionDigits) {
-        npy_uint32 count = digits_right - numFractionDigits;
+        npy_int32 count = digits_right - numFractionDigits;
 
         /* in trim_mode DptZeros, if right padding, add a space for the . */
         if (trim_mode == TrimMode_DptZeros && numFractionDigits == 0
@@ -1769,8 +1776,8 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
     }
     /* add any whitespace padding to left side */
     if (digits_left > numWholeDigits + has_sign) {
-        npy_uint32 shift = digits_left - (numWholeDigits + has_sign);
-        npy_uint32 count = pos;
+        npy_int32 shift = digits_left - (numWholeDigits + has_sign);
+        npy_int32 count = pos;
 
         if (count + shift > maxPrintLen){
             count = maxPrintLen - shift;
@@ -1781,7 +1788,7 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
         }
         pos = shift + count;
         for ( ; shift > 0; shift--) {
-            buffer[shift-1] = ' ';
+            buffer[shift - 1] = ' ';
         }
     }
 
@@ -1871,7 +1878,8 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
     /* insert the decimal point prior to the fractional number */
     numFractionDigits = numDigits-1;
     if (numFractionDigits > 0 && bufferSize > 1) {
-        npy_uint32 maxFractionDigits = bufferSize-2;
+        npy_int32 maxFractionDigits = (npy_int32)bufferSize - 2;
+
         if (numFractionDigits > maxFractionDigits) {
             numFractionDigits =  maxFractionDigits;
         }
@@ -1905,9 +1913,10 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
         if (precision > (npy_int32)numFractionDigits) {
             char *pEnd;
             /* compute the number of trailing zeros needed */
-            npy_uint32 numZeros = (precision - numFractionDigits);
-            if (numZeros > bufferSize-1) {
-                numZeros = bufferSize-1;
+            npy_int32 numZeros = (precision - numFractionDigits);
+
+            if (numZeros > (npy_int32)bufferSize - 1) {
+                numZeros = (npy_int32)bufferSize - 1;
             }
 
             for (pEnd = pCurOut + numZeros; pCurOut < pEnd; ++pCurOut) {
@@ -1941,7 +1950,7 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
     /* print the exponent into a local buffer and copy into output buffer */
     if (bufferSize > 1) {
         char exponentBuffer[7];
-        npy_uint32 digits[5];
+        npy_int32 digits[5];
         npy_int32 i, exp_size, count;
 
         if (exp_digits > 5) {
@@ -1978,8 +1987,8 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, npy_uint64 mantissa,
 
         /* copy the exponent buffer into the output */
         count = exp_size + 2;
-        if (count > bufferSize-1) {
-            count = bufferSize-1;
+        if (count > (npy_int32)bufferSize - 1) {
+            count = (npy_int32)bufferSize - 1;
         }
         memcpy(pCurOut, exponentBuffer, count);
         pCurOut += count;
diff --git a/numpy/core/tests/test_scalarprint.py b/numpy/core/tests/test_scalarprint.py
index 164ab06c7..d57f1a890 100644
--- a/numpy/core/tests/test_scalarprint.py
+++ b/numpy/core/tests/test_scalarprint.py
@@ -152,6 +152,8 @@ class TestRealScalars(object):
         assert_equal(fpos64('1.5', unique=False, precision=3), "1.500")
         assert_equal(fsci32('1.5', unique=False, precision=3), "1.500e+00")
         assert_equal(fsci64('1.5', unique=False, precision=3), "1.500e+00")
+        # gh-10713
+        assert_equal(fpos64('324', unique=False, precision=5, fractional=False), "324.00")
 
     def test_dragon4_interface(self):
         tps = [np.float16, np.float32, np.float64]