From 89b866e7bfc8965046626844ee22ecd5b4e5a774 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 25 Apr 2008 00:39:37 +0400 Subject: Bug#36023: Incorrect handling of zero length caused an assertion to fail. When a zero length is provided to the my_decimal_length_to_precision function along with unsigned_flag set to false it returns a negative value. For queries that employs temporary tables may cause failed assertion or excessive memory consumption while temporary table creation. Now the my_decimal_length_to_precision and the my_decimal_precision_to_length functions take unsigned_flag into account only if the length/precision argument is non-zero. mysql-test/t/type_decimal.test: Added a test case for the bug#36023: Incorrect handling of zero length caused an assertion to fail. mysql-test/r/type_decimal.result: Added a test case for the bug#36023: Incorrect handling of zero length caused an assertion to fail. sql/my_decimal.h: Bug#36023: Incorrect handling of zero length caused an assertion to fail. Now the my_decimal_length_to_precision and the my_decimal_precision_to_length functions take unsigned_flag into account only if the length/precision argument is non-zero. --- sql/my_decimal.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'sql/my_decimal.h') diff --git a/sql/my_decimal.h b/sql/my_decimal.h index c661579ea66..6a0d05921ec 100644 --- a/sql/my_decimal.h +++ b/sql/my_decimal.h @@ -164,14 +164,23 @@ inline int check_result_and_overflow(uint mask, int result, my_decimal *val) inline uint my_decimal_length_to_precision(uint length, uint scale, bool unsigned_flag) { - return (uint) (length - (scale>0 ? 1:0) - (unsigned_flag ? 0:1)); + /* Precision can't be negative thus ignore unsigned_flag when length is 0. */ + DBUG_ASSERT(length || !scale); + return (uint) (length - (scale>0 ? 1:0) - + (unsigned_flag || !length ? 0:1)); } inline uint32 my_decimal_precision_to_length(uint precision, uint8 scale, bool unsigned_flag) { + /* + When precision is 0 it means that original length was also 0. Thus + unsigned_flag is ignored in this case. + */ + DBUG_ASSERT(precision || !scale); set_if_smaller(precision, DECIMAL_MAX_PRECISION); - return (uint32)(precision + (scale>0 ? 1:0) + (unsigned_flag ? 0:1)); + return (uint32)(precision + (scale>0 ? 1:0) + + (unsigned_flag || !precision ? 0:1)); } inline -- cgit v1.2.1