From aef0b4a897aa87820399e716de349293c81ed5f1 Mon Sep 17 00:00:00 2001 From: Sergey Petrunya Date: Thu, 27 Mar 2014 14:55:29 +0400 Subject: MDEV-5962: EITS: value "position" calculated incorrectly for CHAR(n) columns - Dont substract unsigned numbers, use correct calculations. - (there is no testcase because effort is required to come up with it) --- sql/field.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'sql/field.cc') diff --git a/sql/field.cc b/sql/field.cc index 922c9aba6c5..68617d0204e 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1175,6 +1175,15 @@ inline ulonglong char_prefix_to_ulonglong(uchar *src) return uint8korr(src); } +/* + Compute res = a - b, without losing precision and taking care that these are + unsigned numbers. +*/ +static inline double safe_substract(ulonglong a, ulonglong b) +{ + return (a > b)? double(a - b) : -double(b - a); +} + /** @brief @@ -1227,10 +1236,10 @@ double Field::pos_in_interval_val_str(Field *min, Field *max, uint data_offset) minp= char_prefix_to_ulonglong(minp_prefix); maxp= char_prefix_to_ulonglong(maxp_prefix); double n, d; - n= mp - minp; + n= safe_substract(mp, minp); if (n < 0) return 0.0; - d= maxp - minp; + d= safe_substract(maxp, minp); if (d <= 0) return 1.0; return MY_MIN(n/d, 1.0); -- cgit v1.2.1