MDEV-13240 Wrong warning with MAX(datetime_field) OVER (...)

The problem resided in Item_window_func implementation,
and it was revealed by bb-10.2-ext specific changes:

Item_window_func::save_in_field() works differently in bb-10.2-ext vs 10.2:

- 10.2 goes through val_str()
- bb-10.2-ext goes through get_date(), due to Type_handler related changes.
  get_date() tries to convert empty string to DATETIME, hence the warning.

During a discussion with Vicentiu, it was decided to fix
Item_window_func::val_xxx() to return NULL
(instead of an "empty" value, such as 0 for numbers and '' for strings)
when force_return_blank is set.

5 files changed, 45 insertions, 11 deletions

diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result
index ec83998aa98..3aece252991 100644
--- a/mysql-test/r/win.result
+++ b/mysql-test/r/win.result
@@ -3085,3 +3085,15 @@ select max(id), rank() over (order by max(id)) from t1 where id < 3;
 max(id)	rank() over (order by max(id))
 2	1
 drop table t1;
+#
+# Start of 10.3 tests
+#
+#
+# MDEV-13240 Wrong warning with MAX(datetime_field) OVER (...)
+#
+CREATE TABLE t1 (dt DATETIME);
+INSERT INTO t1 VALUES ('2017-05-17');
+SELECT MAX(dt) OVER (ORDER BY dt ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) FROM t1;
+MAX(dt) OVER (ORDER BY dt ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING)
+NULL
+DROP TABLE t1;
diff --git a/mysql-test/r/win_insert_select.result b/mysql-test/r/win_insert_select.result
index c86576df6ae..5eddbf78713 100644
--- a/mysql-test/r/win_insert_select.result
+++ b/mysql-test/r/win_insert_select.result
@@ -1,7 +1,7 @@
 CREATE TABLE t1 (c1 INT, c2 VARCHAR(30));
 PREPARE populate_table FROM "INSERT into t1 values (1, 'manual_insert_1'),
                                                    (4, 'manual_insert_2')";
-INSERT INTO t1 SELECT row_number() over(), "should_have_0" FROM t1;
+INSERT INTO t1 SELECT row_number() over(), "should_have_NULL" FROM t1;
 INSERT INTO t1 SELECT 1 + row_number() over(), "should_have_2" FROM t1;
 EXECUTE populate_table;
 INSERT INTO t1 SELECT 10 + row_number() over(), "should repeat 4 times [11-14]" FROM t1;
@@ -13,8 +13,8 @@ c1	c2
 12	should repeat 4 times [11-14]
 13	should repeat 4 times [11-14]
 14	should repeat 4 times [11-14]
-0	should_have_0
 2	should_have_2
+NULL	should_have_NULL
 DELETE FROM t1;
 EXECUTE populate_table;
 INSERT INTO t1
diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test
index 95d32c5bd14..dfcf00cabf9 100644
--- a/mysql-test/t/win.test
+++ b/mysql-test/t/win.test
@@ -1877,3 +1877,16 @@ select count(max(id)) over (order by max(id)) from t1 where id < 3;
 select max(id), rank() over (order by max(id)) from t1 where id < 3;
 
 drop table t1;
+
+--echo #
+--echo # Start of 10.3 tests
+--echo #
+
+--echo #
+--echo # MDEV-13240 Wrong warning with MAX(datetime_field) OVER (...)
+--echo #
+
+CREATE TABLE t1 (dt DATETIME);
+INSERT INTO t1 VALUES ('2017-05-17');
+SELECT MAX(dt) OVER (ORDER BY dt ROWS BETWEEN 1 FOLLOWING AND 1 FOLLOWING) FROM t1;
+DROP TABLE t1;
diff --git a/mysql-test/t/win_insert_select.test b/mysql-test/t/win_insert_select.test
index 66df7324c4f..a9e7e8f322f 100644
--- a/mysql-test/t/win_insert_select.test
+++ b/mysql-test/t/win_insert_select.test
@@ -3,7 +3,7 @@ CREATE TABLE t1 (c1 INT, c2 VARCHAR(30));
 PREPARE populate_table FROM "INSERT into t1 values (1, 'manual_insert_1'),
                                                    (4, 'manual_insert_2')";
 
-INSERT INTO t1 SELECT row_number() over(), "should_have_0" FROM t1;
+INSERT INTO t1 SELECT row_number() over(), "should_have_NULL" FROM t1;
 INSERT INTO t1 SELECT 1 + row_number() over(), "should_have_2" FROM t1;
 
 EXECUTE populate_table;
diff --git a/sql/item_windowfunc.h b/sql/item_windowfunc.h
index 77cbd556e60..9fe95ed6cee 100644
--- a/sql/item_windowfunc.h
+++ b/sql/item_windowfunc.h
@@ -839,13 +839,24 @@ public:
     read_value_from_result_field= true;
   }
 
+  bool is_null()
+  {
+    if (force_return_blank)
+      return true;
+
+    if (read_value_from_result_field)
+      return result_field->is_null();
+
+    return window_func()->is_null();
+  }
+
   double val_real() 
   {
     double res;
     if (force_return_blank)
     {
       res= 0.0;
-      null_value= false;
+      null_value= true;
     }
     else if (read_value_from_result_field)
     {
@@ -866,7 +877,7 @@ public:
     if (force_return_blank)
     {
       res= 0;
-      null_value= false;
+      null_value= true;
     }
     else if (read_value_from_result_field)
     {
@@ -886,9 +897,8 @@ public:
     String *res;
     if (force_return_blank)
     {
-      null_value= false;
-      str->length(0);
-      res= str;
+      null_value= true;
+      res= NULL;
     }
     else if (read_value_from_result_field)
     {
@@ -910,9 +920,8 @@ public:
     my_decimal *res;
     if (force_return_blank)
     {
-      my_decimal_set_zero(dec);
-      null_value= false;
-      res= dec;
+      null_value= true;
+      res= NULL;
     }
     else if (read_value_from_result_field)
     {