Fix for Bug #37007 Maria: different checksum for MyISAM table depending on CHECKSUM=0|1

This also adds a check that MyISAM tables with incompatible checksums are detected by CHECK TABLE ... [FOR UPGRADE] and thus also by mysql_upgrade.
The tables that are incomatible are MyISAM tables with ROW_FORMAT=fixed and has VARCHAR fields and have CHECKSUM enabled.
Before these tables gave different checksum if you used CHECK TABLE with or without EXTENDED

mysql-test/r/old-mode.result:
  Now we get same results with and without EXTENDED
mysql-test/r/row-checksum-old.result:
  Initial results
mysql-test/r/row-checksum.result:
  Initial results
mysql-test/t/old-mode.test:
  Added test with QUICK to show that the live checksum is not used when running with --old
mysql-test/t/row-checksum-old-master.opt:
  Start mysqld with --old mode to enable old checksum code
mysql-test/t/row-checksum-old.test:
  Run row-checksum test under mysqld --old
mysql-test/t/row-checksum.test:
  Verify that checksum are calculated the same way with and without EXTENDED
  We run this with several storage engines to ensure results are the same over storage engines
sql/ha_partition.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/handler.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/handler.h:
  Split HA_HAS_CHECKSUM into HA_HAS_NEW_CHECKSUM and HA_HAS_OLD_CHECKSUM flags.
  This is a safe API change as only MyISAM and Maria should use these handler flags.
sql/sql_show.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/sql_table.cc:
  Use file->checksum() for live checksums if the life checksum method corresponds to the mysqld --old flag
storage/maria/ha_maria.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
storage/myisam/ha_myisam.cc:
  Set HA_HAS_OLD_CHECKSUM and/or HA_HAS_NEW_CHECKSUM flags depending on if this is a new myisam or old myisam file
  Add method check_for_upgrade() to detect if the table is of old version with a checksum that is incompatible with CHECK TABLE ... EXTENDED
storage/myisam/ha_myisam.h:
  Added check_for_upgrade()
storage/myisam/mi_open.c:
  Removed not neede cast
  Initialize share->has_null_fields and share->has_varchar_fields variables
storage/myisam/myisamdef.h:
  Added share->has_null_fields and share->has_varchar_fields

17 files changed, 306 insertions, 13 deletions

diff --git a/mysql-test/r/old-mode.result b/mysql-test/r/old-mode.result
index df2c0b6fee0..a9815d7dab2 100644
--- a/mysql-test/r/old-mode.result
+++ b/mysql-test/r/old-mode.result
@@ -5,8 +5,12 @@ insert t1 values (1, "aaa", "bbb"), (NULL, "", "ccccc"), (0, NULL, "");
 insert t2 select * from t1;
 checksum table t1, t2;
 Table	Checksum
-test.t1	3442722830
+test.t1	2948697075
 test.t2	2948697075
+checksum table t1, t2 quick;
+Table	Checksum
+test.t1	NULL
+test.t2	NULL
 checksum table t1, t2 extended;
 Table	Checksum
 test.t1	2948697075
diff --git a/mysql-test/r/row-checksum-old.result b/mysql-test/r/row-checksum-old.result
new file mode 100644
index 00000000000..3cf5a7104b9
--- /dev/null
+++ b/mysql-test/r/row-checksum-old.result
@@ -0,0 +1,85 @@
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	4108368782
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	4108368782
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	4108368782
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	4108368782
+drop table t1;
diff --git a/mysql-test/r/row-checksum.result b/mysql-test/r/row-checksum.result
new file mode 100644
index 00000000000..31ae094859b
--- /dev/null
+++ b/mysql-test/r/row-checksum.result
@@ -0,0 +1,85 @@
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	229851577
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	229851577
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 quick;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 extended;
+Table	Checksum
+test.t1	3885665021
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	3885665021
+drop table t1;
diff --git a/mysql-test/t/old-mode.test b/mysql-test/t/old-mode.test
index 4fa21f761ca..6d0fe64bbb8 100644
--- a/mysql-test/t/old-mode.test
+++ b/mysql-test/t/old-mode.test
@@ -12,5 +12,6 @@ create table t2 (a int, b varchar(200), c text not null) checksum=0;
 insert t1 values (1, "aaa", "bbb"), (NULL, "", "ccccc"), (0, NULL, "");
 insert t2 select * from t1;
 checksum table t1, t2;
+checksum table t1, t2 quick;
 checksum table t1, t2 extended;
 drop table t1,t2;
diff --git a/mysql-test/t/row-checksum-old-master.opt b/mysql-test/t/row-checksum-old-master.opt
new file mode 100644
index 00000000000..8e7b7f9e36f
--- /dev/null
+++ b/mysql-test/t/row-checksum-old-master.opt
@@ -0,0 +1 @@
+--old
diff --git a/mysql-test/t/row-checksum-old.test b/mysql-test/t/row-checksum-old.test
new file mode 100644
index 00000000000..71188ddf432
--- /dev/null
+++ b/mysql-test/t/row-checksum-old.test
@@ -0,0 +1,4 @@
+#
+# Run row-checksum.test with old mode
+#
+--source t/row-checksum.test
diff --git a/mysql-test/t/row-checksum.test b/mysql-test/t/row-checksum.test
new file mode 100644
index 00000000000..920a2384aa8
--- /dev/null
+++ b/mysql-test/t/row-checksum.test
@@ -0,0 +1,62 @@
+#
+# Test checksum
+#
+
+-- source include/have_innodb.inc
+-- source include/have_maria.inc
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table if exists t1;
+
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
+
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
+
+
+#
+# These checksums will be different prefixes fixed sizes rows with one extra
+# flag byte
+#
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table if exists t1;
+
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index c90f82d24a1..b239253fbc7 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -4615,7 +4615,7 @@ void ha_partition::get_dynamic_partition_info(PARTITION_INFO *stat_info,
   stat_info->update_time=          file->stats.update_time;
   stat_info->check_time=           file->stats.check_time;
   stat_info->check_sum= 0;
-  if (file->ha_table_flags() & HA_HAS_CHECKSUM)
+  if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
     stat_info->check_sum= file->checksum();
   return;
 }
diff --git a/sql/handler.cc b/sql/handler.cc
index f637ebfb02e..68921eb2a3f 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -3435,7 +3435,7 @@ void handler::get_dynamic_partition_info(PARTITION_INFO *stat_info,
   stat_info->update_time=          stats.update_time;
   stat_info->check_time=           stats.check_time;
   stat_info->check_sum=            0;
-  if (table_flags() & (ulong) HA_HAS_CHECKSUM)
+  if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM))
     stat_info->check_sum= checksum();
   return;
 }
diff --git a/sql/handler.h b/sql/handler.h
index b2234dc4b75..12ba5daf1e1 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -107,7 +107,8 @@
 #define HA_CAN_FULLTEXT        (1 << 21)
 #define HA_CAN_SQL_HANDLER     (1 << 22)
 #define HA_NO_AUTO_INCREMENT   (1 << 23)
-#define HA_HAS_CHECKSUM        (1 << 24)
+/* Has automatic checksums and uses the old checksum format */
+#define HA_HAS_OLD_CHECKSUM    (1 << 24)
 /* Table data are stored in separate files (for lower_case_table_names) */
 #define HA_FILE_BASED	       (1 << 26)
 #define HA_NO_VARCHAR	       (1 << 27)
@@ -124,6 +125,8 @@
 */
 #define HA_BINLOG_ROW_CAPABLE  (LL(1) << 34)
 #define HA_BINLOG_STMT_CAPABLE (LL(1) << 35)
+/* Has automatic checksums and uses the new checksum format */
+#define HA_HAS_NEW_CHECKSUM    (LL(1) << 36)
 
 /*
   Set of all binlog flags. Currently only contain the capabilities
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index e0baa986272..bde1274631a 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -3613,7 +3613,7 @@ static int get_schema_tables_record(THD *thd, TABLE_LIST *tables,
         table->field[16]->store_time(&time, MYSQL_TIMESTAMP_DATETIME);
         table->field[16]->set_notnull();
       }
-      if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM)
+      if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
       {
         table->field[18]->store((longlong) file->checksum(), TRUE);
         table->field[18]->set_notnull();
@@ -4670,7 +4670,7 @@ static void store_schema_partitions_record(THD *thd, TABLE *schema_table,
     table->field[20]->store_time(&time, MYSQL_TIMESTAMP_DATETIME);
     table->field[20]->set_notnull();
   }
-  if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM)
+  if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
   {
     table->field[21]->store((longlong) stat_info.check_sum, TRUE);
     table->field[21]->set_notnull();
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 5ee2ccb5a64..d64b621a7db 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -7287,11 +7287,14 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables,
     }
     else
     {
-      if (t->file->ha_table_flags() & HA_HAS_CHECKSUM &&
-	  !(check_opt->flags & T_EXTEND))
+      /* Call ->checksum() if the table checksum matches 'old_mode' settings */
+      if (!(check_opt->flags & T_EXTEND) &&
+          (((t->file->ha_table_flags() & HA_HAS_OLD_CHECKSUM) &&
+            thd->variables.old_mode) ||
+           ((t->file->ha_table_flags() & HA_HAS_NEW_CHECKSUM) &&
+            !thd->variables.old_mode)))
 	protocol->store((ulonglong)t->file->checksum());
-      else if (!(t->file->ha_table_flags() & HA_HAS_CHECKSUM) &&
-	       (check_opt->flags & T_QUICK))
+      else if (check_opt->flags & T_QUICK)
 	protocol->store_null();
       else
       {
diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc
index 420cc953529..34199fd085e 100644
--- a/storage/maria/ha_maria.cc
+++ b/storage/maria/ha_maria.cc
@@ -923,7 +923,7 @@ int ha_maria::open(const char *name, int mode, uint test_if_locked)
     int_table_flags|= HA_CAN_INSERT_DELAYED;
   }
   if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
-    int_table_flags |= HA_HAS_CHECKSUM;
+    int_table_flags |= HA_HAS_NEW_CHECKSUM;
 
   for (i= 0; i < table->s->keys; i++)
   {
diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc
index ee453d34b75..8e755567a5b 100644
--- a/storage/myisam/ha_myisam.cc
+++ b/storage/myisam/ha_myisam.cc
@@ -690,7 +690,19 @@ int ha_myisam::open(const char *name, int mode, uint test_if_locked)
   if (!table->s->db_record_offset)
     int_table_flags|=HA_REC_NOT_IN_SEQ;
   if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
-    int_table_flags|=HA_HAS_CHECKSUM;
+  {
+    /*
+      Set which type of automatic checksum we have
+      The old checksum and new checksum are identical if there is no
+      null fields.
+      Files with new checksum has the HA_OPTION_NULL_FIELDS bit set.
+    */      
+    if ((file->s->options & HA_OPTION_NULL_FIELDS) ||
+        !file->s->has_null_fields)
+      int_table_flags|= HA_HAS_NEW_CHECKSUM;
+    if (!(file->s->options & HA_OPTION_NULL_FIELDS))
+      int_table_flags|= HA_HAS_OLD_CHECKSUM;
+  }
 
   for (i= 0; i < table->s->keys; i++)
   {
@@ -2042,6 +2054,27 @@ bool ha_myisam::check_if_incompatible_data(HA_CREATE_INFO *info,
   return COMPATIBLE_DATA_YES;
 }
 
+
+/**
+  Check if a table is incompatible with the current version.
+
+  The cases are:
+  - Table has checksum, varchars and are not of dynamic record type
+*/
+
+int ha_myisam::check_for_upgrade(HA_CHECK_OPT *check_opt)
+{
+  if (!(file->s->options & HA_OPTION_NULL_FIELDS) &&
+      !(file->s->options & HA_OPTION_PACK_RECORD) &&
+      file->s->has_varchar_fields)
+  {
+    /* We need alter there to get the HA_OPTION_NULL_FIELDS flag to be set */
+    return HA_ADMIN_NEEDS_ALTER;
+  }
+  return HA_ADMIN_OK;
+}
+
+
 extern int mi_panic(enum ha_panic_function flag);
 int myisam_panic(handlerton *hton, ha_panic_function flag)
 {
diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h
index 606d5ca354f..62792a10a20 100644
--- a/storage/myisam/ha_myisam.h
+++ b/storage/myisam/ha_myisam.h
@@ -119,6 +119,7 @@ class ha_myisam: public handler
                                   ulonglong *nb_reserved_values);
   int rename_table(const char * from, const char * to);
   int delete_table(const char *name);
+  int check_for_upgrade(HA_CHECK_OPT *check_opt);
   int check(THD* thd, HA_CHECK_OPT* check_opt);
   int analyze(THD* thd,HA_CHECK_OPT* check_opt);
   int repair(THD* thd, HA_CHECK_OPT* check_opt);
diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c
index ec8fb7ed1d9..1537633bd9d 100644
--- a/storage/myisam/mi_open.c
+++ b/storage/myisam/mi_open.c
@@ -19,6 +19,7 @@
 #include "sp_defs.h"
 #include "rt_index.h"
 #include <m_ctype.h>
+#include <mysql_version.h>
 
 #if defined(MSDOS) || defined(__WIN__)
 #ifdef __WIN__
@@ -453,13 +454,20 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
       share->rec[i].pack_type=0;
       share->rec[i].huff_tree=0;
       share->rec[i].offset=offset;
-      if (share->rec[i].type == (int) FIELD_BLOB)
+      if (share->rec[i].type == FIELD_BLOB)
       {
 	share->blobs[j].pack_length=
 	  share->rec[i].length - portable_sizeof_char_ptr;
 	share->blobs[j].offset=offset;
 	j++;
       }
+#if MYSQL_VERSION_ID <= 60100
+      /* This is to detect old checksum option */
+      if (share->rec[i].null_bit)
+        share->has_null_fields= 1;
+      if (share->rec[i].type == FIELD_VARCHAR)
+        share->has_varchar_fields= 1;
+#endif
       offset+=share->rec[i].length;
     }
     share->rec[i].type=(int) FIELD_LAST;	/* End marker */
diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h
index db6344e14f8..e432708dba2 100644
--- a/storage/myisam/myisamdef.h
+++ b/storage/myisam/myisamdef.h
@@ -210,6 +210,9 @@ typedef struct st_mi_isam_share
   enum data_file_type data_file_type;
   /* Below flag is needed to make log tables work with concurrent insert */
   my_bool is_log_table;
+  /* This is 1 if they table checksum is of old type */
+  my_bool has_null_fields;
+  my_bool has_varchar_fields;
 
   my_bool changed,                      /* If changed since lock */
     global_changed,                     /* If changed since open */