Fix for Bug #37007 Maria: different checksum for MyISAM table depending on CHECKSUM=0|1

This also adds a check that MyISAM tables with incompatible checksums are detected by CHECK TABLE ... [FOR UPGRADE] and thus also by mysql_upgrade.
The tables that are incomatible are MyISAM tables with ROW_FORMAT=fixed and has VARCHAR fields and have CHECKSUM enabled.
Before these tables gave different checksum if you used CHECK TABLE with or without EXTENDED

mysql-test/r/old-mode.result:
  Now we get same results with and without EXTENDED
mysql-test/r/row-checksum-old.result:
  Initial results
mysql-test/r/row-checksum.result:
  Initial results
mysql-test/t/old-mode.test:
  Added test with QUICK to show that the live checksum is not used when running with --old
mysql-test/t/row-checksum-old-master.opt:
  Start mysqld with --old mode to enable old checksum code
mysql-test/t/row-checksum-old.test:
  Run row-checksum test under mysqld --old
mysql-test/t/row-checksum.test:
  Verify that checksum are calculated the same way with and without EXTENDED
  We run this with several storage engines to ensure results are the same over storage engines
sql/ha_partition.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/handler.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/handler.h:
  Split HA_HAS_CHECKSUM into HA_HAS_NEW_CHECKSUM and HA_HAS_OLD_CHECKSUM flags.
  This is a safe API change as only MyISAM and Maria should use these handler flags.
sql/sql_show.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/sql_table.cc:
  Use file->checksum() for live checksums if the life checksum method corresponds to the mysqld --old flag
storage/maria/ha_maria.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
storage/myisam/ha_myisam.cc:
  Set HA_HAS_OLD_CHECKSUM and/or HA_HAS_NEW_CHECKSUM flags depending on if this is a new myisam or old myisam file
  Add method check_for_upgrade() to detect if the table is of old version with a checksum that is incompatible with CHECK TABLE ... EXTENDED
storage/myisam/ha_myisam.h:
  Added check_for_upgrade()
storage/myisam/mi_open.c:
  Removed not neede cast
  Initialize share->has_null_fields and share->has_varchar_fields variables
storage/myisam/myisamdef.h:
  Added share->has_null_fields and share->has_varchar_fields

5 files changed, 15 insertions, 9 deletions

diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index c90f82d24a1..b239253fbc7 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -4615,7 +4615,7 @@ void ha_partition::get_dynamic_partition_info(PARTITION_INFO *stat_info,
   stat_info->update_time=          file->stats.update_time;
   stat_info->check_time=           file->stats.check_time;
   stat_info->check_sum= 0;
-  if (file->ha_table_flags() & HA_HAS_CHECKSUM)
+  if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
     stat_info->check_sum= file->checksum();
   return;
 }
diff --git a/sql/handler.cc b/sql/handler.cc
index f637ebfb02e..68921eb2a3f 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -3435,7 +3435,7 @@ void handler::get_dynamic_partition_info(PARTITION_INFO *stat_info,
   stat_info->update_time=          stats.update_time;
   stat_info->check_time=           stats.check_time;
   stat_info->check_sum=            0;
-  if (table_flags() & (ulong) HA_HAS_CHECKSUM)
+  if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM))
     stat_info->check_sum= checksum();
   return;
 }
diff --git a/sql/handler.h b/sql/handler.h
index b2234dc4b75..12ba5daf1e1 100644
--- a/sql/handler.h
+++ b/sql/handler.h
@@ -107,7 +107,8 @@
 #define HA_CAN_FULLTEXT        (1 << 21)
 #define HA_CAN_SQL_HANDLER     (1 << 22)
 #define HA_NO_AUTO_INCREMENT   (1 << 23)
-#define HA_HAS_CHECKSUM        (1 << 24)
+/* Has automatic checksums and uses the old checksum format */
+#define HA_HAS_OLD_CHECKSUM    (1 << 24)
 /* Table data are stored in separate files (for lower_case_table_names) */
 #define HA_FILE_BASED	       (1 << 26)
 #define HA_NO_VARCHAR	       (1 << 27)
@@ -124,6 +125,8 @@
 */
 #define HA_BINLOG_ROW_CAPABLE  (LL(1) << 34)
 #define HA_BINLOG_STMT_CAPABLE (LL(1) << 35)
+/* Has automatic checksums and uses the new checksum format */
+#define HA_HAS_NEW_CHECKSUM    (LL(1) << 36)
 
 /*
   Set of all binlog flags. Currently only contain the capabilities
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index e0baa986272..bde1274631a 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -3613,7 +3613,7 @@ static int get_schema_tables_record(THD *thd, TABLE_LIST *tables,
         table->field[16]->store_time(&time, MYSQL_TIMESTAMP_DATETIME);
         table->field[16]->set_notnull();
       }
-      if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM)
+      if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
       {
         table->field[18]->store((longlong) file->checksum(), TRUE);
         table->field[18]->set_notnull();
@@ -4670,7 +4670,7 @@ static void store_schema_partitions_record(THD *thd, TABLE *schema_table,
     table->field[20]->store_time(&time, MYSQL_TIMESTAMP_DATETIME);
     table->field[20]->set_notnull();
   }
-  if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM)
+  if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
   {
     table->field[21]->store((longlong) stat_info.check_sum, TRUE);
     table->field[21]->set_notnull();
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 5ee2ccb5a64..d64b621a7db 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -7287,11 +7287,14 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables,
     }
     else
     {
-      if (t->file->ha_table_flags() & HA_HAS_CHECKSUM &&
-	  !(check_opt->flags & T_EXTEND))
+      /* Call ->checksum() if the table checksum matches 'old_mode' settings */
+      if (!(check_opt->flags & T_EXTEND) &&
+          (((t->file->ha_table_flags() & HA_HAS_OLD_CHECKSUM) &&
+            thd->variables.old_mode) ||
+           ((t->file->ha_table_flags() & HA_HAS_NEW_CHECKSUM) &&
+            !thd->variables.old_mode)))
 	protocol->store((ulonglong)t->file->checksum());
-      else if (!(t->file->ha_table_flags() & HA_HAS_CHECKSUM) &&
-	       (check_opt->flags & T_QUICK))
+      else if (check_opt->flags & T_QUICK)
 	protocol->store_null();
       else
       {