Fix for Bug #37007 Maria: different checksum for MyISAM table depending on CHECKSUM=0|1

This also adds a check that MyISAM tables with incompatible checksums are detected by CHECK TABLE ... [FOR UPGRADE] and thus also by mysql_upgrade.
The tables that are incomatible are MyISAM tables with ROW_FORMAT=fixed and has VARCHAR fields and have CHECKSUM enabled.
Before these tables gave different checksum if you used CHECK TABLE with or without EXTENDED

mysql-test/r/old-mode.result:
  Now we get same results with and without EXTENDED
mysql-test/r/row-checksum-old.result:
  Initial results
mysql-test/r/row-checksum.result:
  Initial results
mysql-test/t/old-mode.test:
  Added test with QUICK to show that the live checksum is not used when running with --old
mysql-test/t/row-checksum-old-master.opt:
  Start mysqld with --old mode to enable old checksum code
mysql-test/t/row-checksum-old.test:
  Run row-checksum test under mysqld --old
mysql-test/t/row-checksum.test:
  Verify that checksum are calculated the same way with and without EXTENDED
  We run this with several storage engines to ensure results are the same over storage engines
sql/ha_partition.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/handler.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/handler.h:
  Split HA_HAS_CHECKSUM into HA_HAS_NEW_CHECKSUM and HA_HAS_OLD_CHECKSUM flags.
  This is a safe API change as only MyISAM and Maria should use these handler flags.
sql/sql_show.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
sql/sql_table.cc:
  Use file->checksum() for live checksums if the life checksum method corresponds to the mysqld --old flag
storage/maria/ha_maria.cc:
  Use new HA_HAS_xxx_CHECKSUM flags
storage/myisam/ha_myisam.cc:
  Set HA_HAS_OLD_CHECKSUM and/or HA_HAS_NEW_CHECKSUM flags depending on if this is a new myisam or old myisam file
  Add method check_for_upgrade() to detect if the table is of old version with a checksum that is incompatible with CHECK TABLE ... EXTENDED
storage/myisam/ha_myisam.h:
  Added check_for_upgrade()
storage/myisam/mi_open.c:
  Removed not neede cast
  Initialize share->has_null_fields and share->has_varchar_fields variables
storage/myisam/myisamdef.h:
  Added share->has_null_fields and share->has_varchar_fields

3 files changed, 175 insertions, 1 deletions

diff --git a/mysql-test/r/old-mode.result b/mysql-test/r/old-mode.result
index df2c0b6fee0..a9815d7dab2 100644
--- a/mysql-test/r/old-mode.result
+++ b/mysql-test/r/old-mode.result
@@ -5,8 +5,12 @@ insert t1 values (1, "aaa", "bbb"), (NULL, "", "ccccc"), (0, NULL, "");
 insert t2 select * from t1;
 checksum table t1, t2;
 Table	Checksum
-test.t1	3442722830
+test.t1	2948697075
 test.t2	2948697075
+checksum table t1, t2 quick;
+Table	Checksum
+test.t1	NULL
+test.t2	NULL
 checksum table t1, t2 extended;
 Table	Checksum
 test.t1	2948697075
diff --git a/mysql-test/r/row-checksum-old.result b/mysql-test/r/row-checksum-old.result
new file mode 100644
index 00000000000..3cf5a7104b9
--- /dev/null
+++ b/mysql-test/r/row-checksum-old.result
@@ -0,0 +1,85 @@
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	4108368782
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	4108368782
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	4108368782
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	4108368782
+drop table t1;
diff --git a/mysql-test/r/row-checksum.result b/mysql-test/r/row-checksum.result
new file mode 100644
index 00000000000..31ae094859b
--- /dev/null
+++ b/mysql-test/r/row-checksum.result
@@ -0,0 +1,85 @@
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	229851577
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	229851577
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 quick;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 extended;
+Table	Checksum
+test.t1	3885665021
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	3885665021
+drop table t1;