MDEV-15824 innodb_defragment=ON trumps innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE

ha_innobase::optimize(): If both innodb_defragment and
innodb_optimize_fulltext_only are at their default settings (OFF),
fall back to ALTER TABLE. Else process one or both options.

4 files changed, 44 insertions, 28 deletions

diff --git a/mysql-test/suite/innodb/r/innodb_defragment_small.result b/mysql-test/suite/innodb/r/innodb_defragment_small.result
index 3f594f79e70..07e58f442d9 100644
--- a/mysql-test/suite/innodb/r/innodb_defragment_small.result
+++ b/mysql-test/suite/innodb/r/innodb_defragment_small.result
@@ -1,6 +1,13 @@
 SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment;
+SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only;
 SET GLOBAL innodb_defragment = 1;
-CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB;
+SET GLOBAL innodb_optimize_fulltext_only = 0;
+#
+# MDEV-12198 innodb_defragment=1 crashes server on
+# OPTIMIZE TABLE when FULLTEXT index exists
+#
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256),
+KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB;
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
@@ -11,14 +18,18 @@ INSERT INTO t1 VALUES (400000, REPEAT('A', 256));
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
-DROP TABLE t1;
 #
-# MDEV-12198 innodb_defragment=1 crashes server on
-# OPTIMIZE TABLE when FULLTEXT index exists
+# MDEV-15824 innodb_defragment=ON trumps
+# innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE
 #
-CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB;
+SET GLOBAL innodb_optimize_fulltext_only = 1;
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+SET GLOBAL innodb_defragment = 0;
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
 DROP TABLE t1;
 SET GLOBAL innodb_defragment = @innodb_defragment_orig;
+SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig;
diff --git a/mysql-test/suite/innodb/t/innodb_defragment_small.test b/mysql-test/suite/innodb/t/innodb_defragment_small.test
index 454333d6986..c52a58fc5a7 100644
--- a/mysql-test/suite/innodb/t/innodb_defragment_small.test
+++ b/mysql-test/suite/innodb/t/innodb_defragment_small.test
@@ -1,10 +1,17 @@
 --source include/have_innodb.inc
 
 SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment;
+SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only;
 SET GLOBAL innodb_defragment = 1;
+SET GLOBAL innodb_optimize_fulltext_only = 0;
 
-# Small tests copied from innodb.innodb_defragment
-CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB;
+--echo #
+--echo # MDEV-12198 innodb_defragment=1 crashes server on
+--echo # OPTIMIZE TABLE when FULLTEXT index exists
+--echo #
+
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256),
+                 KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB;
 OPTIMIZE TABLE t1;
 
 INSERT INTO t1 VALUES (100000, REPEAT('A', 256));
@@ -13,16 +20,18 @@ INSERT INTO t1 VALUES (300000, REPEAT('A', 256));
 INSERT INTO t1 VALUES (400000, REPEAT('A', 256));
 
 OPTIMIZE TABLE t1;
-DROP TABLE t1;
 
 --echo #
---echo # MDEV-12198 innodb_defragment=1 crashes server on
---echo # OPTIMIZE TABLE when FULLTEXT index exists
+--echo # MDEV-15824 innodb_defragment=ON trumps
+--echo # innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE
 --echo #
 
-CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB;
-
+SET GLOBAL innodb_optimize_fulltext_only = 1;
 OPTIMIZE TABLE t1;
+SET GLOBAL innodb_defragment = 0;
+OPTIMIZE TABLE t1;
+
 DROP TABLE t1;
 
 SET GLOBAL innodb_defragment = @innodb_defragment_orig;
+SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig;
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index f2c241d0869..c525c2f0efa 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -13904,6 +13904,7 @@ ha_innobase::optimize(
 
 	This works OK otherwise, but MySQL locks the entire table during
 	calls to OPTIMIZE, which is undesirable. */
+	bool try_alter = true;
 
 	if (srv_defragment) {
 		int err;
@@ -13911,7 +13912,7 @@ ha_innobase::optimize(
 		err = defragment_table(prebuilt->table->name, NULL, false);
 
 		if (err == 0) {
-			return (HA_ADMIN_OK);
+			try_alter = false;
 		} else {
 			push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
 				err,
@@ -13919,9 +13920,7 @@ ha_innobase::optimize(
 				prebuilt->table->name, err);
 
 			if(err == ER_SP_ALREADY_EXISTS) {
-				return (HA_ADMIN_OK);
-			} else {
-				return (HA_ADMIN_TRY_ALTER);
+				try_alter = false;
 			}
 		}
 	}
@@ -13932,11 +13931,10 @@ ha_innobase::optimize(
 			fts_sync_table(prebuilt->table, false, true, false);
 			fts_optimize_table(prebuilt->table);
 		}
-		return(HA_ADMIN_OK);
-	} else {
-
-		return(HA_ADMIN_TRY_ALTER);
+		try_alter = false;
 	}
+
+	return try_alter ? HA_ADMIN_TRY_ALTER : HA_ADMIN_OK;
 }
 
 /*******************************************************************//**
diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc
index a673dd78c70..d642f9c0a06 100644
--- a/storage/xtradb/handler/ha_innodb.cc
+++ b/storage/xtradb/handler/ha_innodb.cc
@@ -14522,6 +14522,7 @@ ha_innobase::optimize(
 
 	This works OK otherwise, but MySQL locks the entire table during
 	calls to OPTIMIZE, which is undesirable. */
+	bool try_alter = true;
 
 	if (srv_defragment) {
 		int err;
@@ -14529,7 +14530,7 @@ ha_innobase::optimize(
 		err = defragment_table(prebuilt->table->name, NULL, false);
 
 		if (err == 0) {
-			return (HA_ADMIN_OK);
+			try_alter = false;
 		} else {
 			push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
 				err,
@@ -14537,9 +14538,7 @@ ha_innobase::optimize(
 				prebuilt->table->name, err);
 
 			if(err == ER_SP_ALREADY_EXISTS) {
-				return (HA_ADMIN_OK);
-			} else {
-				return (HA_ADMIN_TRY_ALTER);
+				try_alter = false;
 			}
 		}
 	}
@@ -14550,11 +14549,10 @@ ha_innobase::optimize(
 			fts_sync_table(prebuilt->table, false, true, false);
 			fts_optimize_table(prebuilt->table);
 		}
-		return(HA_ADMIN_OK);
-	} else {
-
-		return(HA_ADMIN_TRY_ALTER);
+		try_alter = false;
 	}
+
+	return try_alter ? HA_ADMIN_TRY_ALTER : HA_ADMIN_OK;
 }
 
 /*******************************************************************//**