MDEV-11649 Uninitialized field fts_token->position in innodb_fts.innodb_fts_plugin

The field fts_token->position is not initialized in
row_merge_fts_doc_tokenize(). We cannot have that field
without changing the fulltext parser plugin ABI
(adding st_mysql_ftparser_boolean_info::position,
as it was done in MySQL 5.7 in WL#6943).

The InnoDB fulltext parser plugins "ngram" and "Mecab" that were
introduced in MySQL 5.7 do depend on that field. But the simple_parser
does not. Apparently, simple_parser is leaving the field as 0.

So, in our fix we will assume that the missing position field is 0.

4 files changed, 38 insertions, 20 deletions

diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result b/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result
index f057db1d284..b7688e9ef0f 100644
--- a/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result
+++ b/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result
@@ -135,12 +135,29 @@ INSERT INTO articles (title, body) VALUES
 ('How To Use MySQL Well','After you went through a ...'),
 ('Optimizing MySQL','In this tutorial we will show ...'),
 ('1001 MySQL Tricks','How to use full-text search engine'),
-('Go MySQL Tricks','How to use full text search engine');
+('Go MariaDB Tricks','How to use full text search engine');
+SELECT * FROM articles WHERE
+MATCH(title, body) AGAINST('MySQL');
+id	title	body
+6	MySQL Tutorial	DBMS stands for MySQL DataBase ...
+7	How To Use MySQL Well	After you went through a ...
+8	Optimizing MySQL	In this tutorial we will show ...
+9	1001 MySQL Tricks	How to use full-text search engine
+SELECT * FROM articles WHERE
+MATCH(title, body) AGAINST('tutorial');
+id	title	body
+6	MySQL Tutorial	DBMS stands for MySQL DataBase ...
+8	Optimizing MySQL	In this tutorial we will show ...
 SELECT * FROM articles WHERE
 MATCH(title, body) AGAINST('Tricks');
 id	title	body
 9	1001 MySQL Tricks	How to use full-text search engine
-10	Go MySQL Tricks	How to use full text search engine
+10	Go MariaDB Tricks	How to use full text search engine
+SELECT * FROM articles WHERE
+MATCH(title, body) AGAINST('full text search');
+id	title	body
+10	Go MariaDB Tricks	How to use full text search engine
+9	1001 MySQL Tricks	How to use full-text search engine
 SELECT COUNT(*) FROM articles;
 COUNT(*)
 5
diff --git a/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test b/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test
index 9cc1afd4e60..cd31500b23f 100644
--- a/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test
+++ b/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test
@@ -145,13 +145,18 @@ INSERT INTO articles (title, body) VALUES
   ('How To Use MySQL Well','After you went through a ...'),
   ('Optimizing MySQL','In this tutorial we will show ...'),
   ('1001 MySQL Tricks','How to use full-text search engine'),
-  ('Go MySQL Tricks','How to use full text search engine');
+  ('Go MariaDB Tricks','How to use full text search engine');
 
 --source include/restart_mysqld.inc
 
-# Simple term search - 4 records expected
+SELECT * FROM articles WHERE
+         MATCH(title, body) AGAINST('MySQL');
+SELECT * FROM articles WHERE
+         MATCH(title, body) AGAINST('tutorial');
 SELECT * FROM articles WHERE
          MATCH(title, body) AGAINST('Tricks');
+SELECT * FROM articles WHERE
+         MATCH(title, body) AGAINST('full text search');
 SELECT COUNT(*) FROM articles;
 DROP TABLE articles;
 
diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h
index 207fd783cac..b1f3b8ab328 100644
--- a/storage/innobase/include/row0ftsort.h
+++ b/storage/innobase/include/row0ftsort.h
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2015, 2016, MariaDB Corporation.
+Copyright (c) 2015, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -103,7 +103,6 @@ struct fts_psort_t {
 /** Row fts token for plugin parser */
 struct row_fts_token_t {
 	fts_string_t*	text;		/*!< token */
-	ulint		position;	/*!< token position in the document */
 	UT_LIST_NODE_T(row_fts_token_t)
 			token_list;	/*!< next token link */
 };
diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index 321b55e9894..da5d96ccc18 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -527,7 +527,6 @@ row_merge_fts_doc_tokenize(
 	doc id and position to sort buffer */
 	while (t_ctx->processed_len < doc->text.f_len) {
 		ulint		idx = 0;
-		ib_uint32_t	position;
 		ulint		cur_len;
 		doc_id_t	write_doc_id;
 		row_fts_token_t* fts_token = NULL;
@@ -679,20 +678,18 @@ row_merge_fts_doc_tokenize(
 
 		++field;
 
-		/* The third field is the position */
-		if (parser != NULL) {
-			mach_write_to_4(
-				reinterpret_cast<byte*>(&position),
-				(fts_token->position + t_ctx->init_pos));
-		} else {
-			mach_write_to_4(
-				reinterpret_cast<byte*>(&position),
-				(t_ctx->processed_len + inc - str.f_len + t_ctx->init_pos));
+		/* The third field is the position.
+		MySQL 5.7 changed the fulltext parser plugin interface
+		by adding MYSQL_FTPARSER_BOOLEAN_INFO::position.
+		Below we assume that the field is always 0. */
+		unsigned	pos = t_ctx->init_pos;
+		byte		position[4];
+		if (parser == NULL) {
+			pos += t_ctx->processed_len + inc - str.f_len;
 		}
-
-		dfield_set_data(field, &position, sizeof(position));
-		len = dfield_get_len(field);
-		ut_ad(len == sizeof(ib_uint32_t));
+		len = 4;
+		mach_write_to_4(position, pos);
+		dfield_set_data(field, &position, len);
 
 		field->type.mtype = DATA_INT;
 		field->type.prtype = DATA_NOT_NULL;