MDEV-19522 InnoDB commit fails when FTS_DOC_ID valuebb-10.2-MDEV-19522

                    is greater than 4294967295

InnoDB commit fails when consecutive FTS_DOC_ID value
is greater than 4294967295.
Fix is that InnoDB should remove the delta FTS_DOC_ID
value limitations and fts should encode 8 byte value,
remove FTS_DOC_ID_MAX_STEP variable. Renamed the
fts0vlc.ic file to fts0vlc.h

fts_encode_int(): Should be able to encode 8 bytes value

fts_get_encoded_len(): Should get the length of the value
which has 8 bytes

mach_read_uint64_little_endian(): Reads 64 bit stored in
big endian format

uint64_2_exp(): Calculate 2 to power n and return
64 bit value

14 files changed, 86 insertions, 220 deletions

diff --git a/mysql-test/suite/innodb_fts/r/basic.result b/mysql-test/suite/innodb_fts/r/basic.result
index b3fd94509c3..a98de60674a 100644
--- a/mysql-test/suite/innodb_fts/r/basic.result
+++ b/mysql-test/suite/innodb_fts/r/basic.result
@@ -313,9 +313,7 @@ FTS_DOC_ID
 65536
 131071
 drop table t1;
-call mtr.add_suppression("\\[ERROR\\] InnoDB: Doc ID 20030101000000 is too big. Its difference with largest used Doc ID 0 cannot exceed or equal to 65535");
 CREATE TABLE t1 (FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
 title VARCHAR(200), FULLTEXT(title)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (NULL, NULL), (20030101000000, 20030102000000);
-ERROR HY000: Invalid InnoDB FTS Doc ID
 DROP TABLE t1;
diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_misc_1.result b/mysql-test/suite/innodb_fts/r/innodb_fts_misc_1.result
index 8ec0157728c..42730d7916a 100644
--- a/mysql-test/suite/innodb_fts/r/innodb_fts_misc_1.result
+++ b/mysql-test/suite/innodb_fts/r/innodb_fts_misc_1.result
@@ -972,3 +972,24 @@ SELECT * FROM information_schema.innodb_ft_deleted;
 DOC_ID
 DROP TABLE t1;
 SET GLOBAL innodb_ft_aux_table=DEFAULT;
+#
+# MDEV-19522 InnoDB commit fails when FTS_DOC_ID value
+#                is greater than 4294967295
+#
+CREATE TABLE t1(
+FTS_DOC_ID BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+f1 TEXT, f2 TEXT, PRIMARY KEY (FTS_DOC_ID),
+FULLTEXT KEY (f1)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,'txt','bbb');
+UPDATE t1 SET FTS_DOC_ID = 4294967298;
+SELECT * FROM t1 WHERE match(f1) against("txt");
+FTS_DOC_ID	f1	f2
+4294967298	txt	bbb
+SET @@session.insert_id = 100000000000;
+INSERT INTO t1(f1, f2) VALUES ('aaa', 'bbb');
+CREATE FULLTEXT INDEX i ON t1 (f2);
+SELECT * FROM t1 WHERE match(f2) against("bbb");
+FTS_DOC_ID	f1	f2
+4294967298	txt	bbb
+100000000000	aaa	bbb
+DROP TABLE t1;
diff --git a/mysql-test/suite/innodb_fts/t/basic.test b/mysql-test/suite/innodb_fts/t/basic.test
index 7a5c83ffb06..53ad978a5b1 100644
--- a/mysql-test/suite/innodb_fts/t/basic.test
+++ b/mysql-test/suite/innodb_fts/t/basic.test
@@ -277,9 +277,7 @@ insert into t1(f1, f2) values(3, "This is the third record");
 select FTS_DOC_ID from t1;
 drop table t1;
 
-call mtr.add_suppression("\\[ERROR\\] InnoDB: Doc ID 20030101000000 is too big. Its difference with largest used Doc ID 0 cannot exceed or equal to 65535");
 CREATE TABLE t1 (FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
 		 title VARCHAR(200), FULLTEXT(title)) ENGINE=InnoDB;
---error 182
 INSERT INTO t1 VALUES (NULL, NULL), (20030101000000, 20030102000000);
 DROP TABLE t1;
diff --git a/mysql-test/suite/innodb_fts/t/innodb_fts_misc_1.test b/mysql-test/suite/innodb_fts/t/innodb_fts_misc_1.test
index adc10886d66..b0bf2c669ad 100644
--- a/mysql-test/suite/innodb_fts/t/innodb_fts_misc_1.test
+++ b/mysql-test/suite/innodb_fts/t/innodb_fts_misc_1.test
@@ -942,3 +942,21 @@ SET GLOBAL innodb_ft_aux_table='test/t1';
 SELECT * FROM information_schema.innodb_ft_deleted;
 DROP TABLE t1;
 SET GLOBAL innodb_ft_aux_table=DEFAULT;
+
+--echo #
+--echo # MDEV-19522 InnoDB commit fails when FTS_DOC_ID value
+--echo #                is greater than 4294967295
+--echo #
+CREATE TABLE t1(
+        FTS_DOC_ID BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+        f1 TEXT, f2 TEXT, PRIMARY KEY (FTS_DOC_ID),
+        FULLTEXT KEY (f1)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,'txt','bbb');
+UPDATE t1 SET FTS_DOC_ID = 4294967298;
+SELECT * FROM t1 WHERE match(f1) against("txt");
+SET @@session.insert_id = 100000000000;
+INSERT INTO t1(f1, f2) VALUES ('aaa', 'bbb');
+CREATE FULLTEXT INDEX i ON t1 (f2);
+SELECT * FROM t1 WHERE match(f2) against("bbb");
+# Cleanup
+DROP TABLE t1;
diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
index 07077006096..65ba557234a 100644
--- a/storage/innobase/fts/fts0fts.cc
+++ b/storage/innobase/fts/fts0fts.cc
@@ -32,7 +32,7 @@ Full Text Search interface
 #include "fts0priv.h"
 #include "fts0types.h"
 #include "fts0types.ic"
-#include "fts0vlc.ic"
+#include "fts0vlc.h"
 #include "fts0plugin.h"
 #include "dict0priv.h"
 #include "dict0stats.h"
@@ -1247,7 +1247,7 @@ fts_cache_node_add_positions(
 	ulint		enc_len;
 	ulint		last_pos;
 	byte*		ptr_start;
-	ulint		doc_id_delta;
+	doc_id_t	doc_id_delta;
 
 #ifdef UNIV_DEBUG
 	if (cache) {
@@ -1258,7 +1258,7 @@ fts_cache_node_add_positions(
 	ut_ad(doc_id >= node->last_doc_id);
 
 	/* Calculate the space required to store the ilist. */
-	doc_id_delta = (ulint)(doc_id - node->last_doc_id);
+	doc_id_delta = doc_id - node->last_doc_id;
 	enc_len = fts_get_encoded_len(doc_id_delta);
 
 	last_pos = 0;
diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc
index c4cbbfafff4..d399a2e37a2 100644
--- a/storage/innobase/fts/fts0opt.cc
+++ b/storage/innobase/fts/fts0opt.cc
@@ -36,6 +36,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang
 #include "ut0list.h"
 #include "zlib.h"
 #include "fts0opt.h"
+#include "fts0vlc.h"
 
 /** The FTS optimize thread's work queue. */
 ib_wqueue_t* fts_optimize_wq;
@@ -1173,7 +1174,7 @@ fts_optimize_encode_node(
 
 	/* Encode the doc id. Cast to ulint, the delta should be small and
 	therefore no loss of precision. */
-	dst += fts_encode_int((ulint) doc_id_delta, dst);
+	dst += fts_encode_int(doc_id_delta, dst);
 
 	/* Copy the encoded pos array. */
 	memcpy(dst, src, pos_enc_len);
diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc
index b4c72e53afe..342bf56f4a2 100644
--- a/storage/innobase/fts/fts0que.cc
+++ b/storage/innobase/fts/fts0que.cc
@@ -34,6 +34,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang
 #include "fts0pars.h"
 #include "fts0types.h"
 #include "fts0plugin.h"
+#include "fts0vlc.h"
 
 #include <iomanip>
 #include <vector>
@@ -3234,8 +3235,8 @@ fts_query_filter_doc_ids(
 		ulint		freq = 0;
 		fts_doc_freq_t*	doc_freq;
 		fts_match_t*	match = NULL;
-		ulint		last_pos = 0;
-		ulint		pos = fts_decode_vlc(&ptr);
+		doc_id_t	last_pos = 0;
+		doc_id_t	pos = fts_decode_vlc(&ptr);
 
 		/* Some sanity checks. */
 		if (doc_id == 0) {
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index fefd0bdde00..e0a2e253360 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -8543,8 +8543,7 @@ calc_row_difference(
 		    && prebuilt->table->fts
 		    && innobase_strcasecmp(
 			field->field_name, FTS_DOC_ID_COL_NAME) == 0) {
-			doc_id = (doc_id_t) mach_read_from_n_little_endian(
-				n_ptr, 8);
+			doc_id = mach_read_uint64_little_endian(n_ptr, 8);
 			if (doc_id == 0) {
 				return(DB_FTS_INVALID_DOCID);
 			}
@@ -8787,16 +8786,6 @@ calc_row_difference(
 					<< innodb_table->name;
 
 				return(DB_FTS_INVALID_DOCID);
-			} else if ((doc_id
-				    - prebuilt->table->fts->cache->next_doc_id)
-				   >= FTS_DOC_ID_MAX_STEP) {
-
-				ib::warn() << "Doc ID " << doc_id << " is too"
-					" big. Its difference with largest"
-					" Doc ID used " << prebuilt->table->fts
-					->cache->next_doc_id - 1
-					<< " cannot exceed or equal to "
-					<< FTS_DOC_ID_MAX_STEP;
 			}
 
 
diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc
index 03c0efff027..01f0f4baab5 100644
--- a/storage/innobase/handler/i_s.cc
+++ b/storage/innobase/handler/i_s.cc
@@ -58,6 +58,7 @@ Modified Dec 29, 2014 Jan Lindström (Added sys_semaphore_waits)
 #include "fil0fil.h"
 #include "fil0crypt.h"
 #include "dict0crea.h"
+#include "fts0vlc.h"
 
 /** The latest successfully looked up innodb_fts_aux_table */
 UNIV_INTERN table_id_t innodb_ft_aux_table_id;
@@ -2785,7 +2786,7 @@ i_s_fts_index_cache_fill_one_index(
 			ptr = node->ilist;
 
 			while (decoded < node->ilist_size) {
-				ulint	pos = fts_decode_vlc(&ptr);
+				doc_id_t pos = fts_decode_vlc(&ptr);
 
 				doc_id += pos;
 
@@ -3156,7 +3157,7 @@ i_s_fts_index_table_fill_one_fetch(
 			ptr = node->ilist;
 
 			while (decoded < node->ilist_size) {
-				ulint	pos = fts_decode_vlc(&ptr);
+				doc_id_t pos = fts_decode_vlc(&ptr);
 
 				doc_id += pos;
 
diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h
index 84d8ccd26ef..dfac5117c17 100644
--- a/storage/innobase/include/fts0fts.h
+++ b/storage/innobase/include/fts0fts.h
@@ -96,10 +96,6 @@ those defined in mysql file ft_global.h */
 /** Threshold where our optimize thread automatically kicks in */
 #define FTS_OPTIMIZE_THRESHOLD		10000000
 
-/** Threshold to avoid exhausting of doc ids. Consecutive doc id difference
-should not exceed FTS_DOC_ID_MAX_STEP */
-#define FTS_DOC_ID_MAX_STEP		65535
-
 /** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */
 #define FTS_MAX_WORD_LEN		(HA_FT_MAXCHARLEN * 4)
 
diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h
index f5760a16c0e..21d32c7d313 100644
--- a/storage/innobase/include/fts0types.h
+++ b/storage/innobase/include/fts0types.h
@@ -315,16 +315,6 @@ int fts_doc_id_cmp(
 	const void*	p2);			/*!< in: id2 */
 
 /******************************************************************//**
-Decode and return the integer that was encoded using our VLC scheme.*/
-UNIV_INLINE
-ulint
-fts_decode_vlc(
-/*===========*/
-			/*!< out: value decoded */
-	byte**	ptr);	/*!< in: ptr to decode from, this ptr is
-			incremented by the number of bytes decoded */
-
-/******************************************************************//**
 Duplicate a string. */
 UNIV_INLINE
 void
@@ -339,28 +329,6 @@ fts_string_dup(
 	mem_heap_t*		heap);		/*!< in: heap to use */
 
 /******************************************************************//**
-Return length of val if it were encoded using our VLC scheme. */
-UNIV_INLINE
-ulint
-fts_get_encoded_len(
-/*================*/
-						/*!< out: length of value
-						 encoded, in bytes */
-	ulint		val);			/*!< in: value to encode */
-
-/******************************************************************//**
-Encode an integer using our VLC scheme and return the length in bytes. */
-UNIV_INLINE
-ulint
-fts_encode_int(
-/*===========*/
-						/*!< out: length of value
-						encoded, in bytes */
-	ulint		val,			/*!< in: value to encode */
-	byte*		buf);			/*!< in: buffer, must have
-						enough space */
-
-/******************************************************************//**
 Get the selected FTS aux INDEX suffix. */
 UNIV_INLINE
 const char*
@@ -381,6 +349,5 @@ fts_select_index(
 	ulint			len);
 
 #include "fts0types.ic"
-#include "fts0vlc.ic"
 
 #endif /* INNOBASE_FTS0TYPES_H */
diff --git a/storage/innobase/include/fts0vlc.ic b/storage/innobase/include/fts0vlc.ic
deleted file mode 100644
index 75d8535057e..00000000000
--- a/storage/innobase/include/fts0vlc.ic
+++ /dev/null
@@ -1,142 +0,0 @@
-/*****************************************************************************
-
-Copyright (c) 2007, 2011, Oracle and/or its affiliates. All Rights Reserved.
-
-This program is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
-
-*****************************************************************************/
-
-/******************************************************************//**
-@file include/fts0vlc.ic
-Full text variable length integer encoding/decoding.
-
-Created 2007-03-27 Sunny Bains
-*******************************************************/
-
-#ifndef INNOBASE_FTS0VLC_IC
-#define INNOBASE_FTS0VLC_IC
-
-#include "fts0types.h"
-
-/******************************************************************//**
-Return length of val if it were encoded using our VLC scheme.
-FIXME: We will need to be able encode 8 bytes value
-@return length of value encoded, in bytes */
-UNIV_INLINE
-ulint
-fts_get_encoded_len(
-/*================*/
-	ulint	val)	/* in: value to encode */
-{
-	if (val <= 127) {
-		return(1);
-	} else if (val <= 16383) {
-		return(2);
-	} else if (val <= 2097151) {
-		return(3);
-	} else if (val <= 268435455) {
-		return(4);
-	} else {
-		/* Possibly we should care that on 64-bit machines ulint can
-		contain values that we can't encode in 5 bytes, but
-		fts_encode_int doesn't handle them either so it doesn't much
-		matter. */
-
-		return(5);
-	}
-}
-
-/******************************************************************//**
-Encode an integer using our VLC scheme and return the length in bytes.
-@return length of value encoded, in bytes */
-UNIV_INLINE
-ulint
-fts_encode_int(
-/*===========*/
-	ulint	val,	/* in: value to encode */
-	byte*	buf)	/* in: buffer, must have enough space */
-{
-	ulint	len;
-
-	if (val <= 127) {
-		*buf = (byte) val;
-
-		len = 1;
-	} else if (val <= 16383) {
-		*buf++ = (byte)(val >> 7);
-		*buf = (byte)(val & 0x7F);
-
-		len = 2;
-	} else if (val <= 2097151) {
-		*buf++ = (byte)(val >> 14);
-		*buf++ = (byte)((val >> 7) & 0x7F);
-		*buf = (byte)(val & 0x7F);
-
-		len = 3;
-	} else if (val <= 268435455) {
-		*buf++ = (byte)(val >> 21);
-		*buf++ = (byte)((val >> 14) & 0x7F);
-		*buf++ = (byte)((val >> 7) & 0x7F);
-		*buf = (byte)(val & 0x7F);
-
-		len = 4;
-	} else {
-		/* Best to keep the limitations of the 32/64 bit versions
-		identical, at least for the time being. */
-		ut_ad(val <= 4294967295u);
-
-		*buf++ = (byte)(val >> 28);
-		*buf++ = (byte)((val >> 21) & 0x7F);
-		*buf++ = (byte)((val >> 14) & 0x7F);
-		*buf++ = (byte)((val >> 7) & 0x7F);
-		*buf = (byte)(val & 0x7F);
-
-		len = 5;
-	}
-
-	/* High-bit on means "last byte in the encoded integer". */
-	*buf |= 0x80;
-
-	return(len);
-}
-
-/******************************************************************//**
-Decode and return the integer that was encoded using our VLC scheme.
-@return value decoded */
-UNIV_INLINE
-ulint
-fts_decode_vlc(
-/*===========*/
-	byte**	ptr)	/* in: ptr to decode from, this ptr is
-			incremented by the number of bytes decoded */
-{
-	ulint	val = 0;
-
-	for (;;) {
-		byte	b = **ptr;
-
-		++*ptr;
-		val |= (b & 0x7F);
-
-		/* High-bit on means "last byte in the encoded integer". */
-		if (b & 0x80) {
-			break;
-		} else {
-			val <<= 7;
-		}
-	}
-
-	return(val);
-}
-
-#endif
diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h
index 8141c8a91e0..9e02f1bbb8f 100644
--- a/storage/innobase/include/mach0data.h
+++ b/storage/innobase/include/mach0data.h
@@ -316,6 +316,41 @@ mach_read_from_n_little_endian(
 	const byte*	buf,		/*!< in: from where to read */
 	ulint		buf_size)	/*!< in: from how many bytes to read */
 	MY_ATTRIBUTE((warn_unused_result));
+
+
+/** Reads a 64 bit stored in big endian format
+@param	buf		From where to read
+@param	buf_size	How many bytes to read
+@return uint64_t */
+UNIV_INLINE
+uint64_t
+mach_read_uint64_little_endian(const byte* buf, ulint buf_size)
+{
+#ifdef WORDS_BIGENDIAN
+  uint64_t n = 0;
+  const byte *ptr;
+
+  ut_ad(buf_size > 0);
+  ptr = buf + sizeof(uint64_t);
+
+  for (;;)
+  {
+    ptr--;
+    n= n << 8;
+    n+= (ulint)(*ptr);
+
+    if (ptr == buf)
+      break;
+  }
+
+  return(n);
+#else
+  uint64_t n;
+  memcpy(&n, buf, sizeof(uint64_t));
+  return n;
+#endif
+}
+
 /*********************************************************//**
 Writes a ulint in the little-endian format. */
 UNIV_INLINE
diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc
index c2f9186d408..6445f67f3c2 100644
--- a/storage/innobase/row/row0mysql.cc
+++ b/storage/innobase/row/row0mysql.cc
@@ -1465,23 +1465,6 @@ error_exit:
 				trx->error_state = DB_FTS_INVALID_DOCID;
 				goto error_exit;
 			}
-
-			/* Difference between Doc IDs are restricted within
-			4 bytes integer. See fts_get_encoded_len(). Consecutive
-			doc_ids difference should not exceed
-			FTS_DOC_ID_MAX_STEP value. */
-
-			if (doc_id - next_doc_id >= FTS_DOC_ID_MAX_STEP) {
-				 ib::error() << "Doc ID " << doc_id
-					<< " is too big. Its difference with"
-					" largest used Doc ID "
-					<< next_doc_id - 1 << " cannot"
-					" exceed or equal to "
-					<< FTS_DOC_ID_MAX_STEP;
-				err = DB_FTS_INVALID_DOCID;
-				trx->error_state = DB_FTS_INVALID_DOCID;
-				goto error_exit;
-			}
 		}
 
 		if (table->skip_alter_undo) {