diff options
Diffstat (limited to 'storage/innobase')
| -rw-r--r-- | storage/innobase/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | storage/innobase/fts/fts0fts.cc | 24 | ||||
| -rw-r--r-- | storage/innobase/fts/fts0opt.cc | 13 | ||||
| -rw-r--r-- | storage/innobase/fts/fts0que.cc | 7 | ||||
| -rw-r--r-- | storage/innobase/handler/ha_innodb.cc | 36 | ||||
| -rw-r--r-- | storage/innobase/handler/handler0alter.cc | 2 | ||||
| -rw-r--r-- | storage/innobase/handler/i_s.cc | 18 | ||||
| -rw-r--r-- | storage/innobase/include/fts0fts.h | 4 | ||||
| -rw-r--r-- | storage/innobase/include/fts0types.h | 38 | ||||
| -rw-r--r-- | storage/innobase/include/fts0vlc.h | 124 | ||||
| -rw-r--r-- | storage/innobase/include/fts0vlc.ic | 142 | ||||
| -rw-r--r-- | storage/innobase/include/mach0data.h | 22 | ||||
| -rw-r--r-- | storage/innobase/include/ut0wqueue.h | 11 | ||||
| -rw-r--r-- | storage/innobase/row/row0mysql.cc | 17 | ||||
| -rw-r--r-- | storage/innobase/trx/trx0trx.cc | 18 | ||||
| -rw-r--r-- | storage/innobase/unittest/CMakeLists.txt | 22 | ||||
| -rw-r--r-- | storage/innobase/unittest/innodb_fts-t.cc | 52 | ||||
| -rw-r--r-- | storage/innobase/ut/ut0wqueue.cc | 29 |
18 files changed, 301 insertions, 282 deletions
diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index f53f07b79b6..8a4fc0e79d9 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -195,3 +195,7 @@ ENDIF() IF(NOT (PLUGIN_INNOBASE STREQUAL DYNAMIC)) ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup) ENDIF() + +IF(WITH_UNIT_TESTS) + ADD_SUBDIRECTORY(unittest) +ENDIF() diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 6454135b29f..816306d6c88 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -32,7 +32,7 @@ Full Text Search interface #include "fts0priv.h" #include "fts0types.h" #include "fts0types.ic" -#include "fts0vlc.ic" +#include "fts0vlc.h" #include "fts0plugin.h" #include "dict0priv.h" #include "dict0stats.h" @@ -579,6 +579,7 @@ fts_cache_init( cache->sync_heap->arg = mem_heap_create(1024); cache->total_size = 0; + cache->total_size_at_sync = 0; mutex_enter((ib_mutex_t*) &cache->deleted_lock); cache->deleted_doc_ids = ib_vector_create( @@ -1246,7 +1247,7 @@ fts_cache_node_add_positions( ulint enc_len; ulint last_pos; byte* ptr_start; - ulint doc_id_delta; + doc_id_t doc_id_delta; #ifdef UNIV_DEBUG if (cache) { @@ -1257,7 +1258,7 @@ fts_cache_node_add_positions( ut_ad(doc_id >= node->last_doc_id); /* Calculate the space required to store the ilist. */ - doc_id_delta = (ulint)(doc_id - node->last_doc_id); + doc_id_delta = doc_id - node->last_doc_id; enc_len = fts_get_encoded_len(doc_id_delta); last_pos = 0; @@ -1306,14 +1307,14 @@ fts_cache_node_add_positions( ptr_start = ptr; /* Encode the new fragment. */ - ptr += fts_encode_int(doc_id_delta, ptr); + ptr = fts_encode_int(doc_id_delta, ptr); last_pos = 0; for (i = 0; i < ib_vector_size(positions); i++) { ulint pos = *(static_cast<ulint*>( ib_vector_get(positions, i))); - ptr += fts_encode_int(pos - last_pos, ptr); + ptr = fts_encode_int(pos - last_pos, ptr); last_pos = pos; } @@ -3536,11 +3537,14 @@ fts_add_doc_by_id( get_doc->index_cache, doc_id, doc.tokens); - bool need_sync = false; - if ((cache->total_size > fts_max_cache_size / 10 - || fts_need_sync) - && !cache->sync->in_progress) { - need_sync = true; + bool need_sync = !cache->sync->in_progress + && (fts_need_sync + || (cache->total_size + - cache->total_size_at_sync) + > fts_max_cache_size / 10); + if (need_sync) { + cache->total_size_at_sync = + cache->total_size; } rw_lock_x_unlock(&table->fts->cache->lock); diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 392907eb956..3969e342512 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -36,6 +36,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "ut0list.h" #include "zlib.h" #include "fts0opt.h" +#include "fts0vlc.h" /** The FTS optimize thread's work queue. */ ib_wqueue_t* fts_optimize_wq; @@ -1116,7 +1117,7 @@ fts_optimize_encode_node( ulint pos_enc_len; doc_id_t doc_id_delta; dberr_t error = DB_SUCCESS; - byte* src = enc->src_ilist_ptr; + const byte* src = enc->src_ilist_ptr; if (node->first_doc_id == 0) { ut_a(node->last_doc_id == 0); @@ -1173,7 +1174,7 @@ fts_optimize_encode_node( /* Encode the doc id. Cast to ulint, the delta should be small and therefore no loss of precision. */ - dst += fts_encode_int((ulint) doc_id_delta, dst); + dst = fts_encode_int(doc_id_delta, dst); /* Copy the encoded pos array. */ memcpy(dst, src, pos_enc_len); @@ -1220,7 +1221,8 @@ fts_optimize_node( doc_id_t delta; doc_id_t del_doc_id = FTS_NULL_DOC_ID; - delta = fts_decode_vlc(&enc->src_ilist_ptr); + delta = fts_decode_vlc( + (const byte**)&enc->src_ilist_ptr); test_again: /* Check whether the doc id is in the delete list, if @@ -1248,7 +1250,7 @@ test_again: /* Skip the entries for this document. */ while (*enc->src_ilist_ptr) { - fts_decode_vlc(&enc->src_ilist_ptr); + fts_decode_vlc((const byte**)&enc->src_ilist_ptr); } /* Skip the end of word position marker. */ @@ -2640,6 +2642,9 @@ fts_optimize_request_sync_table( ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true); + DBUG_EXECUTE_IF("fts_optimize_wq_count_check", + DBUG_ASSERT(fts_optimize_wq->length <= 1000);); + mutex_exit(&fts_optimize_wq->mutex); } diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 3c689d0c496..7ab7301ce8c 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -34,6 +34,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang #include "fts0pars.h" #include "fts0types.h" #include "fts0plugin.h" +#include "fts0vlc.h" #include <iomanip> #include <vector> @@ -3224,7 +3225,7 @@ fts_query_filter_doc_ids( ulint len, /*!< in: doc id ilist size */ ibool calc_doc_count) /*!< in: whether to remember doc count */ { - byte* ptr = static_cast<byte*>(data); + const byte* ptr = static_cast<byte*>(data); doc_id_t doc_id = 0; ulint decoded = 0; ib_rbt_t* doc_freqs = word_freq->doc_freqs; @@ -3234,8 +3235,8 @@ fts_query_filter_doc_ids( ulint freq = 0; fts_doc_freq_t* doc_freq; fts_match_t* match = NULL; - ulint last_pos = 0; - ulint pos = fts_decode_vlc(&ptr); + doc_id_t last_pos = 0; + doc_id_t pos = fts_decode_vlc(&ptr); /* Some sanity checks. */ if (doc_id == 0) { diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ffe5a53b055..5bbdee683bb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8431,8 +8431,7 @@ calc_row_difference( && prebuilt->table->fts && innobase_strcasecmp( field->field_name.str, FTS_DOC_ID_COL_NAME) == 0) { - doc_id = (doc_id_t) mach_read_from_n_little_endian( - n_ptr, 8); + doc_id = mach_read_uint64_little_endian(n_ptr); if (doc_id == 0) { return(DB_FTS_INVALID_DOCID); } @@ -8673,16 +8672,6 @@ calc_row_difference( << innodb_table->name; return(DB_FTS_INVALID_DOCID); - } else if ((doc_id - - prebuilt->table->fts->cache->next_doc_id) - >= FTS_DOC_ID_MAX_STEP) { - - ib::warn() << "Doc ID " << doc_id << " is too" - " big. Its difference with largest" - " Doc ID used " << prebuilt->table->fts - ->cache->next_doc_id - 1 - << " cannot exceed or equal to " - << FTS_DOC_ID_MAX_STEP; } @@ -11882,10 +11871,6 @@ bool create_table_info_t::innobase_table_flags() ut_min(static_cast<ulint>(UNIV_PAGE_SSIZE_MAX), static_cast<ulint>(PAGE_ZIP_SSIZE_MAX)); - /* Cache the value of innobase_compression_level, in case it is - modified by another thread while the table is being created. */ - const ulint default_compression_level = page_zip_level; - ha_table_option_struct *options= m_form->s->option_struct; m_flags = 0; @@ -12074,12 +12059,23 @@ index_bad: m_flags2 |= DICT_TF2_USE_FILE_PER_TABLE; } + ulint level = ulint(options->page_compression_level); + if (!level) { + level = page_zip_level; + if (!level && options->page_compressed) { + push_warning_printf( + m_thd, Sql_condition::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: PAGE_COMPRESSED requires" + " PAGE_COMPRESSION_LEVEL or" + " innodb_compression_level > 0"); + DBUG_RETURN(false); + } + } + /* Set the table flags */ dict_tf_set(&m_flags, innodb_row_format, zip_ssize, - m_use_data_dir, - options->page_compressed, - options->page_compression_level == 0 ? - default_compression_level : ulint(options->page_compression_level)); + m_use_data_dir, options->page_compressed, level); if (m_form->s->table_type == TABLE_TYPE_SEQUENCE) { m_flags |= DICT_TF_MASK_NO_ROLLBACK; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index d0ced8125a9..7ee07f4160e 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -7814,6 +7814,8 @@ check_if_ok_to_rename: } if (!info.innobase_table_flags()) { + my_error(ER_ILLEGAL_HA_CREATE_OPTION, MYF(0), + table_type(), "PAGE_COMPRESSED"); goto err_exit_no_heap; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index cdca1a3c276..8a166398f66 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -58,6 +58,7 @@ Modified Dec 29, 2014 Jan Lindström (Added sys_semaphore_waits) #include "fil0fil.h" #include "fil0crypt.h" #include "dict0crea.h" +#include "fts0vlc.h" /** The latest successfully looked up innodb_fts_aux_table */ UNIV_INTERN table_id_t innodb_ft_aux_table_id; @@ -2759,7 +2760,7 @@ i_s_fts_index_cache_fill_one_index( /* Decrypt the ilist, and display Dod ID and word position */ for (ulint i = 0; i < ib_vector_size(word->nodes); i++) { fts_node_t* node; - byte* ptr; + const byte* ptr; ulint decoded = 0; doc_id_t doc_id = 0; @@ -2769,13 +2770,11 @@ i_s_fts_index_cache_fill_one_index( ptr = node->ilist; while (decoded < node->ilist_size) { - ulint pos = fts_decode_vlc(&ptr); - doc_id += pos; + doc_id += fts_decode_vlc(&ptr); /* Get position info */ while (*ptr) { - pos = fts_decode_vlc(&ptr); OK(field_store_string( fields[I_S_FTS_WORD], @@ -2796,7 +2795,7 @@ i_s_fts_index_cache_fill_one_index( doc_id, true)); OK(fields[I_S_FTS_ILIST_DOC_POS]->store( - pos, true)); + fts_decode_vlc(&ptr), true)); OK(schema_table_store_record( thd, table)); @@ -3130,7 +3129,7 @@ i_s_fts_index_table_fill_one_fetch( /* Decrypt the ilist, and display Dod ID and word position */ for (ulint i = 0; i < ib_vector_size(word->nodes); i++) { fts_node_t* node; - byte* ptr; + const byte* ptr; ulint decoded = 0; doc_id_t doc_id = 0; @@ -3140,13 +3139,10 @@ i_s_fts_index_table_fill_one_fetch( ptr = node->ilist; while (decoded < node->ilist_size) { - ulint pos = fts_decode_vlc(&ptr); - - doc_id += pos; + doc_id += fts_decode_vlc(&ptr); /* Get position info */ while (*ptr) { - pos = fts_decode_vlc(&ptr); OK(field_store_string( fields[I_S_FTS_WORD], @@ -3165,7 +3161,7 @@ i_s_fts_index_table_fill_one_fetch( longlong(doc_id), true)); OK(fields[I_S_FTS_ILIST_DOC_POS]->store( - pos, true)); + fts_decode_vlc(&ptr), true)); OK(schema_table_store_record( thd, table)); diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 834a2bc9407..f9c3187bee9 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -96,10 +96,6 @@ those defined in mysql file ft_global.h */ /** Threshold where our optimize thread automatically kicks in */ #define FTS_OPTIMIZE_THRESHOLD 10000000 -/** Threshold to avoid exhausting of doc ids. Consecutive doc id difference -should not exceed FTS_DOC_ID_MAX_STEP */ -#define FTS_DOC_ID_MAX_STEP 65535 - /** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */ #define FTS_MAX_WORD_LEN (HA_FT_MAXCHARLEN * 4) diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index f5760a16c0e..d49bc7c0254 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -150,6 +150,9 @@ struct fts_cache_t { size_t total_size; /*!< total size consumed by the ilist field of all nodes. SYNC is run whenever this gets too big */ + /** total_size at the time of the previous SYNC request */ + size_t total_size_at_sync; + fts_sync_t* sync; /*!< sync structure to sync data to disk */ ib_alloc_t* sync_heap; /*!< The heap allocator, for indexes @@ -315,16 +318,6 @@ int fts_doc_id_cmp( const void* p2); /*!< in: id2 */ /******************************************************************//** -Decode and return the integer that was encoded using our VLC scheme.*/ -UNIV_INLINE -ulint -fts_decode_vlc( -/*===========*/ - /*!< out: value decoded */ - byte** ptr); /*!< in: ptr to decode from, this ptr is - incremented by the number of bytes decoded */ - -/******************************************************************//** Duplicate a string. */ UNIV_INLINE void @@ -339,28 +332,6 @@ fts_string_dup( mem_heap_t* heap); /*!< in: heap to use */ /******************************************************************//** -Return length of val if it were encoded using our VLC scheme. */ -UNIV_INLINE -ulint -fts_get_encoded_len( -/*================*/ - /*!< out: length of value - encoded, in bytes */ - ulint val); /*!< in: value to encode */ - -/******************************************************************//** -Encode an integer using our VLC scheme and return the length in bytes. */ -UNIV_INLINE -ulint -fts_encode_int( -/*===========*/ - /*!< out: length of value - encoded, in bytes */ - ulint val, /*!< in: value to encode */ - byte* buf); /*!< in: buffer, must have - enough space */ - -/******************************************************************//** Get the selected FTS aux INDEX suffix. */ UNIV_INLINE const char* @@ -381,6 +352,5 @@ fts_select_index( ulint len); #include "fts0types.ic" -#include "fts0vlc.ic" #endif /* INNOBASE_FTS0TYPES_H */ diff --git a/storage/innobase/include/fts0vlc.h b/storage/innobase/include/fts0vlc.h new file mode 100644 index 00000000000..d6e6037777e --- /dev/null +++ b/storage/innobase/include/fts0vlc.h @@ -0,0 +1,124 @@ +/** + +Copyright (c) 2021, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +**/ +/** +@file include/fts0vlc.h +Full text variable length integer encoding/decoding. + +Created 2021-10-19 Thirunarayanan Balathandayuthapani +**/ + +/** Return length of val if it were encoded using our VLC scheme. +@param val value to encode +@return length of value encoded, in bytes */ +inline size_t fts_get_encoded_len(doc_id_t val) +{ + if (val < static_cast<doc_id_t>(1) << 7) + return 1; + if (val < static_cast<doc_id_t>(1) << 14) + return 2; + if (val < static_cast<doc_id_t>(1) << 21) + return 3; + if (val < static_cast<doc_id_t>(1) << 28) + return 4; + if (val < static_cast<doc_id_t>(1) << 35) + return 5; + if (val < static_cast<doc_id_t>(1) << 42) + return 6; + if (val < static_cast<doc_id_t>(1) << 49) + return 7; + if (val < static_cast<doc_id_t>(1) << 56) + return 8; + if (val < static_cast<doc_id_t>(1) << 63) + return 9; + return 10; +} + +/** Encode an integer using our VLC scheme and return the +length in bytes. +@param val value to encode +@param buf buffer, must have enough space +@return length of value encoded, in bytes */ +inline byte *fts_encode_int(doc_id_t val, byte *buf) +{ + if (val < static_cast<doc_id_t>(1) << 7) + goto add_1; + if (val < static_cast<doc_id_t>(1) << 14) + goto add_2; + if (val < static_cast<doc_id_t>(1) << 21) + goto add_3; + if (val < static_cast<doc_id_t>(1) << 28) + goto add_4; + if (val < static_cast<doc_id_t>(1) << 35) + goto add_5; + if (val < static_cast<doc_id_t>(1) << 42) + goto add_6; + if (val < static_cast<doc_id_t>(1) << 49) + goto add_7; + if (val < static_cast<doc_id_t>(1) << 56) + goto add_8; + if (val < static_cast<doc_id_t>(1) << 63) + goto add_9; + + *buf++= static_cast<byte>(val >> 63); +add_9: + *buf++= static_cast<byte>(val >> 56) & 0x7F; +add_8: + *buf++= static_cast<byte>(val >> 49) & 0x7F; +add_7: + *buf++= static_cast<byte>(val >> 42) & 0x7F; +add_6: + *buf++= static_cast<byte>(val >> 35) & 0x7F; +add_5: + *buf++= static_cast<byte>(val >> 28) & 0x7F; +add_4: + *buf++= static_cast<byte>(val >> 21) & 0x7F; +add_3: + *buf++= static_cast<byte>(val >> 14) & 0x7F; +add_2: + *buf++= static_cast<byte>(val >> 7) & 0x7F; +add_1: + *buf++= static_cast<byte>(val) | 0x80; + return buf; +} + +/** Decode and return the integer that was encoded using +our VLC scheme. +@param ptr pointer to decode from, this ptr is + incremented by the number of bytes decoded +@return value decoded */ +inline doc_id_t fts_decode_vlc(const byte **ptr) +{ + ut_d(const byte *const start= *ptr); + ut_ad(*start); + + doc_id_t val= 0; + for (;;) + { + byte b= *(*ptr)++; + val|= (b & 0x7F); + + /* High-bit on means "last byte in the encoded integer". */ + if (b & 0x80) + break; + ut_ad(val < static_cast<doc_id_t>(1) << (64 - 7)); + val <<= 7; + } + + ut_ad(*ptr - start <= 10); + + return(val); +} diff --git a/storage/innobase/include/fts0vlc.ic b/storage/innobase/include/fts0vlc.ic deleted file mode 100644 index 75d8535057e..00000000000 --- a/storage/innobase/include/fts0vlc.ic +++ /dev/null @@ -1,142 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2007, 2011, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/******************************************************************//** -@file include/fts0vlc.ic -Full text variable length integer encoding/decoding. - -Created 2007-03-27 Sunny Bains -*******************************************************/ - -#ifndef INNOBASE_FTS0VLC_IC -#define INNOBASE_FTS0VLC_IC - -#include "fts0types.h" - -/******************************************************************//** -Return length of val if it were encoded using our VLC scheme. -FIXME: We will need to be able encode 8 bytes value -@return length of value encoded, in bytes */ -UNIV_INLINE -ulint -fts_get_encoded_len( -/*================*/ - ulint val) /* in: value to encode */ -{ - if (val <= 127) { - return(1); - } else if (val <= 16383) { - return(2); - } else if (val <= 2097151) { - return(3); - } else if (val <= 268435455) { - return(4); - } else { - /* Possibly we should care that on 64-bit machines ulint can - contain values that we can't encode in 5 bytes, but - fts_encode_int doesn't handle them either so it doesn't much - matter. */ - - return(5); - } -} - -/******************************************************************//** -Encode an integer using our VLC scheme and return the length in bytes. -@return length of value encoded, in bytes */ -UNIV_INLINE -ulint -fts_encode_int( -/*===========*/ - ulint val, /* in: value to encode */ - byte* buf) /* in: buffer, must have enough space */ -{ - ulint len; - - if (val <= 127) { - *buf = (byte) val; - - len = 1; - } else if (val <= 16383) { - *buf++ = (byte)(val >> 7); - *buf = (byte)(val & 0x7F); - - len = 2; - } else if (val <= 2097151) { - *buf++ = (byte)(val >> 14); - *buf++ = (byte)((val >> 7) & 0x7F); - *buf = (byte)(val & 0x7F); - - len = 3; - } else if (val <= 268435455) { - *buf++ = (byte)(val >> 21); - *buf++ = (byte)((val >> 14) & 0x7F); - *buf++ = (byte)((val >> 7) & 0x7F); - *buf = (byte)(val & 0x7F); - - len = 4; - } else { - /* Best to keep the limitations of the 32/64 bit versions - identical, at least for the time being. */ - ut_ad(val <= 4294967295u); - - *buf++ = (byte)(val >> 28); - *buf++ = (byte)((val >> 21) & 0x7F); - *buf++ = (byte)((val >> 14) & 0x7F); - *buf++ = (byte)((val >> 7) & 0x7F); - *buf = (byte)(val & 0x7F); - - len = 5; - } - - /* High-bit on means "last byte in the encoded integer". */ - *buf |= 0x80; - - return(len); -} - -/******************************************************************//** -Decode and return the integer that was encoded using our VLC scheme. -@return value decoded */ -UNIV_INLINE -ulint -fts_decode_vlc( -/*===========*/ - byte** ptr) /* in: ptr to decode from, this ptr is - incremented by the number of bytes decoded */ -{ - ulint val = 0; - - for (;;) { - byte b = **ptr; - - ++*ptr; - val |= (b & 0x7F); - - /* High-bit on means "last byte in the encoded integer". */ - if (b & 0x80) { - break; - } else { - val <<= 7; - } - } - - return(val); -} - -#endif diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h index 3d0e48253eb..7b776759a67 100644 --- a/storage/innobase/include/mach0data.h +++ b/storage/innobase/include/mach0data.h @@ -315,6 +315,28 @@ mach_read_from_n_little_endian( const byte* buf, /*!< in: from where to read */ ulint buf_size) /*!< in: from how many bytes to read */ MY_ATTRIBUTE((warn_unused_result)); + + +/** Reads a 64 bit stored in big endian format +@param buf From where to read +@return uint64_t */ +UNIV_INLINE +uint64_t +mach_read_uint64_little_endian(const byte* buf) +{ +#ifdef WORDS_BIGENDIAN + return + uint64_t(buf[0]) | uint64_t(buf[1]) << 8 | + uint64_t(buf[2]) << 16 | uint64_t(buf[3]) << 24 | + uint64_t(buf[4]) << 32 | uint64_t(buf[5]) << 40 | + uint64_t(buf[6]) << 48 | uint64_t(buf[7]) << 56; +#else + uint64_t n; + memcpy(&n, buf, sizeof(uint64_t)); + return n; +#endif +} + /*********************************************************//** Writes a ulint in the little-endian format. */ UNIV_INLINE diff --git a/storage/innobase/include/ut0wqueue.h b/storage/innobase/include/ut0wqueue.h index 5a895f4ea3c..d9cc7aec9c9 100644 --- a/storage/innobase/include/ut0wqueue.h +++ b/storage/innobase/include/ut0wqueue.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -46,6 +46,8 @@ struct ib_wqueue_t ib_mutex_t mutex; /** Work item list */ ib_list_t* items; + /** ib_list_len(*items) */ + size_t length; /** event we use to signal additions to list; os_event_set() and os_event_reset() are protected by the mutex */ os_event_t event; @@ -103,12 +105,5 @@ void* ib_wqueue_nowait( /*=============*/ ib_wqueue_t* wq); /*<! in: work queue */ -/******************************************************************** -Get number of items on queue. -@return number of items on queue */ -ulint -ib_wqueue_len( -/*==========*/ - ib_wqueue_t* wq); /*<! in: work queue */ #endif /* IB_WORK_QUEUE_H */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 0453b61b32c..051b9b89cd5 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1458,23 +1458,6 @@ error_exit: trx->error_state = DB_FTS_INVALID_DOCID; goto error_exit; } - - /* Difference between Doc IDs are restricted within - 4 bytes integer. See fts_get_encoded_len(). Consecutive - doc_ids difference should not exceed - FTS_DOC_ID_MAX_STEP value. */ - - if (doc_id - next_doc_id >= FTS_DOC_ID_MAX_STEP) { - ib::error() << "Doc ID " << doc_id - << " is too big. Its difference with" - " largest used Doc ID " - << next_doc_id - 1 << " cannot" - " exceed or equal to " - << FTS_DOC_ID_MAX_STEP; - err = DB_FTS_INVALID_DOCID; - trx->error_state = DB_FTS_INVALID_DOCID; - goto error_exit; - } } if (table->skip_alter_undo) { diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 770c675c14f..038cf9be825 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1377,14 +1377,6 @@ inline void trx_t::commit_in_memory(const mtr_t *mtr) ut_ad(!rsegs.m_redo.undo); ut_ad(UT_LIST_GET_LEN(lock.evicted_tables) == 0); - if (trx_rseg_t *rseg= rsegs.m_redo.rseg) - { - mutex_enter(&rseg->mutex); - ut_ad(rseg->trx_ref_count > 0); - --rseg->trx_ref_count; - mutex_exit(&rseg->mutex); - } - if (mtr) { if (trx_undo_t *&undo= rsegs.m_noredo.undo) @@ -1438,6 +1430,16 @@ inline void trx_t::commit_in_memory(const mtr_t *mtr) ut_ad(!rsegs.m_noredo.undo); + /* Only after trx_undo_commit_cleanup() it is safe to release + our rseg reference. */ + if (trx_rseg_t *rseg= rsegs.m_redo.rseg) + { + mutex_enter(&rseg->mutex); + ut_ad(rseg->trx_ref_count > 0); + --rseg->trx_ref_count; + mutex_exit(&rseg->mutex); + } + /* Free all savepoints, starting from the first. */ trx_named_savept_t *savep= UT_LIST_GET_FIRST(trx_savepoints); diff --git a/storage/innobase/unittest/CMakeLists.txt b/storage/innobase/unittest/CMakeLists.txt new file mode 100644 index 00000000000..df98cddf73e --- /dev/null +++ b/storage/innobase/unittest/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright (c) 2021, MariaDB Corporation. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA + +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/unittest/mytap + ${CMAKE_SOURCE_DIR}/storage/innobase/include) +ADD_EXECUTABLE(innodb_fts-t innodb_fts-t.cc) +TARGET_LINK_LIBRARIES(innodb_fts-t mysys mytap) +ADD_DEPENDENCIES(innodb_fts-t GenError) +MY_ADD_TEST(innodb_fts) diff --git a/storage/innobase/unittest/innodb_fts-t.cc b/storage/innobase/unittest/innodb_fts-t.cc new file mode 100644 index 00000000000..4e10c0e439e --- /dev/null +++ b/storage/innobase/unittest/innodb_fts-t.cc @@ -0,0 +1,52 @@ +#include "tap.h" +#include "fts0fts.h" +#include "fts0vlc.h" + +struct fts_encode_info +{ + const byte buf[10]; + size_t len; + doc_id_t val; +}; + +/* Contains fts encoding min & max value for each length bytes */ +static const fts_encode_info fts_info[]= +{ + {{0x80}, 1, 0}, + {{0xFF}, 1, (1 << 7) - 1}, + {{0x01, 0x80}, 2, 1 << 7}, + {{0x7F, 0XFF}, 2, (1 << 14) - 1}, + {{0x01, 0x00, 0x80}, 3, 1 << 14}, + {{0x7F, 0X7F, 0XFF}, 3, (1 << 21) - 1}, + {{0x01, 0x00, 0x00, 0x80}, 4, 1 << 21}, + {{0x7F, 0X7F, 0X7F, 0xFF}, 4, (1 << 28) - 1}, + {{0x01, 0x00, 0x00, 0x00, 0x80}, 5, 1 << 28}, + {{0x7F, 0X7F, 0X7F, 0x7F, 0xFF}, 5, (1ULL << 35) - 1}, + {{0x01, 0x00, 0x00, 0x00, 0x00, 0x80}, 6, 1ULL << 35}, + {{0x7F, 0X7F, 0X7F, 0x7F, 0x7F, 0xFF}, 6, (1ULL << 42) - 1}, + {{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, 7, 1ULL << 42}, + {{0x7F, 0X7F, 0X7F, 0x7F, 0x7F, 0x7F, 0XFF}, 7, (1ULL << 49) - 1}, + {{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, 8, 1ULL << 49}, + {{0x7F, 0X7F, 0X7F, 0x7F, 0x7F, 0x7F, 0X7F, 0XFF}, 8, (1ULL << 56) -1}, + {{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, 9, 1ULL << 56}, + {{0x7F, 0X7F, 0X7F, 0x7F, 0x7F, 0x7F, 0X7F, 0x7F, 0XFF}, 9, (1ULL << 63) -1}, + {{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}, 10, 1ULL << 63}, + {{0x01, 0X7F, 0X7F, 0x7F, 0x7F, 0x7F, 0X7F, 0x7F, 0x7F, 0xFF}, 10, ~0ULL} +}; + +int main(int, char**) +{ + for (int i= array_elements(fts_info); i--;) + { + byte buf[10]; + const byte* fts_buf= buf; + size_t len= fts_encode_int(fts_info[i].val, buf) - &buf[0]; + if (fts_info[i].len == len && + !memcmp(&fts_info[i].buf, buf, len) && + fts_decode_vlc(&fts_buf) == fts_info[i].val && + fts_buf == &buf[len]) + ok(true, "FTS Encoded for %d bytes", fts_info[i].len); + else + ok(false, "FTS Encoded for %d bytes", fts_info[i].len); + } +} diff --git a/storage/innobase/ut/ut0wqueue.cc b/storage/innobase/ut/ut0wqueue.cc index ae97009430e..45af449cff9 100644 --- a/storage/innobase/ut/ut0wqueue.cc +++ b/storage/innobase/ut/ut0wqueue.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, MariaDB Corporation. +Copyright (c) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,6 +45,7 @@ ib_wqueue_create(void) wq->items = ib_list_create(); wq->event = os_event_create(0); + wq->length = 0; return(wq); } @@ -76,6 +77,8 @@ ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap, bool wq_locked) } ib_list_add_last(wq->items, item, heap); + wq->length++; + ut_ad(wq->length == ib_list_len(wq->items)); os_event_set(wq->event); if (!wq_locked) { @@ -102,12 +105,12 @@ ib_wqueue_wait( if (node) { ib_list_remove(wq->items, node); - - if (!ib_list_get_first(wq->items)) { + if (!--wq->length) { /* We must reset the event when the list gets emptied. */ os_event_reset(wq->event); } + ut_ad(wq->length == ib_list_len(wq->items)); break; } @@ -142,7 +145,8 @@ ib_wqueue_timedwait( if (node) { ib_list_remove(wq->items, node); - + wq->length--; + ut_ad(wq->length == ib_list_len(wq->items)); mutex_exit(&wq->mutex); break; } @@ -204,20 +208,3 @@ bool ib_wqueue_is_empty(ib_wqueue_t* wq) mutex_exit(&wq->mutex); return is_empty; } - -/******************************************************************** -Get number of items on queue. -@return number of items on queue */ -ulint -ib_wqueue_len( -/*==========*/ - ib_wqueue_t* wq) /*<! in: work queue */ -{ - ulint len = 0; - - mutex_enter(&wq->mutex); - len = ib_list_len(wq->items); - mutex_exit(&wq->mutex); - - return(len); -} |