diff options
author | Marko Mäkelä <marko.makela@mariadb.com> | 2023-03-06 17:17:32 +0200 |
---|---|---|
committer | Marko Mäkelä <marko.makela@mariadb.com> | 2023-03-06 17:17:32 +0200 |
commit | b1646d0433c98662c50af029a121d681ddfb7a2b (patch) | |
tree | e0dc08823d0138e9258f7c21a0b038153a0b65f8 | |
parent | 948fb3c27d1e0754d99a96389aaf1cf85370d839 (diff) | |
download | mariadb-git-b1646d0433c98662c50af029a121d681ddfb7a2b.tar.gz |
MDEV-30567 rec_get_offsets() is not optimal
rec_init_offsets_comp_ordinary(), rec_init_offsets(),
rec_get_offsets_reverse(), rec_get_nth_field_offs_old():
Simplify some bitwise arithmetics to avoid conditional jumps,
and add branch prediction hints with the assumption that most
variable-length columns are short.
Tested by: Matthias Leich
-rw-r--r-- | storage/innobase/rem/rem0rec.cc | 208 |
1 files changed, 90 insertions, 118 deletions
diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 902f3f2d5ca..d54dc57655e 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -217,14 +217,12 @@ rec_get_n_extern_new( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (DATA_BIG_COL(col)) { - if (len & 0x80) { - /* 1exxxxxxx xxxxxxxx */ - if (len & 0x40) { - n_extern++; - } - lens--; + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) { + /* 1exxxxxxx xxxxxxxx */ + if (len & 0x40) { + n_extern++; } + lens--; } } } while (++i < n); @@ -244,6 +242,10 @@ enum rec_leaf_format { REC_LEAF_INSTANT }; +#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 11 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wconversion" /* GCC 5 to 10 need this */ +#endif /** Determine the offset to each field in a leaf-page record in ROW_FORMAT=COMPACT,DYNAMIC,COMPRESSED. This is a special case of rec_init_offsets() and rec_get_offsets_func(). @@ -361,8 +363,7 @@ start: do { if (mblob) { if (i == index->first_user_field()) { - offs = static_cast<rec_offs>(offs - + FIELD_REF_SIZE); + offs += FIELD_REF_SIZE; len = combine(offs, STORED_OFFPAGE); any |= REC_OFFS_EXTERNAL; field--; @@ -433,27 +434,23 @@ start: stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if ((len & 0x80) && DATA_BIG_COL(col)) { + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) { /* 1exxxxxxx xxxxxxxx */ - len = static_cast<rec_offs>(len << 8 - | *lens--); - offs = static_cast<rec_offs>(offs - + get_value(len)); - if (UNIV_UNLIKELY(len & 0x4000)) { - ut_ad(index->is_primary()); - any |= REC_OFFS_EXTERNAL; - len = combine(offs, STORED_OFFPAGE); - } else { - len = offs; - } - + len <<= 8; + len |= *lens--; + static_assert(STORED_OFFPAGE == 0x4000, ""); + static_assert(REC_OFFS_EXTERNAL == 0x4000, ""); + const rec_offs ext = len & REC_OFFS_EXTERNAL; + offs += get_value(len); + len = offs | ext; + any |= ext; + ut_ad(!ext || index->is_primary()); continue; } - len = offs = static_cast<rec_offs>(offs + len); + len = offs += len; } else { - len = offs = static_cast<rec_offs>(offs - + field->fixed_len); + len = offs += field->fixed_len; } } while (field++, rec_offs_base(offsets)[++i] = len, i < rec_offs_n_fields(offsets)); @@ -478,7 +475,7 @@ rec_offs_make_valid( { const bool is_alter_metadata = leaf && rec_is_alter_metadata(rec, *index); - ut_ad(is_alter_metadata + ut_ad((leaf && rec_is_metadata(rec, *index)) || index->is_dummy || index->is_ibuf() || (leaf ? rec_offs_n_fields(offsets) @@ -572,7 +569,8 @@ rec_offs_validate( } /* index->n_def == 0 for dummy indexes if !comp */ ut_ad(!comp || index->n_def); - ut_ad(!index->n_def || i <= max_n_fields); + ut_ad(!index->n_def || i <= max_n_fields + || rec_is_metadata(rec, *index)); } while (i--) { ulint curr = get_value(rec_offs_base(offsets)[1 + i]); @@ -610,7 +608,7 @@ rec_init_offsets( ulint i = 0; rec_offs offs; - /* This assertion was relaxed for the btr_cur_open_at_index_side() + /* This assertion was relaxed for the btr_cur_t::open_leaf() call in btr_cur_instant_init_low(). We cannot invoke index->is_instant(), because the same assertion would fail there until btr_cur_instant_init_low() has invoked @@ -678,8 +676,7 @@ rec_init_offsets( do { rec_offs len; if (UNIV_UNLIKELY(i == n_node_ptr_field)) { - len = offs = static_cast<rec_offs>( - offs + REC_NODE_PTR_SIZE); + len = offs += REC_NODE_PTR_SIZE; goto resolved; } @@ -719,29 +716,25 @@ rec_init_offsets( encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (DATA_BIG_COL(col)) { - if (len & 0x80) { - /* 1exxxxxxx xxxxxxxx */ - len = static_cast<rec_offs>( - len << 8 | *lens--); - - /* B-tree node pointers - must not contain externally - stored columns. Thus - the "e" flag must be 0. */ - ut_a(!(len & 0x4000)); - offs = static_cast<rec_offs>( - offs + get_value(len)); - len = offs; - - goto resolved; - } + if (UNIV_UNLIKELY(len & 0x80) + && DATA_BIG_COL(col)) { + /* 1exxxxxxx xxxxxxxx */ + len <<= 8; + len |= *lens--; + + /* B-tree node pointers + must not contain externally + stored columns. Thus + the "e" flag must be 0. */ + ut_a(!(len & 0x4000)); + offs += len & 0x3fff; + len = offs; + goto resolved; } - len = offs = static_cast<rec_offs>(offs + len); + len = offs += len; } else { - len = offs = static_cast<rec_offs>( - offs + field->fixed_len); + len = offs += field->fixed_len; } resolved: rec_offs_base(offsets)[i + 1] = len; @@ -758,35 +751,30 @@ resolved: rec_offs any; if (rec_get_1byte_offs_flag(rec)) { - offs = static_cast<rec_offs>(offs + n_fields); + offs += static_cast<rec_offs>(n_fields); any = offs; /* Determine offsets to fields */ do { offs = rec_1_get_field_end_info(rec, i); if (offs & REC_1BYTE_SQL_NULL_MASK) { - offs &= static_cast<rec_offs>( - ~REC_1BYTE_SQL_NULL_MASK); - set_type(offs, SQL_NULL); + offs ^= REC_1BYTE_SQL_NULL_MASK + | SQL_NULL; } rec_offs_base(offsets)[1 + i] = offs; } while (++i < n); } else { - offs = static_cast<rec_offs>(offs + 2 * n_fields); + offs += static_cast<rec_offs>(2 * n_fields); any = offs; /* Determine offsets to fields */ do { offs = rec_2_get_field_end_info(rec, i); - if (offs & REC_2BYTE_SQL_NULL_MASK) { - offs &= static_cast<rec_offs>( - ~REC_2BYTE_SQL_NULL_MASK); - set_type(offs, SQL_NULL); - } - if (offs & REC_2BYTE_EXTERN_MASK) { - offs &= static_cast<rec_offs>( - ~REC_2BYTE_EXTERN_MASK); - set_type(offs, STORED_OFFPAGE); - any |= REC_OFFS_EXTERNAL; - } + static_assert(REC_2BYTE_SQL_NULL_MASK + == SQL_NULL, ""); + static_assert(REC_2BYTE_EXTERN_MASK + == STORED_OFFPAGE, ""); + static_assert(REC_OFFS_EXTERNAL + == STORED_OFFPAGE, ""); + any |= (offs & REC_OFFS_EXTERNAL); rec_offs_base(offsets)[1 + i] = offs; } while (++i < n); } @@ -838,7 +826,7 @@ rec_get_offsets_func( bool alter_metadata = false; ut_ad(index->n_core_fields >= n_core); - /* This assertion was relaxed for the btr_cur_open_at_index_side() + /* This assertion was relaxed for the btr_cur_t::open_leaf() call in btr_cur_instant_init_low(). We cannot invoke index->is_instant(), because the same assertion would fail there until btr_cur_instant_init_low() has invoked @@ -863,19 +851,19 @@ rec_get_offsets_func( ut_ad(!n_core); n = dict_index_get_n_unique_in_tree_nonleaf(index) + 1; break; + default: + ut_ad("corrupted record header" == 0); + /* fall through */ case REC_STATUS_INFIMUM: case REC_STATUS_SUPREMUM: /* infimum or supremum record */ ut_ad(rec_get_heap_no_new(rec) == ulint(rec_get_status(rec) - == REC_STATUS_INFIMUM - ? PAGE_HEAP_NO_INFIMUM - : PAGE_HEAP_NO_SUPREMUM)); + == REC_STATUS_INFIMUM + ? PAGE_HEAP_NO_INFIMUM + : PAGE_HEAP_NO_SUPREMUM)); n = 1; break; - default: - ut_error; - return(NULL); } } else { n = rec_get_n_fields_old(rec); @@ -897,9 +885,7 @@ rec_get_offsets_func( ut_ad(!is_user_rec || !n_core || index->is_dummy || dict_index_is_ibuf(index) || n == n_fields /* btr_pcur_restore_position() */ - || (n + (index->id == DICT_INDEXES_ID) - >= n_core && n <= index->n_fields - + unsigned(rec_is_alter_metadata(rec, false)))); + || (n + (index->id == DICT_INDEXES_ID) >= n_core)); if (is_user_rec && n_core && n < index->n_fields) { ut_ad(!index->is_dummy); @@ -1000,8 +986,7 @@ rec_get_offsets_reverse( do { rec_offs len; if (UNIV_UNLIKELY(i == n_node_ptr_field)) { - len = offs = static_cast<rec_offs>( - offs + REC_NODE_PTR_SIZE); + len = offs += REC_NODE_PTR_SIZE; goto resolved; } @@ -1038,30 +1023,23 @@ rec_get_offsets_reverse( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (DATA_BIG_COL(col)) { - if (len & 0x80) { - /* 1exxxxxxx xxxxxxxx */ - len = static_cast<rec_offs>( - len << 8 | *lens++); - - offs = static_cast<rec_offs>( - offs + get_value(len)); - if (UNIV_UNLIKELY(len & 0x4000)) { - any_ext = REC_OFFS_EXTERNAL; - len = combine(offs, - STORED_OFFPAGE); - } else { - len = offs; - } - - goto resolved; - } + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) { + /* 1exxxxxxx xxxxxxxx */ + len &= 0x7f; + len <<= 8; + len |= *lens++; + static_assert(STORED_OFFPAGE == 0x4000, ""); + static_assert(REC_OFFS_EXTERNAL == 0x4000, ""); + rec_offs ext = len & REC_OFFS_EXTERNAL; + offs += get_value(len); + len = offs | ext; + any_ext |= ext; + goto resolved; } - len = offs = static_cast<rec_offs>(offs + len); + len = offs += len; } else { - len = offs = static_cast<rec_offs>(offs - + field->fixed_len); + len = offs += field->fixed_len; } resolved: rec_offs_base(offsets)[i + 1] = len; @@ -1101,7 +1079,7 @@ rec_get_nth_field_offs_old( return(os); } - next_os = next_os & ~REC_1BYTE_SQL_NULL_MASK; + next_os &= ~REC_1BYTE_SQL_NULL_MASK; } else { os = rec_2_get_field_start_offs(rec, n); @@ -1113,8 +1091,7 @@ rec_get_nth_field_offs_old( return(os); } - next_os = next_os & ~(REC_2BYTE_SQL_NULL_MASK - | REC_2BYTE_EXTERN_MASK); + next_os &= ~(REC_2BYTE_SQL_NULL_MASK | REC_2BYTE_EXTERN_MASK); } *len = next_os - os; @@ -1267,7 +1244,8 @@ rec_get_converted_size_comp_prefix_low( } else if (dfield_is_ext(dfield)) { ut_ad(DATA_BIG_COL(field->col)); extra_size += 2; - } else if (len < 128 || !DATA_BIG_COL(field->col)) { + } else if (UNIV_LIKELY(len < 128) + || !DATA_BIG_COL(field->col)) { extra_size++; } else { /* For variable-length columns, we look up the @@ -1618,14 +1596,7 @@ start: /* set the null flag if necessary */ if (dfield_is_null(field)) { -#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 6 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wconversion" /* GCC 5 may need this here */ -#endif *nulls |= static_cast<byte>(null_mask); -#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 6 -# pragma GCC diagnostic pop -#endif null_mask <<= 1; continue; } @@ -1734,6 +1705,9 @@ rec_convert_dtuple_to_rec_new( REC_INFO_BITS_MASK, REC_INFO_BITS_SHIFT); return buf; } +#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 11 +# pragma GCC diagnostic pop /* ignored "-Wconversion" */ +#endif /*********************************************************//** Builds a physical record out of a data tuple and @@ -2096,14 +2070,12 @@ rec_copy_prefix_to_buf( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the column is stored externally. */ - if (DATA_BIG_COL(col)) { - if (len & 0x80) { - /* 1exxxxxx */ - len &= 0x3f; - len <<= 8; - len |= *lens--; - UNIV_PREFETCH_R(lens); - } + if (UNIV_UNLIKELY(len & 0x80) && DATA_BIG_COL(col)) { + /* 1exxxxxx */ + len &= 0x3f; + len <<= 8; + len |= *lens--; + UNIV_PREFETCH_R(lens); } prefix_len += len; } |