From 86167e908fe5de6f6e9f5076b4ea8041514d0820 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 15 Nov 2019 23:37:28 +0300 Subject: MDEV-20611: MRR scan over partitioned InnoDB table produces "Out of memory" error Fix partitioning and DS-MRR to work together - In ha_partition::index_end(): take into account that ha_innobase (and other engines using DS-MRR) will have inited=RND when initialized for DS-MRR scan. - In ha_partition::multi_range_read_next(): if the MRR scan is using HA_MRR_NO_ASSOCIATION mode, it is not guaranteed that the partition's handler will store anything into *range_info. - In DsMrr_impl::choose_mrr_impl(): ha_partition will inquire partitions about how much memory their MRR implementation needs by passing *buffer_size=0. DS-MRR code didn't know about this (actually it used uint for buffer size calculation and would have an under-flow). Returning *buffer_size=0 made ha_partition assume that partitions do not need MRR memory and pass the same buffer to each of them. Now, this is fixed. If DS-MRR gets *buffer_size=0, it will return the amount of buffer space needed, but not more than about @@mrr_buffer_size. * Fix ha_{innobase,maria,myisam}::clone. If ha_partition uses MRR on its partitions, and partition use DS-MRR, the code will call handler->clone with TABLE (*NOT partition*) name as an argument. DS-MRR has no way of knowing the partition name, so the solution was to have the ::clone() function for the affected storage engine to ignore the name argument and get it elsewhere. --- sql/multi_range_read.cc | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) (limited to 'sql/multi_range_read.cc') diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index f40c8d0fbd8..6d62ea07dfa 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -1589,11 +1589,10 @@ bool DsMrr_impl::choose_mrr_impl(uint keyno, ha_rows rows, uint *flags, } uint add_len= share->key_info[keyno].key_length + primary_file->ref_length; - *bufsz -= add_len; - if (get_disk_sweep_mrr_cost(keyno, rows, *flags, bufsz, &dsmrr_cost)) + if (get_disk_sweep_mrr_cost(keyno, rows, *flags, bufsz, add_len, + &dsmrr_cost)) return TRUE; - *bufsz += add_len; - + bool force_dsmrr; /* If mrr_cost_based flag is not set, then set cost of DS-MRR to be minimum of @@ -1682,6 +1681,11 @@ static void get_sort_and_sweep_cost(TABLE *table, ha_rows nrows, Cost_estimate * @param rows E(Number of rows to be scanned) @param flags Scan parameters (HA_MRR_* flags) @param buffer_size INOUT Buffer size + IN: Buffer of size 0 means the function + will determine the best size and return it. + @param extra_mem_overhead Extra memory overhead of the MRR implementation + (the function assumes this many bytes of buffer + space will not be usable by DS-MRR) @param cost OUT The cost @retval FALSE OK @@ -1690,7 +1694,9 @@ static void get_sort_and_sweep_cost(TABLE *table, ha_rows nrows, Cost_estimate * */ bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags, - uint *buffer_size, Cost_estimate *cost) + uint *buffer_size, + uint extra_mem_overhead, + Cost_estimate *cost) { ulong max_buff_entries, elem_size; ha_rows rows_in_full_step; @@ -1700,11 +1706,24 @@ bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags, elem_size= primary_file->ref_length + sizeof(void*) * (!MY_TEST(flags & HA_MRR_NO_ASSOCIATION)); - max_buff_entries = *buffer_size / elem_size; - if (!max_buff_entries) + if (!*buffer_size) + { + /* + We are requested to determine how much memory we need. + Request memory to finish the scan in one pass but do not request + more than @@mrr_buff_size. + */ + *buffer_size = MY_MIN(extra_mem_overhead + rows*elem_size, + MY_MAX(table->in_use->variables.mrr_buff_size, + extra_mem_overhead)); + } + + if (elem_size + extra_mem_overhead > *buffer_size) return TRUE; /* Buffer has not enough space for even 1 rowid */ + max_buff_entries = (*buffer_size - extra_mem_overhead) / elem_size; + /* Number of iterations we'll make with full buffer */ n_full_steps= (uint)floor(rows2double(rows) / max_buff_entries); -- cgit v1.2.1 From 409ed60bb844252d46127c5df37d23fecf52159f Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 16 Nov 2019 13:18:24 +0300 Subject: Fix compile failure on Windows: use explicit type casts --- sql/multi_range_read.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sql/multi_range_read.cc') diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc index 6d62ea07dfa..5e0afd6edb2 100644 --- a/sql/multi_range_read.cc +++ b/sql/multi_range_read.cc @@ -1714,9 +1714,9 @@ bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags, Request memory to finish the scan in one pass but do not request more than @@mrr_buff_size. */ - *buffer_size = MY_MIN(extra_mem_overhead + rows*elem_size, - MY_MAX(table->in_use->variables.mrr_buff_size, - extra_mem_overhead)); + *buffer_size= (uint) MY_MIN(extra_mem_overhead + elem_size*(ulong)rows, + MY_MAX(table->in_use->variables.mrr_buff_size, + extra_mem_overhead)); } if (elem_size + extra_mem_overhead > *buffer_size) -- cgit v1.2.1