diff options
author | Varun Gupta <varun.gupta@mariadb.com> | 2019-11-28 20:55:26 +0530 |
---|---|---|
committer | Varun Gupta <varun.gupta@mariadb.com> | 2019-11-29 20:46:46 +0530 |
commit | 93fa99cb36ff2f907fed3653bab55c40c31d683e (patch) | |
tree | d4e33183164557eeca5d635ea280c372c434095b | |
parent | 54af2cc351c6133b398eed28a8ec7d3a06e3cf5d (diff) | |
download | mariadb-git-bb-mdev6915.tar.gz |
Refactoringbb-mdev6915
-rw-r--r-- | include/myisamchk.h | 2 | ||||
-rw-r--r-- | sql/filesort.cc | 294 | ||||
-rw-r--r-- | sql/filesort.h | 7 | ||||
-rw-r--r-- | sql/filesort_utils.h | 9 | ||||
-rw-r--r-- | sql/records.cc | 4 | ||||
-rw-r--r-- | sql/sql_sort.h | 139 | ||||
-rw-r--r-- | sql/uniques.cc | 63 |
7 files changed, 313 insertions, 205 deletions
diff --git a/include/myisamchk.h b/include/myisamchk.h index a0b58a50c52..f1a7348eea0 100644 --- a/include/myisamchk.h +++ b/include/myisamchk.h @@ -121,7 +121,7 @@ typedef struct st_handler_check_param typedef struct st_buffpek { my_off_t file_pos; /* Where we are in the sort file */ - uchar *base, *key, *end; /* Key pointers */ + uchar *base, *key; /* Key pointers */ ha_rows count; /* Number of rows in table */ ha_rows mem_count; /* Numbers of keys in memory */ ha_rows max_keys; /* Max keys in buffert */ diff --git a/sql/filesort.cc b/sql/filesort.cc index 4abd3b81360..98431a433b5 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -57,7 +57,6 @@ static uint suffix_length(ulong string_length); static uint sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, bool *multi_byte_charset); static Addon_fields *get_addon_fields(TABLE *table, uint sortlength, - LEX_STRING *addon_buf, uint *addon_length, uint *m_packable_length); @@ -68,7 +67,7 @@ static bool check_if_pq_applicable(Sort_param *param, SORT_INFO *info, void Sort_param::init_for_filesort(uint sortlen, TABLE *table, ha_rows maxrows, bool sort_positions) { - DBUG_ASSERT(addon_fields == 0 && addon_buf.length == 0); + DBUG_ASSERT(addon_fields == NULL); sort_length= sortlen; ref_length= table->file->ref_length; @@ -79,8 +78,8 @@ void Sort_param::init_for_filesort(uint sortlen, TABLE *table, Get the descriptors of all fields whose values are appended to sorted fields and get its total length in addon_buf.length */ - addon_fields= get_addon_fields(table, sort_length, &addon_buf, - &addon_length, &m_packable_length); + addon_fields= get_addon_fields(table, sort_length, &addon_length, + &m_packable_length); } if (using_addon_fields()) { @@ -169,7 +168,7 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, DBUG_ASSERT(thd->variables.sortbuff_size <= SIZE_T_MAX); size_t memory_available= (size_t)thd->variables.sortbuff_size; uint maxbuffer; - BUFFPEK *buffpek; + Merge_chunk *buffpek; ha_rows num_rows= HA_POS_ERROR; IO_CACHE tempfile, buffpek_pointers, *outfile; Sort_param param; @@ -199,7 +198,7 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, if (!(sort= new SORT_INFO)) return 0; - if (subselect && subselect->filesort_buffer.sort_buffer_size() > 0) + if (subselect && subselect->filesort_buffer.is_allocated()) { // Reuse cache from last call sort->filesort_buffer= subselect->filesort_buffer; @@ -208,6 +207,9 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, subselect->sortbuffer.str=0; } + DBUG_ASSERT(sort->sorted_result_in_fsbuf == FALSE || + sort->record_pointers == NULL); + outfile= &sort->io_cache; my_b_clear(&tempfile); @@ -220,7 +222,6 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, &multi_byte_charset), table, max_rows, filesort->sort_positions); - sort->addon_buf= param.addon_buf; sort->addon_fields= param.addon_fields; if (multi_byte_charset && @@ -278,7 +279,7 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, size_t min_sort_memory= MY_MAX(MIN_SORT_MEMORY, param.sort_length*MERGEBUFF2); - set_if_bigger(min_sort_memory, sizeof(BUFFPEK*)*MERGEBUFF2); + set_if_bigger(min_sort_memory, sizeof(Merge_chunk*)*MERGEBUFF2); while (memory_available >= min_sort_memory) { ulonglong keys= memory_available / (param.rec_length + sizeof(char*)); @@ -305,7 +306,9 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, goto err; param.sort_form= table; - param.end=(param.local_sortorder=filesort->sortorder)+s_length; + param.local_sortorder= + Bounds_checked_array<SORT_FIELD>(filesort->sortorder, s_length); + num_rows= find_all_keys(thd, ¶m, select, sort, &buffpek_pointers, @@ -346,12 +349,20 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, my_free(sort->buffpek.str); sort->buffpek.str= 0; } + + if (param.using_addon_fields()) + { + DBUG_ASSERT(sort->addon_fields); + if (!sort->addon_fields->allocate_addon_buf(param.addon_length)) + goto err; + } + if (!(sort->buffpek.str= (char *) read_buffpek_from_file(&buffpek_pointers, maxbuffer, (uchar*) sort->buffpek.str))) goto err; sort->buffpek.length= maxbuffer; - buffpek= (BUFFPEK *) sort->buffpek.str; + buffpek= (Merge_chunk *) sort->buffpek.str; close_cached_file(&buffpek_pointers); /* Open cached file if it isn't open */ if (! my_b_inited(outfile) && @@ -365,11 +376,10 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, Use also the space previously used by string pointers in sort_buffer for temporary key storage. */ - param.max_keys_per_buffer=((param.max_keys_per_buffer * - (param.rec_length + sizeof(char*))) / - param.rec_length - 1); + param.max_keys_per_buffer= sort->sort_buffer_size() / param.rec_length;; maxbuffer--; // Offset from 0 - if (merge_many_buff(¶m, sort->get_raw_buf(), sort->sort_buffer_size(), + + if (merge_many_buff(¶m, sort->get_raw_buf(), buffpek,&maxbuffer, &tempfile)) goto err; @@ -378,7 +388,6 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, goto err; if (merge_index(¶m, sort->get_raw_buf(), - sort->sort_buffer_size(), buffpek, maxbuffer, &tempfile, @@ -549,10 +558,10 @@ uint Filesort::make_sortorder(THD *thd, JOIN *join, table_map first_table_bit) static uchar *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count, uchar *buf) { - size_t length= sizeof(BUFFPEK)*count; + size_t length= sizeof(Merge_chunk)*count; uchar *tmp= buf; DBUG_ENTER("read_buffpek_from_file"); - if (count > UINT_MAX/sizeof(BUFFPEK)) + if (count > UINT_MAX/sizeof(Merge_chunk)) return 0; /* sizeof(BUFFPEK)*count will overflow */ if (!tmp) tmp= (uchar *)my_malloc(length, MYF(MY_WME | MY_THREAD_SPECIFIC)); @@ -968,7 +977,7 @@ write_keys(Sort_param *param, SORT_INFO *fs_info, uint count, IO_CACHE *buffpek_pointers, IO_CACHE *tempfile) { size_t rec_length; - BUFFPEK buffpek; + Merge_chunk buffpek; DBUG_ENTER("write_keys"); rec_length= param->rec_length; @@ -980,14 +989,14 @@ write_keys(Sort_param *param, SORT_INFO *fs_info, uint count, MYF(MY_WME))) DBUG_RETURN(1); /* purecov: inspected */ /* check we won't have more buffpeks than we can possibly keep in memory */ - if (my_b_tell(buffpek_pointers) + sizeof(BUFFPEK) > (ulonglong)UINT_MAX) + if (my_b_tell(buffpek_pointers) + sizeof(Merge_chunk) > (ulonglong)UINT_MAX) DBUG_RETURN(1); bzero(&buffpek, sizeof(buffpek)); - buffpek.file_pos= my_b_tell(tempfile); + buffpek.set_file_position(my_b_tell(tempfile)); if ((ha_rows) count > param->max_rows) count=(uint) param->max_rows; /* purecov: inspected */ - buffpek.count=(ha_rows) count; + buffpek.set_rowcount(static_cast<ha_rows>(count)); const bool packed_addon_fields= param->using_packed_addons(); for (uint ix= 0; ix < count; ++ix) @@ -1254,8 +1263,8 @@ static uint make_sortkey(Sort_param *param, uchar *to, uchar *ref_pos) uint length; uchar *orig_to= to; - for (sort_field=param->local_sortorder ; - sort_field != param->end ; + for (sort_field=param->local_sortorder.begin() ; + sort_field != param->local_sortorder.end() ; sort_field++) { bool maybe_null=0; @@ -1348,8 +1357,8 @@ static void register_used_fields(Sort_param *param) SORT_FIELD *sort_field; TABLE *table=param->sort_form; - for (sort_field= param->local_sortorder ; - sort_field != param->end ; + for (sort_field= param->local_sortorder.begin() ; + sort_field != param->local_sortorder.end() ; sort_field++) { Field *field; @@ -1554,12 +1563,12 @@ static bool check_if_pq_applicable(Sort_param *param, /** Merge buffers to make < MERGEBUFF2 buffers. */ -int merge_many_buff(Sort_param *param, uchar *sort_buffer, size_t buff_size, - BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file) +int merge_many_buff(Sort_param *param, Sort_buffer sort_buffer, + Merge_chunk *buffpek, uint *maxbuffer, IO_CACHE *t_file) { uint i; IO_CACHE t_file2,*from_file,*to_file,*temp; - BUFFPEK *lastbuff; + Merge_chunk *lastbuff; DBUG_ENTER("merge_many_buff"); if (*maxbuffer < MERGEBUFF2) @@ -1579,11 +1588,11 @@ int merge_many_buff(Sort_param *param, uchar *sort_buffer, size_t buff_size, lastbuff=buffpek; for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) { - if (merge_buffers(param,from_file,to_file,sort_buffer, buff_size, lastbuff++, + if (merge_buffers(param,from_file,to_file,sort_buffer, lastbuff++, buffpek+i,buffpek+i+MERGEBUFF-1,0)) goto cleanup; } - if (merge_buffers(param,from_file,to_file,sort_buffer, buff_size, lastbuff++, + if (merge_buffers(param,from_file,to_file,sort_buffer, lastbuff++, buffpek+i,buffpek+ *maxbuffer,0)) break; /* purecov: inspected */ if (flush_io_cache(to_file)) @@ -1609,27 +1618,27 @@ cleanup: (ulong)-1 if something goes wrong */ -ulong read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, +ulong read_to_buffer(IO_CACHE *fromfile, Merge_chunk *buffpek, Sort_param *param) { ha_rows count; uint rec_length= param->rec_length; - if ((count= MY_MIN((ha_rows) buffpek->max_keys,buffpek->count))) + if ((count= MY_MIN(buffpek->max_keys(),buffpek->rowcount()))) { size_t bytes_to_read; if (param->using_packed_addons()) { - count= buffpek->count; - bytes_to_read= MY_MIN(static_cast<size_t>(buffpek->end - buffpek->base), + count= buffpek->rowcount(); + bytes_to_read= MY_MIN(buffpek->buffer_size(), static_cast<size_t>(fromfile->end_of_file - - buffpek->file_pos)); + buffpek->file_position())); } else bytes_to_read= rec_length * static_cast<size_t>(count); - if (unlikely(my_b_pread(fromfile, (uchar*) buffpek->base, bytes_to_read, - buffpek->file_pos))) + if (unlikely(my_b_pread(fromfile, buffpek->buffer_start(), + bytes_to_read, buffpek->file_position()))) return ((ulong) -1); size_t num_bytes_read; @@ -1640,16 +1649,16 @@ ulong read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, We need to loop through all the records, reading the length fields, and then "chop off" the final incomplete record. */ - uchar *record= buffpek->base; + uchar *record= buffpek->buffer_start(); uint ix= 0; for (; ix < count; ++ix) { if (record + param->sort_length + Addon_fields::size_of_length_field > - buffpek->end) + buffpek->buffer_end()) break; // Incomplete record. uchar *plen= record + param->sort_length; uint res_length= Addon_fields::read_addon_length(plen); - if (plen + res_length > buffpek->end) + if (plen + res_length > buffpek->buffer_end()) break; // Incomplete record. DBUG_ASSERT(res_length > 0); record+= param->sort_length; @@ -1657,17 +1666,17 @@ ulong read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, } DBUG_ASSERT(ix > 0); count= ix; - num_bytes_read= record - buffpek->base; + num_bytes_read= record - buffpek->buffer_start(); DBUG_PRINT("info", ("read %llu bytes of complete records", static_cast<ulonglong>(bytes_to_read))); } else num_bytes_read= bytes_to_read; - buffpek->key=buffpek->base; - buffpek->file_pos+= num_bytes_read; /* New filepos */ - buffpek->count-= count; - buffpek->mem_count= count; + buffpek->init_current_key(); + buffpek->advance_file_position(num_bytes_read); /* New filepos */ + buffpek->decrement_rowcount(count); + buffpek->set_mem_count(count); return (ulong) num_bytes_read; } return 0; @@ -1685,25 +1694,15 @@ ulong read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, @param[in] key_length key length */ -void reuse_freed_buff(QUEUE *queue, BUFFPEK *reuse, uint key_length) +void reuse_freed_buff(QUEUE *queue, Merge_chunk *reuse, uint key_length) { for (uint i= queue_first_element(queue); i <= queue_last_element(queue); i++) { - BUFFPEK *bp= (BUFFPEK *) queue_element(queue, i); - if (bp->end == reuse->base) - { - bp->end = reuse->end; - bp->max_keys+= reuse->max_keys; - return; - } - else if (bp->base == reuse->end) - { - bp->base= reuse->base; - bp->max_keys+= reuse->max_keys; + Merge_chunk *bp= (Merge_chunk *) queue_element(queue, i); + if (reuse->merge_freed_buff(bp)) return; - } } DBUG_ASSERT(0); } @@ -1728,8 +1727,8 @@ void reuse_freed_buff(QUEUE *queue, BUFFPEK *reuse, uint key_length) */ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, - IO_CACHE *to_file, uchar *sort_buffer, size_t buff_size, - BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb, + IO_CACHE *to_file, Sort_buffer sort_buffer, + Merge_chunk *lastbuff, Merge_chunk *Fb, Merge_chunk *Tb, int flag) { bool error= 0; @@ -1739,7 +1738,7 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, ha_rows max_rows,org_max_rows; my_off_t to_start_filepos; uchar *strpos; - BUFFPEK *buffpek; + Merge_chunk *buffpek; QUEUE queue; qsort2_cmp cmp; void *first_cmp_arg; @@ -1765,7 +1764,7 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, uint wr_offset= flag ? offset : 0; maxcount= (ulong) (param->max_keys_per_buffer/((uint) (Tb-Fb) +1)); to_start_filepos= my_b_tell(to_file); - strpos= sort_buffer; + strpos= sort_buffer.array(); org_max_rows=max_rows= param->max_rows; set_if_bigger(maxcount, 1); @@ -1780,20 +1779,23 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, cmp= get_ptr_compare(sort_length); first_cmp_arg= (void*) &sort_length; } - if (unlikely(init_queue(&queue, (uint) (Tb-Fb)+1, offsetof(BUFFPEK,key), 0, + if (unlikely(init_queue(&queue, (uint) (Tb-Fb)+1, + offsetof(Merge_chunk,m_current_key), 0, (queue_compare) cmp, first_cmp_arg, 0, 0))) DBUG_RETURN(1); /* purecov: inspected */ for (buffpek= Fb ; buffpek <= Tb ; buffpek++) { - buffpek->base= strpos; - buffpek->end= buffpek->base + buff_size / ((uint) (Tb-Fb) +1); - buffpek->max_keys= maxcount; + buffpek->set_buffer(strpos, + strpos + (sort_buffer.size()/((uint) (Tb-Fb) +1))); + + buffpek->set_max_keys(maxcount); bytes_read= read_to_buffer(from_file, buffpek, param); if (unlikely(bytes_read == (ulong) -1)) goto err; /* purecov: inspected */ strpos+= bytes_read; - buffpek->end= strpos; - buffpek->max_keys= buffpek->mem_count; // If less data in buffers than expected + buffpek->set_buffer_end(strpos); + // If less data in buffers than expected + buffpek->set_max_keys(buffpek->mem_count()); queue_insert(&queue, (uchar*) buffpek); } @@ -1804,13 +1806,14 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, Copy the first argument to unique_buff for unique removal. Store it also in 'to_file'. */ - buffpek= (BUFFPEK*) queue_top(&queue); - memcpy(unique_buff, buffpek->key, rec_length); + buffpek= (Merge_chunk*) queue_top(&queue); + memcpy(unique_buff, buffpek->current_key(), rec_length); if (min_dupl_count) memcpy(&dupl_count, unique_buff+dupl_count_ofs, sizeof(dupl_count)); - buffpek->key+= rec_length; - if (! --buffpek->mem_count) + buffpek->advance_current_key(rec_length); + buffpek->decrement_mem_count(); + if (buffpek->mem_count() == 0) { if (unlikely(!(bytes_read= read_to_buffer(from_file, buffpek, param)))) @@ -1833,62 +1836,65 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, for (;;) { - buffpek= (BUFFPEK*) queue_top(&queue); - src= buffpek->key; + buffpek= (Merge_chunk*) queue_top(&queue); + src= buffpek->current_key(); if (cmp) // Remove duplicates { - if (!(*cmp)(first_cmp_arg, &unique_buff, - (uchar**) &buffpek->key)) - { + uchar *current_key= buffpek->current_key(); + if (!(*cmp)(first_cmp_arg, &unique_buff, ¤t_key)) + { if (min_dupl_count) - { + { element_count cnt; - memcpy(&cnt, (uchar *) buffpek->key+dupl_count_ofs, sizeof(cnt)); + memcpy(&cnt, buffpek->current_key() + dupl_count_ofs, sizeof(cnt)); dupl_count+= cnt; } goto skip_duplicate; } if (min_dupl_count) - { + { memcpy(unique_buff+dupl_count_ofs, &dupl_count, sizeof(dupl_count)); } - src= unique_buff; + src= unique_buff; } + { - param->get_rec_and_res_len(buffpek->key, + param->get_rec_and_res_len(buffpek->current_key(), &rec_length, &res_length); const uint bytes_to_write= (flag == 0) ? rec_length : res_length; - /* - Do not write into the output file if this is the final merge called - for a Unique object used for intersection and dupl_count is less - than min_dupl_count. - If the Unique object is used to intersect N sets of unique elements - then for any element: - dupl_count >= N <=> the element is occurred in each of these N sets. - */ - if (!check_dupl_count || dupl_count >= min_dupl_count) - { - if (my_b_write(to_file, src+wr_offset, bytes_to_write)) - goto err; /* purecov: inspected */ - } - if (cmp) - { - memcpy(unique_buff, (uchar*) buffpek->key, rec_length); - if (min_dupl_count) - memcpy(&dupl_count, unique_buff+dupl_count_ofs, - sizeof(dupl_count)); - } - if (!--max_rows) - { - /* Nothing more to do */ - goto end; /* purecov: inspected */ - } + /* + Do not write into the output file if this is the final merge called + for a Unique object used for intersection and dupl_count is less + than min_dupl_count. + If the Unique object is used to intersect N sets of unique elements + then for any element: + dupl_count >= N <=> the element is occurred in each of these N sets. + */ + if (!check_dupl_count || dupl_count >= min_dupl_count) + { + if (my_b_write(to_file, src + wr_offset, bytes_to_write)) + goto err; /* purecov: inspected */ + } + if (cmp) + { + memcpy(unique_buff, buffpek->current_key(), rec_length); + if (min_dupl_count) + memcpy(&dupl_count, unique_buff+dupl_count_ofs, + sizeof(dupl_count)); + } + if (!--max_rows) + { + /* Nothing more to do */ + goto end; /* purecov: inspected */ + } } skip_duplicate: - buffpek->key+= rec_length; - if (! --buffpek->mem_count) + buffpek->advance_current_key(rec_length); + buffpek->decrement_mem_count(); + + if (buffpek->mem_count() == 0) { if (unlikely(!(bytes_read= read_to_buffer(from_file, buffpek, param)))) @@ -1903,10 +1909,10 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, queue_replace_top(&queue); /* Top element has been replaced */ } } - buffpek= (BUFFPEK*) queue_top(&queue); - buffpek->base= (uchar*) sort_buffer; - buffpek->end = buffpek->base + buff_size; - buffpek->max_keys= param->max_keys_per_buffer; + buffpek= (Merge_chunk*) queue_top(&queue); + buffpek->set_buffer(sort_buffer.array(), + sort_buffer.array() + sort_buffer.size()); + buffpek->set_max_keys(param->max_keys_per_buffer); /* As we know all entries in the buffer are unique, we only have to @@ -1914,16 +1920,17 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, */ if (cmp) { - if (!(*cmp)(first_cmp_arg, &unique_buff, (uchar**) &buffpek->key)) + uchar *current_key= buffpek->current_key(); + if (!(*cmp)(first_cmp_arg, &unique_buff, ¤t_key)) { if (min_dupl_count) { element_count cnt; - memcpy(&cnt, (uchar *) buffpek->key+dupl_count_ofs, sizeof(cnt)); + memcpy(&cnt, buffpek->current_key() + dupl_count_ofs, sizeof(cnt)); dupl_count+= cnt; } - buffpek->key+= rec_length; - --buffpek->mem_count; + buffpek->advance_current_key(rec_length); + buffpek->decrement_mem_count(); } if (min_dupl_count) @@ -1942,27 +1949,29 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, do { - if ((ha_rows) buffpek->mem_count > max_rows) + if (buffpek->mem_count() > max_rows) { /* Don't write too many records */ - buffpek->mem_count= (uint) max_rows; - buffpek->count= 0; /* Don't read more */ + buffpek->set_mem_count(max_rows); + buffpek->set_rowcount(0); /* Don't read more */ } - max_rows-= buffpek->mem_count; - for (uint ix= 0; ix < buffpek->mem_count; ++ix) + max_rows-= buffpek->mem_count(); + for (uint ix= 0; ix < buffpek->mem_count(); ++ix) { - param->get_rec_and_res_len(buffpek->key, + param->get_rec_and_res_len(buffpek->current_key(), &rec_length, &res_length); const uint bytes_to_write= (flag == 0) ? rec_length : res_length; if (check_dupl_count) { memcpy((uchar *) &dupl_count, - buffpek->key + offset + dupl_count_ofs, sizeof(dupl_count)); + buffpek->current_key() + offset + dupl_count_ofs, + sizeof(dupl_count)); if (dupl_count < min_dupl_count) continue; } - if (my_b_write(to_file, buffpek->key + wr_offset, bytes_to_write)) + if (my_b_write(to_file, buffpek->current_key() + wr_offset, + bytes_to_write)) goto err; - buffpek->key+= rec_length; + buffpek->advance_current_key(rec_length); } } while (likely(!(error= @@ -1971,8 +1980,9 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file, bytes_read != 0); end: - lastbuff->count= MY_MIN(org_max_rows-max_rows, param->max_rows); - lastbuff->file_pos= to_start_filepos; + lastbuff->set_rowcount(MY_MIN(org_max_rows-max_rows, param->max_rows)); + lastbuff->set_file_position(to_start_filepos); + cleanup: delete_queue(&queue); DBUG_RETURN(error); @@ -1986,13 +1996,13 @@ err: /* Do a merge to output-file (save only positions) */ -int merge_index(Sort_param *param, uchar *sort_buffer, size_t buff_size, - BUFFPEK *buffpek, uint maxbuffer, - IO_CACHE *tempfile, IO_CACHE *outfile) +int merge_index(Sort_param *param, Sort_buffer sort_buffer, + Merge_chunk *buffpek, uint maxbuffer, + IO_CACHE *tempfile, IO_CACHE *outfile) { DBUG_ENTER("merge_index"); - if (merge_buffers(param,tempfile,outfile,sort_buffer, buff_size, buffpek,buffpek, - buffpek+maxbuffer,1)) + if (merge_buffers(param, tempfile, outfile, sort_buffer, buffpek, buffpek, + buffpek + maxbuffer, 1)) DBUG_RETURN(1); /* purecov: inspected */ DBUG_RETURN(0); } /* merge_index */ @@ -2115,7 +2125,7 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, sortorder->length= (uint)cs->coll->strnxfrmlen(cs, sortorder->length); } if (sortorder->field->maybe_null()) - length++; // Place for NULL marker + length++; // Place for NULL marker } else { @@ -2126,12 +2136,12 @@ sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length, *multi_byte_charset= true; } if (sortorder->item->maybe_null) - length++; // Place for NULL marker + length++; // Place for NULL marker } set_if_smaller(sortorder->length, thd->variables.max_sort_length); length+=sortorder->length; } - sortorder->field= (Field*) 0; // end marker + sortorder->field= NULL; // end marker DBUG_PRINT("info",("sort_length: %d",length)); return length; } @@ -2196,7 +2206,7 @@ bool filesort_use_addons(TABLE *table, uint sortlength, */ static Addon_fields* -get_addon_fields(TABLE *table, uint sortlength, LEX_STRING *addon_buf, +get_addon_fields(TABLE *table, uint sortlength, uint *addon_length, uint *m_packable_length) { Field **pfield; @@ -2215,8 +2225,6 @@ get_addon_fields(TABLE *table, uint sortlength, LEX_STRING *addon_buf, the values directly from sorted fields. But beware the case when item->cmp_type() != item->result_type() */ - addon_buf->str= 0; - addon_buf->length= 0; // see remove_const() for HA_SLOW_RND_POS explanation if (table->file->ha_table_flags() & HA_SLOW_RND_POS) @@ -2230,7 +2238,6 @@ get_addon_fields(TABLE *table, uint sortlength, LEX_STRING *addon_buf, &raw_mem, sizeof(Addon_fields), &raw_mem_addon_field, sizeof(SORT_ADDON_FIELD) * fields, - &addon_buf->str, length, NullS))) DBUG_RETURN(0); @@ -2238,7 +2245,6 @@ get_addon_fields(TABLE *table, uint sortlength, LEX_STRING *addon_buf, addon_array(static_cast<SORT_ADDON_FIELD*>(raw_mem_addon_field), fields); Addon_fields *addon_fields= new (raw_mem) Addon_fields(addon_array); - addon_buf->length= length; (*addon_length)= length; (*m_packable_length)= packable_length; @@ -2342,6 +2348,12 @@ bool SORT_INFO::using_packed_addons() return addon_fields != NULL && addon_fields->using_packed_addons(); } +void SORT_INFO::free_addon_buff() +{ + if (addon_fields) + addon_fields->free_addon_buff(); +} + /** Free SORT_INFO */ diff --git a/sql/filesort.h b/sql/filesort.h index 0580e4a9c42..cf012e68e46 100644 --- a/sql/filesort.h +++ b/sql/filesort.h @@ -99,6 +99,7 @@ public: void free_data() { close_cached_file(&io_cache); + free_addon_buff(); my_free(record_pointers); my_free(buffpek.str); my_free(addon_fields); @@ -114,6 +115,8 @@ public: sorted_result_in_fsbuf= false; } + void free_addon_buff(); + IO_CACHE io_cache; /* If sorted through filesort */ LEX_STRING buffpek; /* Buffer for buffpek structures */ @@ -165,12 +168,14 @@ public: void adjust_next_record_pointer(uint val) { filesort_buffer.adjust_next_record_pointer(val); } - uchar *get_raw_buf() + Bounds_checked_array<uchar> get_raw_buf() { return filesort_buffer.get_raw_buf(); } size_t sort_buffer_size() const { return filesort_buffer.sort_buffer_size(); } + bool is_allocated() const + { return filesort_buffer.is_allocated(); } void set_sort_length(uint val) { filesort_buffer.set_sort_length(val); } uint get_sort_length() const diff --git a/sql/filesort_utils.h b/sql/filesort_utils.h index 4527e2952c1..31451cfc286 100644 --- a/sql/filesort_utils.h +++ b/sql/filesort_utils.h @@ -168,6 +168,11 @@ public: return m_size_in_bytes; } + bool is_allocated() const + { + return m_rawmem; + } + /** Allocates the buffer, but does *not* initialize pointers. Total size = (num_records * record_length) + (num_records * sizeof(pointer)) @@ -211,9 +216,9 @@ public: @returns The entire buffer, as a character array. This is for reusing the memory for merge buffers. */ - uchar *get_raw_buf() + Bounds_checked_array<uchar> get_raw_buf() { - return m_rawmem; + return Bounds_checked_array<uchar>(m_rawmem, m_size_in_bytes); } /** diff --git a/sql/records.cc b/sql/records.cc index 53b08bf3485..77bb725aedd 100644 --- a/sql/records.cc +++ b/sql/records.cc @@ -202,8 +202,8 @@ bool init_read_record(READ_RECORD *info,THD *thd, TABLE *table, if (using_addon_fields) { - info->rec_buf= (uchar*) filesort->addon_buf.str; - info->ref_length= (uint)filesort->addon_buf.length; + info->rec_buf= filesort->addon_fields->get_addon_buf(); + info->ref_length= filesort->addon_fields->get_addon_buf_length(); } else { diff --git a/sql/sql_sort.h b/sql/sql_sort.h index 4f53bc53a3f..50fea16b4f7 100644 --- a/sql/sql_sort.h +++ b/sql/sql_sort.h @@ -20,8 +20,6 @@ #include <my_sys.h> /* qsort2_cmp */ #include "queues.h" -typedef struct st_buffpek BUFFPEK; - struct SORT_FIELD; class Field; struct TABLE; @@ -63,6 +61,88 @@ struct BUFFPEK_COMPARE_CONTEXT void *key_compare_arg; }; +struct Merge_chunk { +public: + Merge_chunk(): m_current_key(NULL), + m_file_position(0), + m_buffer_start(NULL), + m_buffer_end(NULL), + m_rowcount(0), + m_mem_count(0), + m_max_keys(0) + {} + + my_off_t file_position() const { return m_file_position; } + void set_file_position(my_off_t val) { m_file_position= val; } + void advance_file_position(my_off_t val) { m_file_position+= val; } + + uchar *buffer_start() { return m_buffer_start; } + const uchar *buffer_end() const { return m_buffer_end; } + + void set_buffer(uchar *start, uchar *end) + { + m_buffer_start= start; + m_buffer_end= end; + } + void set_buffer_start(uchar *start) + { + m_buffer_start= start; + } + void set_buffer_end(uchar *end) + { + DBUG_ASSERT(m_buffer_end == NULL || end <= m_buffer_end); + m_buffer_end= end; + } + + void init_current_key() { m_current_key= m_buffer_start; } + uchar *current_key() { return m_current_key; } + void advance_current_key(uint val) { m_current_key+= val; } + + void decrement_rowcount(ha_rows val) { m_rowcount-= val; } + void set_rowcount(ha_rows val) { m_rowcount= val; } + ha_rows rowcount() const { return m_rowcount; } + + ha_rows mem_count() const { return m_mem_count; } + void set_mem_count(ha_rows val) { m_mem_count= val; } + ha_rows decrement_mem_count() { return --m_mem_count; } + + ha_rows max_keys() const { return m_max_keys; } + void set_max_keys(ha_rows val) { m_max_keys= val; } + + size_t buffer_size() const { return m_buffer_end - m_buffer_start; } + + /** + Tries to merge *this with *mc, returns true if successful. + The assumption is that *this is no longer in use, + and the space it has been allocated can be handed over to a + buffer which is adjacent to it. + */ + bool merge_freed_buff(Merge_chunk *mc) const + { + if (mc->m_buffer_end == m_buffer_start) + { + mc->m_buffer_end= m_buffer_end; + mc->m_max_keys+= m_max_keys; + return true; + } + else if (mc->m_buffer_start == m_buffer_end) + { + mc->m_buffer_start= m_buffer_start; + mc->m_max_keys+= m_max_keys; + return true; + } + return false; + } + + uchar *m_current_key; /// The current key for this chunk. + my_off_t m_file_position;/// Current position in the file to be sorted. + uchar *m_buffer_start; /// Start of main-memory buffer for this chunk. + uchar *m_buffer_end; /// End of main-memory buffer for this chunk. + ha_rows m_rowcount; /// Number of unread rows in this chunk. + ha_rows m_mem_count; /// Number of rows in the main-memory buffer. + ha_rows m_max_keys; /// If we have fixed-size rows: + /// max number of rows in buffer. +}; typedef Bounds_checked_array<SORT_ADDON_FIELD> Addon_fields_array; @@ -78,8 +158,8 @@ class Addon_fields { public: Addon_fields(Addon_fields_array arr) : m_field_descriptors(arr), - m_addon_buf(NULL), - m_addon_buf_length(0), + m_addon_buf(), + m_addon_buf_length(), m_using_packed_addons(false) { DBUG_ASSERT(!arr.is_null()); @@ -87,25 +167,24 @@ public: SORT_ADDON_FIELD *begin() { return m_field_descriptors.begin(); } SORT_ADDON_FIELD *end() { return m_field_descriptors.end(); } - size_t num_field_descriptors() const { return m_field_descriptors.size(); } -/* - /// rr_unpack_from_tempfile needs an extra buffer when unpacking. + /// rr_unpack_from_tempfile needs an extra buffer when unpacking. uchar *allocate_addon_buf(uint sz) { - if (m_addon_buf != NULL) - { - DBUG_ASSERT(m_addon_buf_length == sz); - return m_addon_buf; - } - m_addon_buf= static_cast<uchar*>(sql_alloc(sz)); + m_addon_buf= (uchar *)my_malloc(sz, MYF(MY_WME | MY_THREAD_SPECIFIC)); if (m_addon_buf) m_addon_buf_length= sz; return m_addon_buf; } -*/ - uchar *get_addon_buf() { return m_addon_buf; } + void free_addon_buff() + { + my_free(m_addon_buf); + m_addon_buf= NULL; + m_addon_buf_length= 0; + } + + uchar *get_addon_buf() { return m_addon_buf; } uint get_addon_buf_length() const { return m_addon_buf_length; } void set_using_packed_addons(bool val) @@ -165,10 +244,12 @@ public: ha_rows max_rows; // Select limit, or HA_POS_ERROR if unlimited. ha_rows examined_rows; // Number of examined rows. TABLE *sort_form; // For quicker make_sortkey. - SORT_FIELD *local_sortorder; - SORT_FIELD *end; + /** + ORDER BY list with some precalculated info for filesort. + Array is created and owned by a Filesort instance. + */ + Bounds_checked_array<SORT_FIELD> local_sortorder; Addon_fields *addon_fields; // Descriptors for companion fields. - LEX_STRING addon_buf; // Buffer & length of added packed fields. bool using_pq; uchar *unique_buff; @@ -227,19 +308,19 @@ private: bool m_using_packed_addons; ///< caches the value of using_packed_addons() }; +typedef Bounds_checked_array<uchar> Sort_buffer; -int merge_many_buff(Sort_param *param, uchar *sort_buffer, size_t buff_size, - BUFFPEK *buffpek, - uint *maxbuffer, IO_CACHE *t_file); -ulong read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, +int merge_many_buff(Sort_param *param, Sort_buffer sort_buffer, + Merge_chunk *buffpek, uint *maxbuffer, IO_CACHE *t_file); +ulong read_to_buffer(IO_CACHE *fromfile, Merge_chunk *buffpek, Sort_param *param); bool merge_buffers(Sort_param *param,IO_CACHE *from_file, - IO_CACHE *to_file, uchar *sort_buffer, size_t buff_size, - BUFFPEK *lastbuff,BUFFPEK *Fb, - BUFFPEK *Tb,int flag); -int merge_index(Sort_param *param, uchar *sort_buffer, size_t buff_size, - BUFFPEK *buffpek, uint maxbuffer, - IO_CACHE *tempfile, IO_CACHE *outfile); -void reuse_freed_buff(QUEUE *queue, BUFFPEK *reuse, uint key_length); + IO_CACHE *to_file, Sort_buffer sort_buffer, + Merge_chunk *lastbuff, Merge_chunk *Fb, + Merge_chunk *Tb, int flag); +int merge_index(Sort_param *param, Sort_buffer sort_buffer, + Merge_chunk *buffpek, uint maxbuffer, + IO_CACHE *tempfile, IO_CACHE *outfile); +void reuse_freed_buff(QUEUE *queue, Merge_chunk *reuse, uint key_length); #endif /* SQL_SORT_INCLUDED */ diff --git a/sql/uniques.cc b/sql/uniques.cc index 4e0d40c5233..71286e3184d 100644 --- a/sql/uniques.cc +++ b/sql/uniques.cc @@ -39,7 +39,6 @@ #include "my_tree.h" // element_count #include "uniques.h" // Unique #include "sql_sort.h" -#include "myisamchk.h" // BUFFPEK int unique_write_to_file(uchar* key, element_count count, Unique *unique) { @@ -94,7 +93,7 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg, init_tree(&tree, (max_in_memory_size / 16), 0, size, comp_func, NULL, comp_func_fixed_arg, MYF(MY_THREAD_SPECIFIC)); /* If the following fail's the next add will also fail */ - my_init_dynamic_array(&file_ptrs, sizeof(BUFFPEK), 16, 16, + my_init_dynamic_array(&file_ptrs, sizeof(Merge_chunk), 16, 16, MYF(MY_THREAD_SPECIFIC)); /* If you change the following, change it in get_max_elements function, too. @@ -375,10 +374,10 @@ Unique::~Unique() /* Write tree to disk; clear tree */ bool Unique::flush() { - BUFFPEK file_ptr; + Merge_chunk file_ptr; elements+= tree.elements_in_tree; - file_ptr.count=tree.elements_in_tree; - file_ptr.file_pos=my_b_tell(&file); + file_ptr.set_rowcount(tree.elements_in_tree); + file_ptr.set_file_position(my_b_tell(&file)); tree_walk_action action= min_dupl_count ? (tree_walk_action) unique_write_to_file_with_count : @@ -490,7 +489,7 @@ void put_counter_into_merged_element(void *ptr, uint ofs, element_count cnt) */ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, - uint key_length, BUFFPEK *begin, BUFFPEK *end, + uint key_length, Merge_chunk *begin, Merge_chunk *end, tree_walk_action walk_action, void *walk_action_arg, qsort_cmp2 compare, void *compare_arg, IO_CACHE *file, bool with_counters) @@ -499,7 +498,8 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, QUEUE queue; if (end <= begin || merge_buffer_size < (size_t) (key_length * (end - begin + 1)) || - init_queue(&queue, (uint) (end - begin), offsetof(BUFFPEK, key), 0, + init_queue(&queue, (uint) (end - begin), + offsetof(Merge_chunk, m_current_key), 0, buffpek_compare, &compare_context, 0, 0)) return 1; /* we need space for one key when a piece of merge buffer is re-read */ @@ -510,7 +510,7 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, /* if piece_size is aligned reuse_freed_buffer will always hit */ uint piece_size= max_key_count_per_piece * key_length; ulong bytes_read; /* to hold return value of read_to_buffer */ - BUFFPEK *top; + Merge_chunk *top; int res= 1; uint cnt_ofs= key_length - (with_counters ? sizeof(element_count) : 0); element_count cnt; @@ -528,16 +528,16 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, */ for (top= begin; top != end; ++top) { - top->base= merge_buffer + (top - begin) * piece_size; - top->end= top->base + piece_size; - top->max_keys= max_key_count_per_piece; + top->set_buffer_start(merge_buffer + (top - begin) * piece_size); + top->set_buffer_end(top->buffer_start() + piece_size); + top->set_max_keys(max_key_count_per_piece); bytes_read= read_to_buffer(file, top, &sort_param); if (unlikely(bytes_read == (ulong) -1)) goto end; DBUG_ASSERT(bytes_read); queue_insert(&queue, (uchar *) top); } - top= (BUFFPEK *) queue_top(&queue); + top= (Merge_chunk *) queue_top(&queue); while (queue.elements > 1) { /* @@ -547,13 +547,14 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, elements in each tree are unique. Action is applied only to unique elements. */ - void *old_key= top->key; + void *old_key= top->current_key(); /* read next key from the cache or from the file and push it to the queue; this gives new top. */ - top->key+= key_length; - if (--top->mem_count) + top->advance_current_key(key_length); + top->decrement_mem_count(); + if (top->mem_count()) queue_replace_top(&queue); else /* next piece should be read */ { @@ -575,9 +576,9 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, reuse_freed_buff(&queue, top, key_length); } } - top= (BUFFPEK *) queue_top(&queue); + top= (Merge_chunk *) queue_top(&queue); /* new top has been obtained; if old top is unique, apply the action */ - if (compare(compare_arg, old_key, top->key)) + if (compare(compare_arg, old_key, top->current_key())) { cnt= with_counters ? get_counter_from_merged_element(old_key, cnt_ofs) : 1; @@ -586,9 +587,9 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, } else if (with_counters) { - cnt= get_counter_from_merged_element(top->key, cnt_ofs); + cnt= get_counter_from_merged_element(top->current_key(), cnt_ofs); cnt+= get_counter_from_merged_element(old_key, cnt_ofs); - put_counter_into_merged_element(top->key, cnt_ofs, cnt); + put_counter_into_merged_element(top->current_key(), cnt_ofs, cnt); } } /* @@ -602,12 +603,12 @@ static bool merge_walk(uchar *merge_buffer, size_t merge_buffer_size, { cnt= with_counters ? - get_counter_from_merged_element(top->key, cnt_ofs) : 1; - if (walk_action(top->key, cnt, walk_action_arg)) + get_counter_from_merged_element(top->current_key(), cnt_ofs) : 1; + if (walk_action(top->current_key(), cnt, walk_action_arg)) goto end; - top->key+= key_length; + top->advance_current_key(key_length); } - while (--top->mem_count); + while (top->decrement_mem_count()); bytes_read= read_to_buffer(file, top, &sort_param); if (unlikely(bytes_read == (ulong) -1)) goto end; @@ -670,8 +671,8 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) if (!res) { res= merge_walk(merge_buffer, buff_sz, full_size, - (BUFFPEK *) file_ptrs.buffer, - (BUFFPEK *) file_ptrs.buffer + file_ptrs.elements, + (Merge_chunk *) file_ptrs.buffer, + (Merge_chunk *) file_ptrs.buffer + file_ptrs.elements, action, walk_action_arg, tree.compare, tree.custom_arg, &file, with_counters); } @@ -698,10 +699,11 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg) <> 0 error */ -bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size, bool without_last_merge) +bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size, + bool without_last_merge) { IO_CACHE *outfile= &sort.io_cache; - BUFFPEK *file_ptr= (BUFFPEK*) file_ptrs.buffer; + Merge_chunk *file_ptr= (Merge_chunk*) file_ptrs.buffer; uint maxbuffer= file_ptrs.elements - 1; my_off_t save_pos; bool error= 1; @@ -732,7 +734,9 @@ bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size, bool without_las sort_param.cmp_context.key_compare_arg= tree.custom_arg; /* Merge the buffers to one file, removing duplicates */ - if (merge_many_buff(&sort_param,buff, buff_size, file_ptr,&maxbuffer,&file)) + if (merge_many_buff(&sort_param, + Bounds_checked_array<uchar>(buff, buff_size), + file_ptr,&maxbuffer,&file)) goto err; if (flush_io_cache(&file) || reinit_io_cache(&file,READ_CACHE,0L,0,0)) @@ -744,7 +748,8 @@ bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size, bool without_las file_ptrs.elements= maxbuffer+1; return 0; } - if (merge_index(&sort_param, buff, buff_size, file_ptr, maxbuffer, &file, outfile)) + if (merge_index(&sort_param, Bounds_checked_array<uchar>(buff, buff_size), + file_ptr, maxbuffer, &file, outfile)) goto err; error= 0; err: |