diff options
author | Igor Babaev <igor@askmonty.org> | 2013-03-11 07:44:24 -0700 |
---|---|---|
committer | Igor Babaev <igor@askmonty.org> | 2013-03-11 07:44:24 -0700 |
commit | fc1c8ffdadfd14eb51969ecfde43e3204f10f6f8 (patch) | |
tree | 70ee4b3628cc62a706476d6c7b873fcfc9aa0e10 /sql | |
parent | 938d47dcdc50ae4f127197ed72fd044b33ea7524 (diff) | |
download | mariadb-git-fc1c8ffdadfd14eb51969ecfde43e3204f10f6f8.tar.gz |
The pilot patch for mwl#253.
Diffstat (limited to 'sql')
-rw-r--r-- | sql/field.cc | 56 | ||||
-rw-r--r-- | sql/field.h | 27 | ||||
-rw-r--r-- | sql/item_cmpfunc.cc | 20 | ||||
-rw-r--r-- | sql/item_cmpfunc.h | 2 | ||||
-rw-r--r-- | sql/opt_range.cc | 219 | ||||
-rw-r--r-- | sql/opt_range.h | 2 | ||||
-rw-r--r-- | sql/opt_range_mrr.cc | 6 | ||||
-rw-r--r-- | sql/opt_subselect.cc | 2 | ||||
-rw-r--r-- | sql/opt_subselect.h | 1 | ||||
-rw-r--r-- | sql/sql_class.h | 1 | ||||
-rw-r--r-- | sql/sql_priv.h | 1 | ||||
-rw-r--r-- | sql/sql_select.cc | 268 | ||||
-rw-r--r-- | sql/sql_select.h | 6 | ||||
-rw-r--r-- | sql/sql_statistics.cc | 49 | ||||
-rw-r--r-- | sql/sql_statistics.h | 6 | ||||
-rw-r--r-- | sql/sys_vars.cc | 16 | ||||
-rw-r--r-- | sql/table.cc | 6 | ||||
-rw-r--r-- | sql/table.h | 3 |
18 files changed, 657 insertions, 34 deletions
diff --git a/sql/field.cc b/sql/field.cc index d4468ba3c5b..e54a3e59795 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1273,6 +1273,20 @@ out_of_range: return 1; } + +double Field_num::middle_point_pos(Field *min, Field *max) +{ + double n, d; + n= val_real() - min->val_real(); + if (n < 0) + return 0.0; + d= max->val_real() - min->val_real(); + if (d <= 0) + return 1.0; + return min(n/d, 1.0); +} + + /** Process decimal library return codes and issue warnings for overflow and truncation. @@ -1344,6 +1358,8 @@ Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg, comment.length=0; field_index= 0; is_stat_field= FALSE; + cond_selectivity= 1.0; + next_equal_field= NULL; } @@ -6167,6 +6183,46 @@ int Field_str::store(double nr) return store(buff, length, &my_charset_numeric); } +static +inline ulonglong char_prefix_to_ulonglong(uchar *src) +{ + uint sz= sizeof(ulonglong); + for (uint i= 0; i < sz/2; i++) + { + uchar tmp= src[i]; + src[i]= src[sz-1-i]; + src[sz-1-i]= tmp; + } + return uint8korr(src); +} + +double Field_str::middle_point_pos(Field *min, Field *max) +{ + uchar mp_prefix[sizeof(ulonglong)]; + uchar minp_prefix[sizeof(ulonglong)]; + uchar maxp_prefix[sizeof(ulonglong)]; + ulonglong mp, minp, maxp; + my_strnxfrm(charset(), mp_prefix, sizeof(mp), + ptr + length_size(), sizeof(mp) * charset()->mbmaxlen); + my_strnxfrm(charset(), minp_prefix, sizeof(minp), + min->ptr + length_size(), + sizeof(minp) * charset()->mbmaxlen); + my_strnxfrm(charset(), maxp_prefix, sizeof(maxp), + max->ptr + length_size(), + sizeof(maxp) * charset()->mbmaxlen); + mp= char_prefix_to_ulonglong(mp_prefix); + minp= char_prefix_to_ulonglong(minp_prefix); + maxp= char_prefix_to_ulonglong(maxp_prefix); + double n, d; + n= mp - minp; + if (n < 0) + return 0.0; + d= maxp - minp; + if (d <= 0) + return 1.0; + return min(n/d, 1.0); +} + uint Field::is_equal(Create_field *new_field) { diff --git a/sql/field.h b/sql/field.h index e832928b114..40cbcca09ad 100644 --- a/sql/field.h +++ b/sql/field.h @@ -220,7 +220,23 @@ public: */ bool is_created_from_null_item; - bool is_stat_field; /* TRUE in Field objects created for column min/max values */ + /* TRUE in Field objects created for column min/max values */ + bool is_stat_field; + + /* + Selectivity of the range condition over this field. + When calculating this selectivity a range predicate + is taken into account only if: + - it is extracted from the WHERE clause + - it depends only on the table the field belongs to + */ + double cond_selectivity; + + /* + The next field in the class of equal fields at the top AND level + of the WHERE clause + */ + Field *next_equal_field; /* This structure is used for statistical data on the column @@ -703,6 +719,11 @@ public: virtual bool hash_join_is_possible() { return TRUE; } virtual bool eq_cmp_as_binary() { return TRUE; } + virtual double middle_point_pos(Field *min, Field *max) + { + return (double) 1.0; + } + friend int cre_myisam(char * name, register TABLE *form, uint options, ulonglong auto_increment_value); friend class Copy_field; @@ -821,6 +842,7 @@ public: bool get_int(CHARSET_INFO *cs, const char *from, uint len, longlong *rnd, ulonglong unsigned_max, longlong signed_min, longlong signed_max); + double middle_point_pos(Field *min, Field *max); }; @@ -866,6 +888,8 @@ public: virtual bool str_needs_quotes() { return TRUE; } uint is_equal(Create_field *new_field); bool eq_cmp_as_binary() { return test(flags & BINARY_FLAG); } + virtual uint length_size() { return 0; } + double middle_point_pos(Field *min, Field *max); }; /* base class for Field_string, Field_varstring and Field_blob */ @@ -1894,6 +1918,7 @@ public: uint new_null_bit); uint is_equal(Create_field *new_field); void hash(ulong *nr, ulong *nr2); + uint length_size() { return length_bytes; } private: int do_save_field_metadata(uchar *first_byte); }; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index b1208243e6b..b95cd2dbc90 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -5567,7 +5567,8 @@ Item *Item_bool_rowready_func2::negated_item() */ Item_equal::Item_equal(Item *f1, Item *f2, bool with_const_item) - : Item_bool_func(), eval_item(0), cond_false(0), context_field(NULL) + : Item_bool_func(), eval_item(0), cond_false(0), context_field(NULL), + link_equal_fields(FALSE) { const_item_cache= 0; with_const= with_const_item; @@ -5590,7 +5591,8 @@ Item_equal::Item_equal(Item *f1, Item *f2, bool with_const_item) */ Item_equal::Item_equal(Item_equal *item_equal) - : Item_bool_func(), eval_item(0), cond_false(0), context_field(NULL) + : Item_bool_func(), eval_item(0), cond_false(0), context_field(NULL), + link_equal_fields(FALSE) { const_item_cache= 0; List_iterator_fast<Item> li(item_equal->equal_items); @@ -5833,6 +5835,9 @@ bool Item_equal::fix_fields(THD *thd, Item **ref) DBUG_ASSERT(fixed == 0); Item_equal_fields_iterator it(*this); Item *item; + Field *first_equal_field; + Field *last_equal_field; + Field *prev_equal_field= NULL; not_null_tables_cache= used_tables_cache= 0; const_item_cache= 0; while ((item= it++)) @@ -5846,7 +5851,18 @@ bool Item_equal::fix_fields(THD *thd, Item **ref) maybe_null= 1; if (!item->get_item_equal()) item->set_item_equal(this); + if (link_equal_fields && item->real_item()->type() == FIELD_ITEM) + { + last_equal_field= ((Item_field *) (item->real_item()))->field; + if (!prev_equal_field) + first_equal_field= last_equal_field; + else + prev_equal_field->next_equal_field= last_equal_field; + prev_equal_field= last_equal_field; + } } + if (prev_equal_field && last_equal_field != first_equal_field) + last_equal_field->next_equal_field= first_equal_field; fix_length_and_dec(); fixed= 1; return FALSE; diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 5a5f9ad60fd..5ac918895fa 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1730,6 +1730,7 @@ class Item_equal: public Item_bool_func used in the original equality. */ Item_field *context_field; + bool link_equal_fields; public: inline Item_equal() : Item_bool_func(), with_const(FALSE), eval_item(0), cond_false(0), @@ -1762,6 +1763,7 @@ public: CHARSET_INFO *compare_collation(); void set_context_field(Item_field *ctx_field) { context_field= ctx_field; } + void set_link_equal_fields(bool flag) { link_equal_fields= flag; } friend class Item_equal_fields_iterator; friend Item *eliminate_item_equal(COND *cond, COND_EQUAL *upper_levels, Item_equal *item_equal); diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 7a3a0c94a18..364b01aebb2 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -117,6 +117,7 @@ #include "records.h" // init_read_record, end_read_record #include <m_ctype.h> #include "sql_select.h" +#include "sql_statistics.h" #include "filesort.h" // filesort_free_buffers #ifndef EXTRA_DEBUG @@ -3212,6 +3213,224 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, } /**************************************************************************** + * Condition selectivity module + ****************************************************************************/ + +static +bool create_key_parts_for_pseudo_indexes(RANGE_OPT_PARAM *param, + MY_BITMAP *used_fields) +{ + Field **field_ptr; + TABLE *table= param->table; + uint parts= 0; + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + if (bitmap_is_set(used_fields, (*field_ptr)->field_index)) + parts++; + } + + KEY_PART *key_part; + uint keys= 0; + + if (!(key_part= (KEY_PART *) alloc_root(param->mem_root, + sizeof(KEY_PART) * parts))) + return TRUE; + + param->key_parts= key_part; + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + if (bitmap_is_set(used_fields, (*field_ptr)->field_index)) + { + Field *field= *field_ptr; + uint16 store_length; + key_part->key= keys; + key_part->part= 0; + key_part->length= (uint16) field->key_length(); + store_length= key_part->length; + if (field->real_maybe_null()) + store_length+= HA_KEY_NULL_LENGTH; + if (field->real_type() == MYSQL_TYPE_VARCHAR) + store_length+= HA_KEY_BLOB_LENGTH; + key_part->store_length= store_length; + key_part->field= field; + key_part->image_type= Field::itRAW; + param->key[keys]= key_part; + keys++; + key_part++; + } + } + param->keys= keys; + param->key_parts_end= key_part; + + return FALSE; +} + + +static +double records_in_column_ranges(PARAM *param, uint idx, + SEL_ARG *tree) +{ + SEL_ARG_RANGE_SEQ seq; + KEY_MULTI_RANGE range; + range_seq_t seq_it; + double rows; + Field *field; + uint flags= 0; + double total_rows= 0; + RANGE_SEQ_IF seq_if = {NULL, sel_arg_range_seq_init, + sel_arg_range_seq_next, 0, 0}; + + /* Handle cases when we don't have a valid non-empty list of range */ + if (!tree) + return HA_POS_ERROR; + if (tree->type == SEL_ARG::IMPOSSIBLE) + return (0L); + + field= tree->field; + + seq.keyno= idx; + seq.real_keyno= MAX_KEY; + seq.param= param; + seq.start= tree; + + seq_it= seq_if.init((void *) &seq, 0, flags); + + while (!seq_if.next(seq_it, &range)) + { + key_range *min_endp, *max_endp; + min_endp= range.start_key.length? &range.start_key : NULL; + max_endp= range.end_key.length? &range.end_key : NULL; + rows= get_column_range_cardinality(field, min_endp, max_endp); + if (HA_POS_ERROR == rows) + { + total_rows= HA_POS_ERROR; + break; + } + total_rows += rows; + } + return total_rows; +} + + +bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond) +{ + uint keynr; + uint max_quick_key_parts= 0; + MY_BITMAP *used_fields= &table->cond_set; + double table_records= table->stat_records(); + DBUG_ENTER("calculate_cond_selectivity_for_table"); + + table->cond_selectivity= 1.0; + + if (bitmap_is_clear_all(used_fields)) + DBUG_RETURN(FALSE); + + PARAM param; + MEM_ROOT alloc; + init_sql_alloc(&alloc, thd->variables.range_alloc_block_size, 0, + MYF(MY_THREAD_SPECIFIC)); + param.thd= thd; + param.mem_root= &alloc; + param.old_root= thd->mem_root; + param.table= table; + param.is_ror_scan= FALSE; + + if (create_key_parts_for_pseudo_indexes(¶m, used_fields)) + { + free_root(&alloc, MYF(0)); + DBUG_RETURN(FALSE); + } + + param.prev_tables= param.read_tables= 0; + param.current_table= table->map; + param.using_real_indexes= FALSE; + param.real_keynr[0]= 0; + param.alloced_sel_args= 0; + + thd->no_errors=1; // Don't warn about NULL + + SEL_TREE *tree; + SEL_ARG **key, **end; + uint idx= 0; + + tree= get_mm_tree(¶m, cond); + + if (!tree) + goto end; + + + for (key= tree->keys, end= key + param.keys; key != end; key++, idx++) + { + double rows; + if (*key) + { + rows= records_in_column_ranges(¶m, idx, *key); + if (rows != HA_POS_ERROR) + (*key)->field->cond_selectivity= rows/table_records; + } + } + + for (Field **field_ptr= table->field; *field_ptr; field_ptr++) + { + Field *table_field= *field_ptr; + if (bitmap_is_set(table->read_set, table_field->field_index) && + table_field->cond_selectivity < 1.0) + table->cond_selectivity*= table_field->cond_selectivity; + } + + /* Calculate the selectivity of the range conditions supported by indexes */ + + bitmap_clear_all(used_fields); + + for (keynr= 0; keynr < table->s->keys; keynr++) + { + if (table->quick_keys.is_set(keynr)) + set_if_bigger(max_quick_key_parts, table->quick_key_parts[keynr]); + } + + for (uint quick_key_parts= max_quick_key_parts; + quick_key_parts; quick_key_parts--) + { + for (keynr= 0; keynr < table->s->keys; keynr++) + { + if (table->quick_keys.is_set(keynr) && + table->quick_key_parts[keynr] == quick_key_parts) + { + uint i; + uint used_key_parts= table->quick_key_parts[keynr]; + double quick_cond_selectivity= table->quick_rows[keynr] / + table_records; + KEY *key_info= table->key_info + keynr; + KEY_PART_INFO* key_part= key_info->key_part; + for (i= 0; i < used_key_parts; i++, key_part++) + { + if (bitmap_is_set(used_fields, key_part->fieldnr-1)) + break; + bitmap_set_bit(used_fields, key_part->fieldnr-1); + } + if (i) + { + double f1= key_info->actual_rec_per_key(i-1); + double f2= key_info->actual_rec_per_key(i); + table->cond_selectivity*= quick_cond_selectivity * f1 / f2; + } + } + } + } + +end: + thd->mem_root= param.old_root; + free_root(&alloc, MYF(0)); + DBUG_RETURN(FALSE); +} + +/**************************************************************************** + * Condition selectivity code ends + ****************************************************************************/ + +/**************************************************************************** * Partition pruning module ****************************************************************************/ #ifdef WITH_PARTITION_STORAGE_ENGINE diff --git a/sql/opt_range.h b/sql/opt_range.h index c59b2a7eb02..484f508e49c 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -1042,6 +1042,8 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables, table_map read_tables, COND *conds, bool allow_null_cond, int *error); +bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item *cond); + #ifdef WITH_PARTITION_STORAGE_ENGINE bool prune_partitions(THD *thd, TABLE *table, Item *pprune_cond); void store_key_image_to_rec(Field *field, uchar *ptr, uint len); diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc index 1f4e36178db..8029dbf000f 100644 --- a/sql/opt_range_mrr.cc +++ b/sql/opt_range_mrr.cc @@ -268,8 +268,10 @@ walk_up_n_right: range->end_key.keypart_map= make_prev_keypart_map(cur->max_key_parts); if (!(cur->min_key_flag & ~NULL_RANGE) && !cur->max_key_flag && - (uint)key_tree->part+1 == seq->param->table->key_info[seq->real_keyno].key_parts && - (seq->param->table->key_info[seq->real_keyno].flags & HA_NOSAME) && + (seq->real_keyno == MAX_KEY || + ((uint)key_tree->part+1 == + seq->param->table->key_info[seq->real_keyno].key_parts && + (seq->param->table->key_info[seq->real_keyno].flags & HA_NOSAME))) && range->start_key.length == range->end_key.length && !memcmp(seq->param->min_key,seq->param->max_key,range->start_key.length)) range->range_flag= UNIQUE_RANGE | (cur->min_key_flag & NULL_RANGE); diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index 6698d4af28d..36ab3a9a2e0 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -3889,7 +3889,7 @@ SJ_TMP_TABLE::create_sj_weedout_tmp_table(THD *thd) &tmpname, (uint) strlen(path)+1, &group_buff, (!using_unique_constraint ? uniq_tuple_length_arg : 0), - &bitmaps, bitmap_buffer_size(1)*3, + &bitmaps, bitmap_buffer_size(1)*5, NullS)) { if (temp_pool_slot != MY_BIT_NONE) diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h index 7b8f3142851..508531b8c37 100644 --- a/sql/opt_subselect.h +++ b/sql/opt_subselect.h @@ -283,6 +283,7 @@ public: { pos->records_read= best_loose_scan_records; pos->key= best_loose_scan_start_key; + pos->cond_selectivity= 1.0; pos->loosescan_picker.loosescan_key= best_loose_scan_key; pos->loosescan_picker.loosescan_parts= best_max_loose_keypart + 1; pos->use_join_buffer= FALSE; diff --git a/sql/sql_class.h b/sql/sql_class.h index 87b9783c1ed..e2fe3a53dcc 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -499,6 +499,7 @@ typedef struct system_variables ulong net_write_timeout; ulong optimizer_prune_level; ulong optimizer_search_depth; + ulong optimizer_use_condition_selectivity; ulong use_stat_tables; ulong preload_buff_size; ulong profiling_history_size; diff --git a/sql/sql_priv.h b/sql/sql_priv.h index 4e356b48497..d14ec5d0983 100644 --- a/sql/sql_priv.h +++ b/sql/sql_priv.h @@ -228,6 +228,7 @@ template <class T> bool valid_buffer_range(T jump, #define OPTIMIZER_SWITCH_TABLE_ELIMINATION (1ULL << 26) #define OPTIMIZER_SWITCH_EXTENDED_KEYS (1ULL << 27) #define OPTIMIZER_SWITCH_EXISTS_TO_IN (1ULL << 28) +#define OPTIMIZER_SWITCH_USE_CONDITION_SELECTIVITY (1ULL << 29) #define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \ OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \ diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 81f510a4563..bc9e05fe2d9 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -89,12 +89,14 @@ void best_access_path(JOIN *join, JOIN_TAB *s, POSITION *pos, POSITION *loose_scan_pos); static void optimize_straight_join(JOIN *join, table_map join_tables); static bool greedy_search(JOIN *join, table_map remaining_tables, - uint depth, uint prune_level); + uint depth, uint prune_level, + uint use_cond_selectivity); static bool best_extension_by_limited_search(JOIN *join, table_map remaining_tables, uint idx, double record_count, double read_time, uint depth, - uint prune_level); + uint prune_level, + uint use_cond_selectivity); static uint determine_search_depth(JOIN* join); C_MODE_START static int join_tab_cmp(const void *dummy, const void* ptr1, const void* ptr2); @@ -133,7 +135,8 @@ static int return_zero_rows(JOIN *join, select_result *res, static COND *build_equal_items(THD *thd, COND *cond, COND_EQUAL *inherited, List<TABLE_LIST> *join_list, - COND_EQUAL **cond_equal_ref); + COND_EQUAL **cond_equal_ref, + bool link_equal_fields= FALSE); static COND* substitute_for_best_equal_field(JOIN_TAB *context_tab, COND *cond, COND_EQUAL *cond_equal, @@ -149,7 +152,8 @@ static uint build_bitmap_for_nested_joins(List<TABLE_LIST> *join_list, static COND *optimize_cond(JOIN *join, COND *conds, List<TABLE_LIST> *join_list, Item::cond_result *cond_value, - COND_EQUAL **cond_equal); + COND_EQUAL **cond_equal, + int flags= 0); bool const_expression_in_where(COND *conds,Item *item, Item **comp_item); static bool create_internal_tmp_table_from_heap2(THD *, TABLE *, ENGINE_COLUMNDEF *, ENGINE_COLUMNDEF **, @@ -277,6 +281,8 @@ enum enum_exec_or_opt {WALK_OPTIMIZATION_TABS , WALK_EXECUTION_TABS}; JOIN_TAB *first_breadth_first_tab(JOIN *join, enum enum_exec_or_opt tabs_kind); JOIN_TAB *next_breadth_first_tab(JOIN *join, enum enum_exec_or_opt tabs_kind, JOIN_TAB *tab); +static double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, + table_map rem_tables); #ifndef DBUG_OFF @@ -1155,7 +1161,8 @@ TODO: make view to decide if it is possible to write to WHERE directly or make S if (setup_jtbm_semi_joins(this, join_list, &conds)) DBUG_RETURN(1); - conds= optimize_cond(this, conds, join_list, &cond_value, &cond_equal); + conds= optimize_cond(this, conds, join_list, &cond_value, &cond_equal, + OPT_LINK_EQUAL_FIELDS); if (thd->is_error()) { @@ -3350,6 +3357,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, table->pos_in_table_list= tables; error= tables->fetch_number_of_rows(); set_statistics_for_table(join->thd, table); + bitmap_clear_all(&table->cond_set); #ifdef WITH_PARTITION_STORAGE_ENGINE const bool no_partitions_used= table->no_partitions_used; @@ -3837,6 +3845,11 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, } delete select; } + + if (join->thd->variables.optimizer_use_condition_selectivity > 1) + calculate_cond_selectivity_for_table(join->thd, s->table, + *s->on_expr_ref ? + *s->on_expr_ref : conds); } if (pull_out_semijoin_tables(join)) @@ -4190,11 +4203,12 @@ add_key_field(JOIN *join, else if (!(field->flags & PART_KEY_FLAG)) { // Don't remove column IS NULL on a LEFT JOIN table - if (!eq_func || (*value)->type() != Item::NULL_ITEM || - !field->table->maybe_null || field->null_ptr) - return; // Not a key. Skip it - optimize= KEY_OPTIMIZE_EXISTS; - DBUG_ASSERT(num_values == 1); + if (eq_func && (*value)->type() == Item::NULL_ITEM && + field->table->maybe_null && !field->null_ptr) + { + optimize= KEY_OPTIMIZE_EXISTS; + DBUG_ASSERT(num_values == 1); + } } if (optimize != KEY_OPTIMIZE_EXISTS) { @@ -4243,7 +4257,11 @@ add_key_field(JOIN *join, break; } if (is_const) + { stat[0].const_keys.merge(possible_keys); + if (possible_keys.is_clear_all()) + bitmap_set_bit(&field->table->cond_set, field->field_index); + } else if (!eq_func) { /* @@ -5313,6 +5331,7 @@ void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key) join->positions[idx].table= table; join->positions[idx].key=key; join->positions[idx].records_read=1.0; /* This is a const table */ + join->positions[idx].cond_selectivity= 1.0; join->positions[idx].ref_depend_map= 0; // join->positions[idx].loosescan_key= MAX_KEY; /* Not a LooseScan */ @@ -6127,6 +6146,8 @@ choose_plan(JOIN *join, table_map join_tables) { uint search_depth= join->thd->variables.optimizer_search_depth; uint prune_level= join->thd->variables.optimizer_prune_level; + uint use_cond_selectivity= + join->thd->variables.optimizer_use_condition_selectivity; bool straight_join= test(join->select_options & SELECT_STRAIGHT_JOIN); DBUG_ENTER("choose_plan"); @@ -6191,7 +6212,8 @@ choose_plan(JOIN *join, table_map join_tables) if (search_depth == 0) /* Automatically determine a reasonable value for 'search_depth' */ search_depth= determine_search_depth(join); - if (greedy_search(join, join_tables, search_depth, prune_level)) + if (greedy_search(join, join_tables, search_depth, prune_level, + use_cond_selectivity)) DBUG_RETURN(TRUE); } } @@ -6465,6 +6487,8 @@ optimize_straight_join(JOIN *join, table_map join_tables) bool disable_jbuf= join->thd->variables.join_cache_level == 0; double record_count= 1.0; double read_time= 0.0; + uint use_cond_selectivity= + join->thd->variables.optimizer_use_condition_selectivity; POSITION loose_scan_pos; for (JOIN_TAB **pos= join->best_ref + idx ; (s= *pos) ; pos++) @@ -6481,6 +6505,11 @@ optimize_straight_join(JOIN *join, table_map join_tables) &loose_scan_pos); join_tables&= ~(s->table->map); + double pushdown_cond_selectivity= 1.0; + if (use_cond_selectivity > 1) + pushdown_cond_selectivity= table_cond_selectivity(join, idx, s, + join_tables); + join->positions[idx].cond_selectivity= pushdown_cond_selectivity; ++idx; } @@ -6568,6 +6597,8 @@ optimize_straight_join(JOIN *join, table_map join_tables) @param search_depth controlls the exhaustiveness of the search @param prune_level the pruning heuristics that should be applied during search + @param use_cond_selectivity specifies how the selectivity of the conditions + pushed to a table should be taken into account @retval FALSE ok @@ -6579,7 +6610,8 @@ static bool greedy_search(JOIN *join, table_map remaining_tables, uint search_depth, - uint prune_level) + uint prune_level, + uint use_cond_selectivity) { double record_count= 1.0; double read_time= 0.0; @@ -6604,7 +6636,8 @@ greedy_search(JOIN *join, /* Find the extension of the current QEP with the lowest cost */ join->best_read= DBL_MAX; if (best_extension_by_limited_search(join, remaining_tables, idx, record_count, - read_time, search_depth, prune_level)) + read_time, search_depth, prune_level, + use_cond_selectivity)) DBUG_RETURN(TRUE); /* 'best_read < DBL_MAX' means that optimizer managed to find @@ -6843,6 +6876,158 @@ double JOIN::get_examined_rows() } +static +double table_multi_eq_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, + table_map rem_tables, TABLE_REF *ref) +{ + double sel= 1.0; + COND_EQUAL *cond_equal= join->cond_equal; + + if (!cond_equal || !cond_equal->current_level.elements) + return sel; + + Item_equal *item_equal; + List_iterator_fast<Item_equal> it(cond_equal->current_level); + table_map table_bit= s->table->map; + + if (!s->keyuse) + return sel; + + KEY *key_info= s->get_keyinfo_by_key_no(s->ref.key); + + while ((item_equal= it++)) + { + /* + Check whether we need to take into account the selectivity of + multiple equality item_equal. If this is the case multiply + the current value of sel by this selectivity + */ + table_map used_tables= item_equal->used_tables(); + if (!(used_tables & table_bit)) + continue; + if (item_equal->get_const()) + continue; + + Field *fld; + bool adjust_sel= FALSE; + Item_equal_fields_iterator fi(*item_equal); + while((fi++) && !adjust_sel) + { + Field *fld= fi.get_curr_field(); + if (fld->table->map != table_bit) + continue; + if (ref == 0) + adjust_sel= TRUE; + else + { + uint i; + for (i= 0; i < ref->key_parts; i++) + { + if (fld->field_index == key_info->key_part[i].fieldnr - 1) + break; + } + if (i == ref->key_parts) + { + /* + Field fld is included in multiple equality item_equal + and is not a part of the ref key. + The selectivity of the multiple equality must be taken + into account unless one of the ref arguments is + equal to fld. + */ + adjust_sel= TRUE; + for (uint j= 0; j < ref->key_parts && adjust_sel; j++) + { + if (ref->items[j]->real_item()->type() == Item::FIELD_ITEM) + { + Item_field *field_item= (Item_field *) (ref->items[j]); + if (item_equal->contains(field_item->field)) + adjust_sel= FALSE; + } + } + } + } + } + if (adjust_sel) + { + /* + If ref == 0 and there are no fields in the multiple equality + item_equal that belong to the tables joined prior to s + then the selectivity of multiple equality will be set to 1.0. + */ + double eq_fld_sel= 1.0; + fi.rewind(); + while ((fi++)) + { + double curr_eq_fld_sel; + fld= fi.get_curr_field(); + if (!fld->table->map & ~(table_bit | rem_tables)) + continue; + curr_eq_fld_sel= get_column_avg_frequency(fld) / + fld->table->stat_records(); + if (curr_eq_fld_sel < 1.0) + set_if_bigger(eq_fld_sel, curr_eq_fld_sel); + } + sel*= eq_fld_sel; + } + } + return sel; +} + +static +double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, + table_map rem_tables) +{ + Field *field; + TABLE *table= s->table; + MY_BITMAP *read_set= table->read_set; + double sel= s->table->cond_selectivity; + double table_records= table->stat_records(); + TABLE_REF *ref= s->type == JT_REF || s->type == JT_EQ_REF ? &s->ref : NULL; + + /* Discount the selectivity of the access method used to join table s */ + if (s->quick && s->quick->index != MAX_KEY) + { + /* A range scan by index s->quick->index is used to access table s */ + sel*= table_records/table->quick_rows[s->quick->index]; + } + else if (ref) + { + /* A ref/ access or hash join is used to join table s */ + KEY *key_info= s->get_keyinfo_by_key_no(ref->key); + for (uint i= 0; i < ref->key_parts; i++) + { + if (ref->items[i]->const_item()) + { + uint fldno= key_info->key_part[i].fieldnr - 1; + sel*= table->field[fldno]->cond_selectivity; + } + } + } + + for (Field **f_ptr=table->field ; (field= *f_ptr) ; f_ptr++) + { + if (!bitmap_is_set(read_set, field->field_index) || + !field->next_equal_field) + continue; + for (Field *next_field= field->next_equal_field; + next_field != field; + next_field= next_field->next_equal_field) + { + if (!(next_field->table->map & rem_tables) && next_field->table != table) + { + sel/= field->cond_selectivity; + break; + } + } + } + + sel*= table_multi_eq_cond_selectivity(join, idx, s, rem_tables, ref); + + return sel; +} + + /** Find a good, possibly optimal, query execution plan (QEP) by a possibly exhaustive search. @@ -6953,6 +7138,8 @@ double JOIN::get_examined_rows() @param prune_level pruning heuristics that should be applied during optimization (values: 0 = EXHAUSTIVE, 1 = PRUNE_BY_TIME_OR_ROWS) + @param use_cond_selectivity specifies how the selectivity of the conditions + pushed to a table should be taken into account @retval FALSE ok @@ -6967,7 +7154,8 @@ best_extension_by_limited_search(JOIN *join, double record_count, double read_time, uint search_depth, - uint prune_level) + uint prune_level, + uint use_cond_selectivity) { DBUG_ENTER("best_extension_by_limited_search"); @@ -7070,16 +7258,25 @@ best_extension_by_limited_search(JOIN *join, } } + double pushdown_cond_selectivity= 1.0; + if (use_cond_selectivity > 1) + pushdown_cond_selectivity= table_cond_selectivity(join, idx, s, + remaining_tables & + ~real_table_bit); + join->positions[idx].cond_selectivity= pushdown_cond_selectivity; + double partial_join_cardinality= current_record_count * + pushdown_cond_selectivity; if ( (search_depth > 1) && (remaining_tables & ~real_table_bit) & allowed_tables ) { /* Recursively expand the current partial plan */ swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); if (best_extension_by_limited_search(join, remaining_tables & ~real_table_bit, idx + 1, - current_record_count, + partial_join_cardinality, current_read_time, search_depth - 1, - prune_level)) + prune_level, + use_cond_selectivity)) DBUG_RETURN(TRUE); swap_variables(JOIN_TAB*, join->best_ref[idx], *pos); } @@ -7097,7 +7294,7 @@ best_extension_by_limited_search(JOIN *join, { memcpy((uchar*) join->best_positions, (uchar*) join->positions, sizeof(POSITION) * (idx + 1)); - join->record_count= current_record_count; + join->record_count= partial_join_cardinality; join->best_read= current_read_time - 0.001; } DBUG_EXECUTE("opt", print_plan(join, idx+1, @@ -7742,6 +7939,7 @@ get_best_combination(JOIN *join) */ SJ_MATERIALIZATION_INFO *sjm= cur_pos->table->emb_sj_nest->sj_mat_info; j->records= j->records_read= (ha_rows)(sjm->is_sj_scan? sjm->rows : 1); + j->cond_selectivity= 1.0; JOIN_TAB *jt; JOIN_TAB_RANGE *jt_range; if (!(jt= (JOIN_TAB*)join->thd->alloc(sizeof(JOIN_TAB)*sjm->tables)) || @@ -7805,6 +8003,7 @@ get_best_combination(JOIN *join) to access join->best_positions[]. */ j->records_read= (ha_rows)join->best_positions[tablenr].records_read; + j->cond_selectivity= join->best_positions[tablenr].cond_selectivity; join->map2table[j->table->tablenr]= j; /* If we've reached the end of sjm nest, switch back to main sequence */ @@ -11779,7 +11978,8 @@ static bool check_equality(THD *thd, Item *item, COND_EQUAL *cond_equal, */ static COND *build_equal_items_for_cond(THD *thd, COND *cond, - COND_EQUAL *inherited) + COND_EQUAL *inherited, + bool link_item_fields) { Item_equal *item_equal; COND_EQUAL cond_equal; @@ -11826,6 +12026,7 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, List_iterator_fast<Item_equal> it(cond_equal.current_level); while ((item_equal= it++)) { + item_equal->set_link_equal_fields(link_item_fields); item_equal->fix_fields(thd, NULL); item_equal->update_used_tables(); set_if_bigger(thd->lex->current_select->max_equal_elems, @@ -11843,7 +12044,8 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, while ((item= li++)) { Item *new_item; - if ((new_item= build_equal_items_for_cond(thd, item, inherited)) != item) + if ((new_item= build_equal_items_for_cond(thd, item, inherited, FALSE)) + != item) { /* This replacement happens only for standalone equalities */ /* @@ -12000,13 +12202,14 @@ static COND *build_equal_items_for_cond(THD *thd, COND *cond, static COND *build_equal_items(THD *thd, COND *cond, COND_EQUAL *inherited, List<TABLE_LIST> *join_list, - COND_EQUAL **cond_equal_ref) + COND_EQUAL **cond_equal_ref, + bool link_equal_fields) { COND_EQUAL *cond_equal= 0; if (cond) { - cond= build_equal_items_for_cond(thd, cond, inherited); + cond= build_equal_items_for_cond(thd, cond, inherited, link_equal_fields); cond->update_used_tables(); if (cond->type() == Item::COND_ITEM && ((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC) @@ -13454,7 +13657,8 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab, static COND * optimize_cond(JOIN *join, COND *conds, List<TABLE_LIST> *join_list, - Item::cond_result *cond_value, COND_EQUAL **cond_equal) + Item::cond_result *cond_value, COND_EQUAL **cond_equal, + int flags) { THD *thd= join->thd; DBUG_ENTER("optimize_cond"); @@ -13475,8 +13679,9 @@ optimize_cond(JOIN *join, COND *conds, List<TABLE_LIST> *join_list, multiple equality contains a constant. */ DBUG_EXECUTE("where", print_where(conds, "original", QT_ORDINARY);); - conds= build_equal_items(join->thd, conds, NULL, join_list, cond_equal); - DBUG_EXECUTE("where",print_where(conds,"after equal_items", QT_ORDINARY);); + conds= build_equal_items(join->thd, conds, NULL, join_list, cond_equal, + test(flags & OPT_LINK_EQUAL_FIELDS)); + DBUG_EXECUTE("where",print_where(conds,"after equal_items", QT_ORDINARY);); /* change field = field to field = const for each found field = const */ propagate_cond_constants(thd, (I_List<COND_CMP> *) 0, conds, conds); @@ -14258,6 +14463,9 @@ void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps) bitmap_init(&table->eq_join_set, (my_bitmap_map*) (bitmaps+ 3*bitmap_buffer_size(field_count)), field_count, FALSE); + bitmap_init(&table->cond_set, + (my_bitmap_map*) (bitmaps+ 4*bitmap_buffer_size(field_count)), + field_count, FALSE); /* write_set and all_set are copies of read_set */ table->def_write_set= table->def_read_set; table->s->all_set= table->def_read_set; @@ -14421,7 +14629,7 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields, &tmpname, (uint) strlen(path)+1, &group_buff, (group && ! using_unique_constraint ? param->group_length : 0), - &bitmaps, bitmap_buffer_size(field_count)*4, + &bitmaps, bitmap_buffer_size(field_count)*5, NullS)) { if (temp_pool_slot != MY_BIT_NONE) @@ -15176,7 +15384,7 @@ TABLE *create_virtual_tmp_table(THD *thd, List<Create_field> &field_list) &share, sizeof(*share), &field, (field_count + 1) * sizeof(Field*), &blob_field, (field_count+1) *sizeof(uint), - &bitmaps, bitmap_buffer_size(field_count)*4, + &bitmaps, bitmap_buffer_size(field_count)*5, NullS)) return 0; @@ -22068,7 +22276,13 @@ int JOIN::print_explain(select_result_sink *result, uint8 explain_flags, { float f= 0.0; if (examined_rows) - f= (float) (100.0 * tab->records_read / examined_rows); + { + double pushdown_cond_selectivity= tab->cond_selectivity; + if (pushdown_cond_selectivity == 1.0) + f= (float) (100.0 * tab->records_read / examined_rows); + else + f= (float) (100.0 * pushdown_cond_selectivity); + } set_if_smaller(f, 100.0); item_list.push_back(new Item_float(f, 2)); } diff --git a/sql/sql_select.h b/sql/sql_select.h index 638de926d75..87c09611009 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -292,6 +292,8 @@ typedef struct st_join_table { /* psergey-todo: make the below have type double, like POSITION::records_read? */ ha_rows records_read; + double cond_selectivity; + /* Startup cost for execution */ double startup_cost; @@ -762,6 +764,8 @@ typedef struct st_position :public Sql_alloc */ double records_read; + double cond_selectivity; + /* Cost accessing the table in course of the entire complete join execution, i.e. cost of one access method use (e.g. 'range' or 'ref' scan ) times @@ -1804,6 +1808,8 @@ void eliminate_tables(JOIN *join); /* Index Condition Pushdown entry point function */ void push_index_cond(JOIN_TAB *tab, uint keyno); +#define OPT_LINK_EQUAL_FIELDS 1 + /**************************************************************************** Temporary table support for SQL Runtime ***************************************************************************/ diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index e34b4b21819..5338632067a 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -26,6 +26,7 @@ #include "sql_base.h" #include "key.h" #include "sql_statistics.h" +#include "opt_range.h" #include "my_atomic.h" /* @@ -3054,3 +3055,51 @@ void set_statistics_for_table(THD *thd, TABLE *table) key_info->read_stats->get_avg_frequency(0) > 0.5); } } + + +double get_column_avg_frequency(Field * field) +{ + double res; + TABLE *table= field->table; + Column_statistics *col_stats= table->s->field[field->field_index]->read_stats; + + if (!col_stats) + res= table->stat_records(); + else + res= col_stats->get_avg_frequency(); + return res; +} + + +double get_column_range_cardinality(Field *field, + key_range *min_endp, + key_range *max_endp) +{ + double res; + TABLE *table= field->table; + Column_statistics *col_stats= table->field[field->field_index]->read_stats; + + if (!col_stats) + res= table->stat_records(); + else if (min_endp->length == max_endp->length && + !memcmp(min_endp->key, max_endp->key, min_endp->length)) + { + res= col_stats->get_avg_frequency(); + } + else + { + if (col_stats->min_value && col_stats->max_value) + { + store_key_image_to_rec(field, (uchar *) min_endp->key, min_endp->length); + double min_mp_pos= field->middle_point_pos(col_stats->min_value, + col_stats->max_value); + store_key_image_to_rec(field, (uchar *) max_endp->key, max_endp->length); + double max_mp_pos= field->middle_point_pos(col_stats->min_value, + col_stats->max_value); + res= table->stat_records() * (max_mp_pos - min_mp_pos); + } + else + res= table->stat_records(); + } + return res; +} diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 17f22cec4e5..6bf552b92a0 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -90,6 +90,12 @@ int rename_column_in_stat_tables(THD *thd, TABLE *tab, Field *col, const char *new_name); void set_statistics_for_table(THD *thd, TABLE *table); +double get_column_avg_frequency(Field * field); + +double get_column_range_cardinality(Field *field, + key_range *min_endp, + key_range *max_endp); + class Columns_statistics; class Index_statistics; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index fcd2df7b338..7f6a00f3c62 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1424,6 +1424,22 @@ static Sys_var_ulong Sys_optimizer_prune_level( SESSION_VAR(optimizer_prune_level), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 1), DEFAULT(1), BLOCK_SIZE(1)); +static Sys_var_ulong Sys_optimizer_use_condition_selectivity( + "optimizer_use_condition_selectivity", + "Controls selectivity of which conditions the optimizer takes into " + "account to calculate cardinality of a partial join when it searches " + "for the best execution plan " + "Meaning: " + "1 - use selectivity of index backed range conditions to calculate " + "cardinality of the partial join if the last joined table is " + "accessed by full table scan or an index scan " + "2 - use selectivity of index backed range conditions to calculate " + "cardinality of the partial join in any case " + "3 - additionally always use selectivity of range conditions that are " + "not backed by any index to calculate cardinality of the partial join", + SESSION_VAR(optimizer_use_condition_selectivity), CMD_LINE(REQUIRED_ARG), + VALID_RANGE(1, 3), DEFAULT(1), BLOCK_SIZE(1)); + /** Warns about deprecated value 63 */ static bool fix_optimizer_search_depth(sys_var *self, THD *thd, enum_var_type type) diff --git a/sql/table.cc b/sql/table.cc index a011fa845a7..e4dc2adef64 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2630,7 +2630,7 @@ partititon_err: /* Allocate bitmaps */ bitmap_size= share->column_bitmap_size; - if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size*5))) + if (!(bitmaps= (uchar*) alloc_root(&outparam->mem_root, bitmap_size*6))) goto err; bitmap_init(&outparam->def_read_set, (my_bitmap_map*) bitmaps, share->fields, FALSE); @@ -2642,8 +2642,12 @@ partititon_err: (my_bitmap_map*) (bitmaps+bitmap_size*3), share->fields, FALSE); bitmap_init(&outparam->eq_join_set, (my_bitmap_map*) (bitmaps+bitmap_size*4), share->fields, FALSE); + bitmap_init(&outparam->cond_set, + (my_bitmap_map*) (bitmaps+bitmap_size*5), share->fields, FALSE); outparam->default_column_bitmaps(); + outparam->cond_selectivity= 1.0; + /* The table struct is now initialized; Open the table */ error= 2; if (db_stat) diff --git a/sql/table.h b/sql/table.h index 1a567ae75d1..85b31af8a77 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1026,6 +1026,7 @@ public: my_bitmap_map *bitmap_init_value; MY_BITMAP def_read_set, def_write_set, def_vcol_set, tmp_set; MY_BITMAP eq_join_set; /* used to mark equi-joined fields */ + MY_BITMAP cond_set; /* used to mark fields from sargable conditions*/ MY_BITMAP *read_set, *write_set, *vcol_set; /* Active column sets */ /* The ID of the query that opened and is using this table. Has different @@ -1078,6 +1079,8 @@ public: */ ha_rows quick_condition_rows; + double cond_selectivity; + table_map map; /* ID bit of table (1,2,4,8,16...) */ uint lock_position; /* Position in MYSQL_LOCK.table */ |