diff options
author | Alexander Nozdrin <alik@sun.com> | 2010-05-14 22:11:25 +0400 |
---|---|---|
committer | Alexander Nozdrin <alik@sun.com> | 2010-05-14 22:11:25 +0400 |
commit | 6facd4cb13170e8695d9ae4f1e30611af0b0a472 (patch) | |
tree | 288d8745f229bd71630b65bb9f3cbd594db8d7e8 /sql | |
parent | 4333980a493e7062e108899b34cc809b03862c34 (diff) | |
download | mariadb-git-6facd4cb13170e8695d9ae4f1e30611af0b0a472.tar.gz |
Patch for Bug#27863 (excessive memory usage for many small queries in a
multiquery packet).
Background:
- a query can contain multiple SQL statements;
- the server frees resources allocated to process a query when the
whole query is handled. In other words, resources allocated to process
one SQL statement from a multi-statement query are freed when all SQL
statements are handled.
The problem was that the parser allocated a buffer of size of the whole
query for each SQL statement in a multi-statement query. Thus, if a query
had many SQL-statements (so, the query was long), but each SQL statement
was short, ther parser tried to allocate huge amount of memory (number of
small SQL statements * length of the whole query).
The memory was allocated for a so-called "cpp buffer", which is intended to
store pre-processed SQL statement -- SQL text without version specific
comments.
The fix is to allocate memory for the "cpp buffer" once for all SQL
statements (once for a query).
Diffstat (limited to 'sql')
-rw-r--r-- | sql/ha_ndbcluster_binlog.cc | 4 | ||||
-rw-r--r-- | sql/log_event.cc | 4 | ||||
-rw-r--r-- | sql/sql_lex.cc | 79 | ||||
-rw-r--r-- | sql/sql_lex.h | 26 | ||||
-rw-r--r-- | sql/sql_parse.cc | 36 | ||||
-rw-r--r-- | sql/sql_parse.h | 2 | ||||
-rw-r--r-- | sql/sql_signal.cc | 4 |
7 files changed, 95 insertions, 60 deletions
diff --git a/sql/ha_ndbcluster_binlog.cc b/sql/ha_ndbcluster_binlog.cc index edb6aeebb5c..330891c1a10 100644 --- a/sql/ha_ndbcluster_binlog.cc +++ b/sql/ha_ndbcluster_binlog.cc @@ -263,7 +263,7 @@ static void run_query(THD *thd, char *buf, char *end, ulonglong save_thd_options= thd->variables.option_bits; DBUG_ASSERT(sizeof(save_thd_options) == sizeof(thd->variables.option_bits)); NET save_thd_net= thd->net; - const char* found_semicolon= NULL; + Parser_state parser_state(thd, thd->query(), thd->query_length()); bzero((char*) &thd->net, sizeof(NET)); thd->set_query(buf, (uint) (end - buf)); @@ -277,7 +277,7 @@ static void run_query(THD *thd, char *buf, char *end, DBUG_ASSERT(!thd->in_sub_stmt); DBUG_ASSERT(!thd->locked_tables_mode); - mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon); + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); if (no_print_error && thd->is_slave_error) { diff --git a/sql/log_event.cc b/sql/log_event.cc index 023dc3f66c3..2fdc1da4249 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -3290,8 +3290,8 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli, thd->table_map_for_update= (table_map)table_map_for_update; /* Execute the query (note that we bypass dispatch_command()) */ - const char* found_semicolon= NULL; - mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon); + Parser_state parser_state(thd, thd->query(), thd->query_length()); + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); log_slow_statement(thd); /* diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 13f85b24299..860296b4050 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -142,37 +142,64 @@ st_parsing_options::reset() allows_derived= TRUE; } + +/** + Perform initialization of Lex_input_stream instance. + + Basically, a buffer for pre-processed query. This buffer should be large + enough to keep multi-statement query. The allocation is done once in the + Lex_input_stream constructor in order to prevent memory pollution when + the server is processing large multi-statement queries. + + @todo Check return value of THD::alloc(). +*/ + Lex_input_stream::Lex_input_stream(THD *thd, const char* buffer, unsigned int length) -: m_thd(thd), - yylineno(1), - yytoklen(0), - yylval(NULL), - lookahead_token(-1), - lookahead_yylval(NULL), - m_ptr(buffer), - m_tok_start(NULL), - m_tok_end(NULL), - m_end_of_query(buffer + length), - m_tok_start_prev(NULL), - m_buf(buffer), - m_buf_length(length), - m_echo(TRUE), - m_cpp_tok_start(NULL), - m_cpp_tok_start_prev(NULL), - m_cpp_tok_end(NULL), - m_body_utf8(NULL), - m_cpp_utf8_processed_ptr(NULL), - next_state(MY_LEX_START), - found_semicolon(NULL), - ignore_space(test(thd->variables.sql_mode & MODE_IGNORE_SPACE)), - stmt_prepare_mode(FALSE), - multi_statements(TRUE), - in_comment(NO_COMMENT), - m_underscore_cs(NULL) + :m_thd(thd) { m_cpp_buf= (char*) thd->alloc(length + 1); + reset(buffer, length); +} + + +/** + Prepare Lex_input_stream instance state for use for handling next SQL statement. + + It should be called between two statements in a multi-statement query. + The operation resets the input stream to the beginning-of-parse state, + but does not reallocate m_cpp_buf. +*/ + +void +Lex_input_stream::reset(const char *buffer, unsigned int length) +{ + yylineno= 1; + yytoklen= 0; + yylval= NULL; + lookahead_token= -1; + lookahead_yylval= NULL; + m_ptr= buffer; + m_tok_start= NULL; + m_tok_end= NULL; + m_end_of_query= buffer + length; + m_tok_start_prev= NULL; + m_buf= buffer; + m_buf_length= length; + m_echo= TRUE; + m_cpp_tok_start= NULL; + m_cpp_tok_start_prev= NULL; + m_cpp_tok_end= NULL; + m_body_utf8= NULL; + m_cpp_utf8_processed_ptr= NULL; + next_state= MY_LEX_START; + found_semicolon= NULL; + ignore_space= test(m_thd->variables.sql_mode & MODE_IGNORE_SPACE); + stmt_prepare_mode= FALSE; + multi_statements= TRUE; + in_comment=NO_COMMENT; + m_underscore_cs= NULL; m_cpp_ptr= m_cpp_buf; } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 85ae2697f21..5230753e374 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -1377,6 +1377,8 @@ public: Lex_input_stream(THD *thd, const char* buff, unsigned int length); ~Lex_input_stream(); + void reset(const char *buff, unsigned int length); + /** Set the echo mode. @@ -2206,8 +2208,8 @@ struct LEX: public Query_tables_list class Set_signal_information { public: - /** Constructor. */ - Set_signal_information(); + /** Empty default constructor, use clear() */ + Set_signal_information() {} /** Copy constructor. */ Set_signal_information(const Set_signal_information& set); @@ -2220,7 +2222,7 @@ public: void clear(); /** - For each contition item assignment, m_item[] contains the parsed tree + For each condition item assignment, m_item[] contains the parsed tree that represents the expression assigned, if any. m_item[] is an array indexed by Diag_condition_item_name. */ @@ -2237,8 +2239,16 @@ class Yacc_state { public: Yacc_state() - : yacc_yyss(NULL), yacc_yyvs(NULL) - {} + { + reset(); + } + + void reset() + { + yacc_yyss= NULL; + yacc_yyvs= NULL; + m_set_signal_info.clear(); + } ~Yacc_state(); @@ -2283,6 +2293,12 @@ public: Lex_input_stream m_lip; Yacc_state m_yacc; + + void reset(const char *found_semicolon, unsigned int length) + { + m_lip.reset(found_semicolon, length); + m_yacc.reset(); + } }; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3c39a414e1f..c787ee02f28 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -458,7 +458,6 @@ static void handle_bootstrap_impl(THD *thd) { MYSQL_FILE *file= bootstrap_file; char *buff; - const char* found_semicolon= NULL; DBUG_ENTER("handle_bootstrap"); @@ -531,7 +530,8 @@ static void handle_bootstrap_impl(THD *thd) mode we have only one thread. */ thd->set_time(); - mysql_parse(thd, thd->query(), length, & found_semicolon); + Parser_state parser_state(thd, thd->query(), length); + mysql_parse(thd, thd->query(), length, &parser_state); close_thread_tables(thd); // Free tables bootstrap_error= thd->is_error(); @@ -1074,19 +1074,21 @@ bool dispatch_command(enum enum_server_command command, THD *thd, (char *) thd->security_ctx->host_or_ip); char *packet_end= thd->query() + thd->query_length(); /* 'b' stands for 'buffer' parameter', special for 'my_snprintf' */ - const char* end_of_stmt= NULL; general_log_write(thd, command, thd->query(), thd->query_length()); DBUG_PRINT("query",("%-.4096s",thd->query())); #if defined(ENABLED_PROFILING) thd->profiling.set_query_source(thd->query(), thd->query_length()); #endif + Parser_state parser_state(thd, thd->query(), thd->query_length()); - mysql_parse(thd, thd->query(), thd->query_length(), &end_of_stmt); + mysql_parse(thd, thd->query(), thd->query_length(), &parser_state); - while (!thd->killed && (end_of_stmt != NULL) && ! thd->is_error()) + while (!thd->killed && (parser_state.m_lip.found_semicolon != NULL) && + ! thd->is_error()) { - char *beginning_of_next_stmt= (char*) end_of_stmt; + char *beginning_of_next_stmt= (char*) + parser_state.m_lip.found_semicolon; thd->protocol->end_statement(); query_cache_end_of_result(thd); @@ -1127,8 +1129,9 @@ bool dispatch_command(enum enum_server_command command, THD *thd, */ statistic_increment(thd->status_var.questions, &LOCK_status); thd->set_time(); /* Reset the query start time. */ + parser_state.reset(beginning_of_next_stmt, length); /* TODO: set thd->lex->sql_command to SQLCOM_END here */ - mysql_parse(thd, beginning_of_next_stmt, length, &end_of_stmt); + mysql_parse(thd, beginning_of_next_stmt, length, &parser_state); } DBUG_PRINT("info",("query ready")); @@ -5721,7 +5724,7 @@ void mysql_init_multi_delete(LEX *lex) */ void mysql_parse(THD *thd, const char *inBuf, uint length, - const char ** found_semicolon) + Parser_state *parser_state) { int error; DBUG_ENTER("mysql_parse"); @@ -5751,10 +5754,7 @@ void mysql_parse(THD *thd, const char *inBuf, uint length, { LEX *lex= thd->lex; - Parser_state parser_state(thd, inBuf, length); - - bool err= parse_sql(thd, & parser_state, NULL); - *found_semicolon= parser_state.m_lip.found_semicolon; + bool err= parse_sql(thd, parser_state, NULL); if (!err) { @@ -5769,6 +5769,7 @@ void mysql_parse(THD *thd, const char *inBuf, uint length, { if (! thd->is_error()) { + const char *found_semicolon= parser_state->m_lip.found_semicolon; /* Binlog logs a string starting from thd->query and having length thd->query_length; so we set thd->query_length correctly (to not @@ -5779,12 +5780,12 @@ void mysql_parse(THD *thd, const char *inBuf, uint length, PROCESSLIST. Note that we don't need LOCK_thread_count to modify query_length. */ - if (*found_semicolon && (ulong) (*found_semicolon - thd->query())) + if (found_semicolon && (ulong) (found_semicolon - thd->query())) thd->set_query_inner(thd->query(), - (uint32) (*found_semicolon - + (uint32) (found_semicolon - thd->query() - 1)); /* Actually execute the query */ - if (*found_semicolon) + if (found_semicolon) { lex->safe_to_cache_query= 0; thd->server_status|= SERVER_MORE_RESULTS_EXISTS; @@ -5821,11 +5822,6 @@ void mysql_parse(THD *thd, const char *inBuf, uint length, thd->cleanup_after_query(); DBUG_ASSERT(thd->change_list.is_empty()); } - else - { - /* There are no multi queries in the cache. */ - *found_semicolon= NULL; - } DBUG_VOID_RETURN; } diff --git a/sql/sql_parse.h b/sql/sql_parse.h index 3c34b063d91..e1543a09549 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -84,7 +84,7 @@ bool is_log_table_write_query(enum enum_sql_command command); bool alloc_query(THD *thd, const char *packet, uint packet_length); void mysql_init_select(LEX *lex); void mysql_parse(THD *thd, const char *inBuf, uint length, - const char ** semicolon); + Parser_state *parser_state); void mysql_reset_thd_for_next_command(THD *thd); bool mysql_new_select(LEX *lex, bool move_down); void create_select_for_variable(const char *var_name); diff --git a/sql/sql_signal.cc b/sql/sql_signal.cc index 8ba9cb369aa..f340da373e8 100644 --- a/sql/sql_signal.cc +++ b/sql/sql_signal.cc @@ -75,10 +75,6 @@ const LEX_STRING Diag_statement_item_names[]= { C_STRING_WITH_LEN("TRANSACTION_ACTIVE") } }; -Set_signal_information::Set_signal_information() -{ - clear(); -} Set_signal_information::Set_signal_information( const Set_signal_information& set) |