From 7b047a31a09a7999353232cb6407b4e6338d5b70 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Mon, 8 Nov 2010 13:43:54 +0200 Subject: Make SQLString reallocation addaptive Avoid doing reallocs Prealloc some strings / provide extension allocation size to some strings This gave a 25 % speedup in some mysql-test-run tests. mysys/safemalloc.c: More DBUG_PRINT sql/net_serv.cc: Make all mallocs() look the similar. (just-for-safety fix) sql/protocol.cc: Ensure that communication packet buffer is allocated. (It's freed by stored precedures and some DLL statements) sql/sp.cc: Fixed valgrind warning sql/sql_select.cc: Set extent allocation for buffer that has a lot of append() calls. sql/sql_show.cc: Fixed wrong usage of string buffer. Old code worked in test suite 'just-by-chance' sql/sql_string.cc: Call realloc_with_extra_if_needed() in append() functions. sql/sql_string.h: Added 'extra_alloc' member, to specify chunck size for realloc(). extra_alloc is addaptive to catch cases where preallocation of buffers is not done properly. Simplified free() to allow compiler to optimize things better (and to keep things consistent). Fixed shrink() to take into account the extra memory added to the Alloced_length in realloc(). This saves us a realloc() per query. sql/sql_test.cc: Set extent allocation for buffer that has a lot of append() calls. sql/table.cc: Set extent allocation for buffer that has a lot of append() calls. --- mysys/safemalloc.c | 4 ++++ sql/net_serv.cc | 4 ++-- sql/protocol.cc | 3 +++ sql/sp.cc | 2 +- sql/sql_select.cc | 1 + sql/sql_show.cc | 2 +- sql/sql_string.cc | 18 +++++++++--------- sql/sql_string.h | 48 ++++++++++++++++++++++++++++++------------------ sql/sql_test.cc | 3 ++- sql/table.cc | 4 +++- 10 files changed, 56 insertions(+), 33 deletions(-) diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 0d0ae2dabe5..0e489263c69 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -224,6 +224,8 @@ void *_myrealloc(register void *ptr, register size_t size, struct st_irem *irem; char *data; DBUG_ENTER("_myrealloc"); + DBUG_PRINT("my",("ptr: 0x%lx size: %lu my_flags: %d", (long) ptr, + (ulong) size, MyFlags)); if (!ptr && (MyFlags & MY_ALLOW_ZERO_PTR)) DBUG_RETURN(_mymalloc(size, filename, lineno, MyFlags)); @@ -245,6 +247,8 @@ void *_myrealloc(register void *ptr, register size_t size, (void) fflush(stderr); DBUG_RETURN((uchar*) NULL); } + DBUG_PRINT("my", ("old_size: %lu -> new_size: %lu", + (ulong) irem->datasize, (ulong) size)); if ((data= _mymalloc(size,filename,lineno,MyFlags))) /* Allocate new area */ { diff --git a/sql/net_serv.cc b/sql/net_serv.cc index 4796a5601bf..6a4079b1fef 100644 --- a/sql/net_serv.cc +++ b/sql/net_serv.cc @@ -116,7 +116,7 @@ my_bool my_net_init(NET *net, Vio* vio) net->vio = vio; my_net_local_init(net); /* Set some limits */ if (!(net->buff=(uchar*) my_malloc((size_t) net->max_packet+ - NET_HEADER_SIZE + COMP_HEADER_SIZE, + NET_HEADER_SIZE + COMP_HEADER_SIZE +1, MYF(MY_WME)))) DBUG_RETURN(1); net->buff_end=net->buff+net->max_packet; @@ -580,7 +580,7 @@ net_real_write(NET *net,const uchar *packet, size_t len) uchar *b; uint header_length=NET_HEADER_SIZE+COMP_HEADER_SIZE; if (!(b= (uchar*) my_malloc(len + NET_HEADER_SIZE + - COMP_HEADER_SIZE, MYF(MY_WME)))) + COMP_HEADER_SIZE + 1, MYF(MY_WME)))) { net->error= 2; net->last_errno= ER_OUT_OF_RESOURCES; diff --git a/sql/protocol.cc b/sql/protocol.cc index 8a9d712077d..70b171fa652 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -632,6 +632,9 @@ bool Protocol::send_fields(List *list, uint flags) uint count= 0; #endif + /* We have to reallocate it here as a stored procedure may have reset it */ + (void) local_packet->alloc(thd->variables.net_buffer_length); + while ((item=it++)) { char *pos; diff --git a/sql/sp.cc b/sql/sp.cc index ebf2b3c360d..68e2f19fd47 100644 --- a/sql/sp.cc +++ b/sql/sp.cc @@ -785,7 +785,7 @@ db_load_routine(THD *thd, int type, sp_name *name, sp_head **sphp, { Parser_state parser_state; - if (parser_state.init(thd, defstr.c_ptr(), defstr.length())) + if (parser_state.init(thd, defstr.c_ptr_safe(), defstr.length())) { ret= SP_INTERNAL_ERROR; goto end; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index e185996126d..e6be979becb 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -15997,6 +15997,7 @@ change_to_use_tmp_fields(THD *thd, Item **ref_pointer_array, char buff[256]; String str(buff,sizeof(buff),&my_charset_bin); str.length(0); + str.extra_allocation(1024); item->print(&str, QT_ORDINARY); item_field->name= sql_strmake(str.ptr(),str.length()); } diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 8f1f938d36e..265c86f55e7 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -4329,7 +4329,7 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables, base_type [(dimension)] [unsigned] [zerofill]. For DATA_TYPE column we extract only base type. */ - tmp_buff= strchr(type.ptr(), '('); + tmp_buff= strchr(type.c_ptr_safe(), '('); if (!tmp_buff) /* if there is no dimention part then check the presence of diff --git a/sql/sql_string.cc b/sql/sql_string.cc index eafd8502706..b359b2a7168 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -411,7 +411,7 @@ bool String::append(const String &s) { if (s.length()) { - if (realloc(str_length+s.length())) + if (realloc_with_extra_if_needed(str_length+s.length())) return TRUE; memcpy(Ptr+str_length,s.ptr(),s.length()); str_length+=s.length(); @@ -436,7 +436,7 @@ bool String::append(const char *s,uint32 arg_length) { uint32 add_length=arg_length * str_charset->mbmaxlen; uint dummy_errors; - if (realloc(str_length+ add_length)) + if (realloc_with_extra_if_needed(str_length+ add_length)) return TRUE; str_length+= copy_and_convert(Ptr+str_length, add_length, str_charset, s, arg_length, &my_charset_latin1, @@ -447,7 +447,7 @@ bool String::append(const char *s,uint32 arg_length) /* For an ASCII compatinble string we can just append. */ - if (realloc(str_length+arg_length)) + if (realloc_with_extra_if_needed(str_length+arg_length)) return TRUE; memcpy(Ptr+str_length,s,arg_length); str_length+=arg_length; @@ -478,14 +478,14 @@ bool String::append(const char *s,uint32 arg_length, CHARSET_INFO *cs) { uint32 add_length= arg_length / cs->mbminlen * str_charset->mbmaxlen; uint dummy_errors; - if (realloc(str_length + add_length)) + if (realloc_with_extra_if_needed(str_length + add_length)) return TRUE; str_length+= copy_and_convert(Ptr+str_length, add_length, str_charset, s, arg_length, cs, &dummy_errors); } else { - if (realloc(str_length + arg_length)) + if (realloc_with_extra_if_needed(str_length + arg_length)) return TRUE; memcpy(Ptr + str_length, s, arg_length); str_length+= arg_length; @@ -497,7 +497,7 @@ bool String::append(const char *s,uint32 arg_length, CHARSET_INFO *cs) #ifdef TO_BE_REMOVED bool String::append(FILE* file, uint32 arg_length, myf my_flags) { - if (realloc(str_length+arg_length)) + if (realloc_with_extra_if_needed(str_length+arg_length)) return TRUE; if (my_fread(file, (uchar*) Ptr + str_length, arg_length, my_flags)) { @@ -511,7 +511,7 @@ bool String::append(FILE* file, uint32 arg_length, myf my_flags) bool String::append(IO_CACHE* file, uint32 arg_length) { - if (realloc(str_length+arg_length)) + if (realloc_with_extra_if_needed(str_length+arg_length)) return TRUE; if (my_b_read(file, (uchar*) Ptr + str_length, arg_length)) { @@ -527,7 +527,7 @@ bool String::append_with_prefill(const char *s,uint32 arg_length, { int t_length= arg_length > full_length ? arg_length : full_length; - if (realloc(str_length + t_length)) + if (realloc_with_extra_if_needed(str_length + t_length)) return TRUE; t_length= full_length - arg_length; if (t_length > 0) @@ -636,7 +636,7 @@ bool String::replace(uint32 offset,uint32 arg_length, { if (diff) { - if (realloc(str_length+(uint32) diff)) + if (realloc_with_extra_if_needed(str_length+(uint32) diff)) return TRUE; bmove_upp((uchar*) Ptr+str_length+diff, (uchar*) Ptr+str_length, str_length-offset-arg_length); diff --git a/sql/sql_string.h b/sql/sql_string.h index e595de47f99..dc9122a4fcb 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -53,23 +53,24 @@ uint convert_to_printable(char *to, size_t to_len, class String { char *Ptr; - uint32 str_length,Alloced_length; + uint32 str_length,Alloced_length, extra_alloc; bool alloced; CHARSET_INFO *str_charset; public: String() { - Ptr=0; str_length=Alloced_length=0; alloced=0; + Ptr=0; str_length=Alloced_length=extra_alloc=0; alloced=0; str_charset= &my_charset_bin; } String(uint32 length_arg) { - alloced=0; Alloced_length=0; (void) real_alloc(length_arg); + alloced=0; Alloced_length= extra_alloc= 0; (void) real_alloc(length_arg); str_charset= &my_charset_bin; } String(const char *str, CHARSET_INFO *cs) { - Ptr=(char*) str; str_length=(uint) strlen(str); Alloced_length=0; alloced=0; + Ptr=(char*) str; str_length= (uint32) strlen(str); + Alloced_length= extra_alloc= 0; alloced=0; str_charset=cs; } /* @@ -79,18 +80,18 @@ public: */ String(const char *str,uint32 len, CHARSET_INFO *cs) { - Ptr=(char*) str; str_length=len; Alloced_length=0; alloced=0; + Ptr=(char*) str; str_length=len; Alloced_length= extra_alloc=0; alloced=0; str_charset=cs; } String(char *str,uint32 len, CHARSET_INFO *cs) { - Ptr=(char*) str; Alloced_length=str_length=len; alloced=0; + Ptr=(char*) str; Alloced_length=str_length=len; extra_alloc= 0; alloced=0; str_charset=cs; } String(const String &str) { Ptr=str.Ptr ; str_length=str.str_length ; - Alloced_length=str.Alloced_length; alloced=0; + Alloced_length=str.Alloced_length; extra_alloc= 0; alloced=0; str_charset=str.str_charset; } static void *operator new(size_t size, MEM_ROOT *mem_root) throw () @@ -106,8 +107,10 @@ public: inline CHARSET_INFO *charset() const { return str_charset; } inline uint32 length() const { return str_length;} inline uint32 alloced_length() const { return Alloced_length;} + inline uint32 extra_allocation() const { return extra_alloc;} inline char& operator [] (uint32 i) const { return Ptr[i]; } inline void length(uint32 len) { str_length=len ; } + inline void extra_allocation(uint32 len) { extra_alloc= len; } inline bool is_empty() const { return (str_length == 0); } inline void mark_as_const() { Alloced_length= 0;} inline const char *ptr() const { return Ptr; } @@ -136,23 +139,21 @@ public: { DBUG_ASSERT(&str != this); free(); - Ptr=(char*) str.ptr()+offset; str_length=arg_length; alloced=0; + Ptr=(char*) str.ptr()+offset; str_length=arg_length; if (str.Alloced_length) Alloced_length=str.Alloced_length-offset; - else - Alloced_length=0; str_charset=str.str_charset; } inline void set(char *str,uint32 arg_length, CHARSET_INFO *cs) { free(); - Ptr=(char*) str; str_length=Alloced_length=arg_length ; alloced=0; + Ptr=(char*) str; str_length=Alloced_length=arg_length; str_charset=cs; } inline void set(const char *str,uint32 arg_length, CHARSET_INFO *cs) { free(); - Ptr=(char*) str; str_length=arg_length; Alloced_length=0 ; alloced=0; + Ptr=(char*) str; str_length=arg_length; str_charset=cs; } bool set_ascii(const char *str, uint32 arg_length); @@ -203,11 +204,11 @@ public: if (alloced) { alloced=0; - Alloced_length=0; my_free(Ptr,MYF(0)); - Ptr=0; - str_length=0; /* Safety */ } + Alloced_length= extra_alloc= 0; + Ptr=0; + str_length=0; /* Safety */ } inline bool alloc(uint32 arg_length) { @@ -217,9 +218,21 @@ public: } bool real_alloc(uint32 arg_length); // Empties old string bool realloc(uint32 arg_length); - inline void shrink(uint32 arg_length) // Shrink buffer + bool realloc_with_extra(uint32 arg_length) + { + if (extra_alloc < 4096) + extra_alloc= extra_alloc*2+128; + return realloc(arg_length + extra_alloc); + } + bool realloc_with_extra_if_needed(uint32 arg_length) { if (arg_length < Alloced_length) + return 0; + return realloc_with_extra(arg_length); + } + inline void shrink(uint32 arg_length) // Shrink buffer + { + if (ALIGN_SIZE(arg_length+1) < Alloced_length) { char *new_ptr; if (!(new_ptr=(char*) my_realloc(Ptr,arg_length,MYF(0)))) @@ -246,7 +259,6 @@ public: DBUG_ASSERT(!s.uses_buffer_owned_by(this)); free(); Ptr=s.Ptr ; str_length=s.str_length ; Alloced_length=s.Alloced_length; - alloced=0; } return *this; } @@ -281,7 +293,7 @@ public: } else { - if (realloc(str_length+1)) + if (realloc_with_extra(str_length + 1)) return 1; Ptr[str_length++]=chr; } diff --git a/sql/sql_test.cc b/sql/sql_test.cc index 541fcc155af..185c0a02799 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -57,9 +57,10 @@ print_where(COND *cond,const char *info, enum_query_type query_type) { if (cond) { - char buff[256]; + char buff[1024]; String str(buff,(uint32) sizeof(buff), system_charset_info); str.length(0); + str.extra_allocation(1024); cond->print(&str, query_type); str.append('\0'); DBUG_LOCK_FILE; diff --git a/sql/table.cc b/sql/table.cc index 7a2581d5ab8..45c1f5ab378 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3328,11 +3328,13 @@ Table_check_intact::check(TABLE *table, const TABLE_FIELD_DEF *table_def) is backward compatible. */ } - char buffer[STRING_BUFFER_USUAL_SIZE]; + char buffer[1024]; for (i=0 ; i < table_def->count; i++, field_def++) { String sql_type(buffer, sizeof(buffer), system_charset_info); sql_type.length(0); + /* Allocate min 256 characters at once */ + sql_type.extra_allocation(256); if (i < table->s->fields) { Field *field= table->field[i]; -- cgit v1.2.1